{"id":8403372,"name":"@microsoft/tiktokenizer","ecosystem":"npm","description":"Tokenizer for OpenAI large language models.","homepage":"https://github.com/Microsoft/Tokenizer#readme","licenses":"MIT","normalized_licenses":["MIT"],"repository_url":"https://github.com/Microsoft/Tokenizer","keywords_array":[],"namespace":"microsoft","versions_count":10,"first_release_published_at":"2023-10-09T21:54:01.392Z","latest_release_published_at":"2025-04-25T18:17:08.927Z","latest_release_number":"1.0.10","last_synced_at":"2026-04-13T05:13:38.240Z","created_at":"2023-10-09T21:55:14.149Z","updated_at":"2026-04-13T05:13:38.240Z","registry_url":"https://www.npmjs.com/package/@microsoft/tiktokenizer","install_command":"npm install @microsoft/tiktokenizer","documentation_url":null,"metadata":{"funding":null,"dist-tags":{"latest":"1.0.10"}},"repo_metadata":{"id":151681970,"uuid":"620176227","full_name":"microsoft/Tokenizer","owner":"microsoft","description":"Typescript and .NET implementation of BPE tokenizer for OpenAI LLMs.","archived":false,"fork":false,"pushed_at":"2024-07-18T23:31:28.000Z","size":2220,"stargazers_count":130,"open_issues_count":2,"forks_count":27,"subscribers_count":9,"default_branch":"main","last_synced_at":"2024-08-02T16:46:12.680Z","etag":null,"topics":["ai","gpt","llm","openai","tokenizer"],"latest_commit_sha":null,"homepage":"https://github.com/microsoft/Tokenizer/","language":"C#","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":"mit","status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/microsoft.png","metadata":{"files":{"readme":"README.md","changelog":null,"contributing":"CONTRIBUTING.md","funding":null,"license":"LICENSE","code_of_conduct":"CODE_OF_CONDUCT.md","threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":"SECURITY.md","support":"SUPPORT.md","governance":null,"roadmap":null,"authors":null,"dei":null,"publiccode":null,"codemeta":null}},"created_at":"2023-03-28T07:14:39.000Z","updated_at":"2024-08-02T02:43:03.000Z","dependencies_parsed_at":"2024-01-11T06:26:28.771Z","dependency_job_id":"5842d478-120f-4a33-bbcd-27bb54841202","html_url":"https://github.com/microsoft/Tokenizer","commit_stats":null,"previous_names":[],"tags_count":4,"template":false,"template_full_name":null,"repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/microsoft%2FTokenizer","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/microsoft%2FTokenizer/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/microsoft%2FTokenizer/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/microsoft%2FTokenizer/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/microsoft","download_url":"https://codeload.github.com/microsoft/Tokenizer/tar.gz/refs/heads/main","host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":213706898,"owners_count":15626712,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2022-07-04T15:15:14.044Z","host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"},"owner_record":{"login":"microsoft","name":"Microsoft","uuid":"6154722","kind":"organization","description":"Open source projects and samples from Microsoft","email":"opensource@microsoft.com","website":"https://opensource.microsoft.com","location":"Redmond, WA","twitter":"OpenAtMicrosoft","company":null,"icon_url":"https://avatars.githubusercontent.com/u/6154722?v=4","repositories_count":6227,"last_synced_at":"2024-04-08T00:10:14.916Z","metadata":{"has_sponsors_listing":false},"html_url":"https://github.com/microsoft","funding_links":[],"total_stars":2659258,"followers":63990,"following":0,"created_at":"2022-11-02T16:20:12.368Z","updated_at":"2024-04-08T00:20:04.754Z","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/microsoft","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/microsoft/repositories"},"tags":[{"name":"v1.3.3","sha":"2c9ba5d343de52eb27521afef7c0c2f0f76c9c52","kind":"commit","published_at":"2024-01-11T18:09:30.000Z","download_url":"https://codeload.github.com/microsoft/Tokenizer/tar.gz/v1.3.3","html_url":"https://github.com/microsoft/Tokenizer/releases/tag/v1.3.3","dependencies_parsed_at":null,"dependency_job_id":null,"tag_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/microsoft%2FTokenizer/tags/v1.3.3","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/microsoft%2FTokenizer/tags/v1.3.3/manifests"},{"name":"v1.3.2","sha":"3834de5a3b3b4f692a453026f482eb6dac4e5a29","kind":"commit","published_at":"2023-05-24T16:06:24.000Z","download_url":"https://codeload.github.com/microsoft/Tokenizer/tar.gz/v1.3.2","html_url":"https://github.com/microsoft/Tokenizer/releases/tag/v1.3.2","dependencies_parsed_at":null,"dependency_job_id":null,"tag_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/microsoft%2FTokenizer/tags/v1.3.2","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/microsoft%2FTokenizer/tags/v1.3.2/manifests"},{"name":"v1.3.1","sha":"dc52d2e85763e450a7ed8fa12c2d79c75fc843e2","kind":"commit","published_at":"2023-05-12T17:05:52.000Z","download_url":"https://codeload.github.com/microsoft/Tokenizer/tar.gz/v1.3.1","html_url":"https://github.com/microsoft/Tokenizer/releases/tag/v1.3.1","dependencies_parsed_at":null,"dependency_job_id":null,"tag_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/microsoft%2FTokenizer/tags/v1.3.1","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/microsoft%2FTokenizer/tags/v1.3.1/manifests"},{"name":"v1.3.0","sha":"858c5155997237088f4f24d1b0f732ea84224215","kind":"commit","published_at":"2023-04-06T18:17:52.000Z","download_url":"https://codeload.github.com/microsoft/Tokenizer/tar.gz/v1.3.0","html_url":"https://github.com/microsoft/Tokenizer/releases/tag/v1.3.0","dependencies_parsed_at":null,"dependency_job_id":null,"tag_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/microsoft%2FTokenizer/tags/v1.3.0","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/microsoft%2FTokenizer/tags/v1.3.0/manifests"}]},"repo_metadata_updated_at":"2024-07-30T01:35:11.784Z","dependent_packages_count":0,"downloads":303991,"downloads_period":"last-month","dependent_repos_count":1,"rankings":{"downloads":2.796509583321756,"dependent_repos_count":10.30985113979121,"dependent_packages_count":51.90012905202741,"stargazers_count":5.218851062086598,"forks_count":5.273643064357867,"docker_downloads_count":null,"average":15.099796780316968},"purl":"pkg:npm/%40microsoft/tiktokenizer","advisories":[],"docker_usage_url":"https://docker.ecosyste.ms/usage/npm/@microsoft/tiktokenizer","docker_dependents_count":null,"docker_downloads_count":null,"usage_url":"https://repos.ecosyste.ms/usage/npm/@microsoft/tiktokenizer","dependent_repositories_url":"https://repos.ecosyste.ms/api/v1/usage/npm/@microsoft/tiktokenizer/dependencies","status":null,"funding_links":[],"critical":null,"issue_metadata":{"last_synced_at":"2024-07-11T20:56:06.201Z","issues_count":11,"pull_requests_count":38,"avg_time_to_close_issue":809338.8,"avg_time_to_close_pull_request":366574.55555555556,"issues_closed_count":10,"pull_requests_closed_count":36,"pull_request_authors_count":12,"issue_authors_count":8,"avg_comments_per_issue":0.8181818181818182,"avg_comments_per_pull_request":0.42105263157894735,"merged_pull_requests_count":31,"bot_issues_count":0,"bot_pull_requests_count":2,"past_year_issues_count":10,"past_year_pull_requests_count":25,"past_year_avg_time_to_close_issue":500549.1111111111,"past_year_avg_time_to_close_pull_request":341163.39130434784,"past_year_issues_closed_count":9,"past_year_pull_requests_closed_count":23,"past_year_pull_request_authors_count":11,"past_year_issue_authors_count":7,"past_year_avg_comments_per_issue":0.8,"past_year_avg_comments_per_pull_request":0.48,"past_year_bot_issues_count":0,"past_year_bot_pull_requests_count":2,"past_year_merged_pull_requests_count":18,"issues_url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/repositories/microsoft%2FTokenizer/issues","maintainers":[{"login":"shengyfu","count":14,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/shengyfu"},{"login":"connor4312","count":10,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/connor4312"},{"login":"mjbvz","count":4,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/mjbvz"},{"login":"sbatten","count":3,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/sbatten"},{"login":"ericstj","count":2,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/ericstj"},{"login":"andreamah","count":1,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/andreamah"},{"login":"lramos15","count":1,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/lramos15"}],"active_maintainers":[{"login":"connor4312","count":10,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/connor4312"},{"login":"mjbvz","count":4,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/mjbvz"},{"login":"sbatten","count":3,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/sbatten"},{"login":"shengyfu","count":3,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/shengyfu"},{"login":"ericstj","count":2,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/ericstj"},{"login":"andreamah","count":1,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/andreamah"},{"login":"lramos15","count":1,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/lramos15"}]},"versions_url":"https://packages.ecosyste.ms/api/v1/registries/npmjs.org/packages/@microsoft%2Ftiktokenizer/versions","version_numbers_url":"https://packages.ecosyste.ms/api/v1/registries/npmjs.org/packages/@microsoft%2Ftiktokenizer/version_numbers","dependent_packages_url":"https://packages.ecosyste.ms/api/v1/registries/npmjs.org/packages/@microsoft%2Ftiktokenizer/dependent_packages","related_packages_url":"https://packages.ecosyste.ms/api/v1/registries/npmjs.org/packages/@microsoft%2Ftiktokenizer/related_packages","codemeta_url":"https://packages.ecosyste.ms/api/v1/registries/npmjs.org/packages/@microsoft%2Ftiktokenizer/codemeta","maintainers":[{"uuid":"shengyfu","login":"shengyfu","name":null,"email":"shengyfu@microsoft.com","url":null,"packages_count":1,"html_url":"https://www.npmjs.com/~shengyfu","role":null,"created_at":"2023-10-09T21:55:16.993Z","updated_at":"2023-10-09T21:55:16.993Z","packages_url":"https://packages.ecosyste.ms/api/v1/registries/npmjs.org/maintainers/shengyfu/packages"}]}