{"id":10695781,"name":"@tmfi/tokenizers","ecosystem":"npm","description":"Provides an implementation of today's most used tokenizers, with a focus on performances and versatility.","homepage":"https://github.com/vhain/tokenizers/tree/master/bindings/node","licenses":"Apache-2.0","normalized_licenses":["Apache-2.0"],"repository_url":"https://github.com/vhain/tokenizers","keywords_array":[],"namespace":"tmfi","versions_count":2,"first_release_published_at":"2024-07-16T02:33:25.730Z","latest_release_published_at":"2024-08-22T22:57:07.751Z","latest_release_number":"0.20.0-dev0","last_synced_at":"2026-03-19T23:34:50.845Z","created_at":"2024-07-16T02:35:11.273Z","updated_at":"2026-03-19T23:34:50.845Z","registry_url":"https://www.npmjs.com/package/@tmfi/tokenizers","install_command":"npm install @tmfi/tokenizers","documentation_url":null,"metadata":{"funding":null,"dist-tags":{"latest":"0.20.0-dev0"}},"repo_metadata":{"id":248541208,"uuid":"828979480","full_name":"vhain/tokenizers","owner":"vhain","description":"Forked version of https://github.com/huggingface/tokenizers to publish working npm package. 💥 Fast State-of-the-Art Tokenizers optimized for Research and Production","archived":false,"fork":true,"pushed_at":"2024-08-22T22:55:57.000Z","size":6421,"stargazers_count":0,"open_issues_count":0,"forks_count":0,"subscribers_count":0,"default_branch":"main","last_synced_at":"2025-09-17T14:14:08.396Z","etag":null,"topics":[],"latest_commit_sha":null,"homepage":"https://huggingface.co/docs/tokenizers","language":"Rust","has_issues":false,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":"huggingface/tokenizers","license":"apache-2.0","status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/vhain.png","metadata":{},"created_at":"2024-07-15T14:06:15.000Z","updated_at":"2024-08-22T22:42:34.000Z","dependencies_parsed_at":null,"dependency_job_id":null,"html_url":"https://github.com/vhain/tokenizers","commit_stats":null,"previous_names":["vhain/tokenizers"],"tags_count":3,"template":false,"template_full_name":null,"purl":"pkg:github/vhain/tokenizers","repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/vhain%2Ftokenizers","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/vhain%2Ftokenizers/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/vhain%2Ftokenizers/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/vhain%2Ftokenizers/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/vhain","download_url":"https://codeload.github.com/vhain/tokenizers/tar.gz/refs/heads/main","sbom_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/vhain%2Ftokenizers/sbom","scorecard":null,"host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":276695093,"owners_count":25687772,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2022-07-04T15:15:14.044Z","status":"online","status_checked_at":"2025-09-24T02:00:09.776Z","response_time":97,"last_error":null,"robots_txt_status":"success","robots_txt_updated_at":"2025-07-24T06:49:26.215Z","robots_txt_url":"https://github.com/robots.txt","online":true,"can_crawl_api":true,"host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"},"owner_record":{"login":"vhain","name":"Juwan Yoo","uuid":"1812260","kind":"user","description":"","email":"","website":null,"location":"San Francisco, CA, USA","twitter":null,"company":"@netflix","icon_url":"https://avatars.githubusercontent.com/u/1812260?u=ed45f69666c00e8903a33d65bc90ce9adfa27070\u0026v=4","repositories_count":16,"last_synced_at":"2023-03-28T12:05:20.961Z","metadata":{"has_sponsors_listing":false},"html_url":"https://github.com/vhain","funding_links":[],"total_stars":null,"followers":null,"following":null,"created_at":"2022-11-04T03:47:45.567Z","updated_at":"2023-03-28T12:06:56.540Z","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/vhain","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/vhain/repositories"},"tags":[{"name":"v0.20.0-dev0","sha":"c0846baf8c0ff09e0178637135e16eb3f0ce4bec","kind":"commit","published_at":"2024-08-22T22:42:26.000Z","download_url":"https://codeload.github.com/vhain/tokenizers/tar.gz/v0.20.0-dev0","html_url":"https://github.com/vhain/tokenizers/releases/tag/v0.20.0-dev0","dependencies_parsed_at":null,"dependency_job_id":null,"purl":"pkg:github/vhain/tokenizers@v0.20.0-dev0","tag_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/vhain%2Ftokenizers/tags/v0.20.0-dev0","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/vhain%2Ftokenizers/tags/v0.20.0-dev0/manifests"},{"name":"v0.20.0","sha":"06dcdf40e1f8a58b00ea0b0d02ad824a9e06882d","kind":"commit","published_at":"2024-08-22T12:45:32.000Z","download_url":"https://codeload.github.com/vhain/tokenizers/tar.gz/v0.20.0","html_url":"https://github.com/vhain/tokenizers/releases/tag/v0.20.0","dependencies_parsed_at":null,"dependency_job_id":null,"purl":"pkg:github/vhain/tokenizers@v0.20.0","tag_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/vhain%2Ftokenizers/tags/v0.20.0","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/vhain%2Ftokenizers/tags/v0.20.0/manifests"},{"name":"v0.19.1-dev0","sha":"4984b2ed95a042f0dacb08d00cc694584d8d2b11","kind":"commit","published_at":"2024-07-15T18:04:35.000Z","download_url":"https://codeload.github.com/vhain/tokenizers/tar.gz/v0.19.1-dev0","html_url":"https://github.com/vhain/tokenizers/releases/tag/v0.19.1-dev0","dependencies_parsed_at":null,"dependency_job_id":null,"purl":"pkg:github/vhain/tokenizers@v0.19.1-dev0","tag_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/vhain%2Ftokenizers/tags/v0.19.1-dev0","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/vhain%2Ftokenizers/tags/v0.19.1-dev0/manifests"}]},"repo_metadata_updated_at":"2025-11-04T14:35:27.817Z","dependent_packages_count":0,"downloads":374,"downloads_period":"last-month","dependent_repos_count":0,"rankings":{"downloads":null,"dependent_repos_count":26.44362042000762,"dependent_packages_count":38.4211304117047,"stargazers_count":null,"forks_count":null,"docker_downloads_count":null,"average":32.43237541585616},"purl":"pkg:npm/%40tmfi/tokenizers","advisories":[],"docker_usage_url":"https://docker.ecosyste.ms/usage/npm/@tmfi/tokenizers","docker_dependents_count":null,"docker_downloads_count":null,"usage_url":"https://repos.ecosyste.ms/usage/npm/@tmfi/tokenizers","dependent_repositories_url":"https://repos.ecosyste.ms/api/v1/usage/npm/@tmfi/tokenizers/dependencies","status":null,"funding_links":[],"critical":null,"issue_metadata":null,"versions_url":"https://packages.ecosyste.ms/api/v1/registries/npmjs.org/packages/@tmfi%2Ftokenizers/versions","version_numbers_url":"https://packages.ecosyste.ms/api/v1/registries/npmjs.org/packages/@tmfi%2Ftokenizers/version_numbers","dependent_packages_url":"https://packages.ecosyste.ms/api/v1/registries/npmjs.org/packages/@tmfi%2Ftokenizers/dependent_packages","related_packages_url":"https://packages.ecosyste.ms/api/v1/registries/npmjs.org/packages/@tmfi%2Ftokenizers/related_packages","codemeta_url":"https://packages.ecosyste.ms/api/v1/registries/npmjs.org/packages/@tmfi%2Ftokenizers/codemeta","maintainers":[{"uuid":"vhain","login":"vhain","name":null,"email":"vhain6512@gmail.com","url":null,"packages_count":17,"html_url":"https://www.npmjs.com/~vhain","role":null,"created_at":"2024-07-16T02:35:13.329Z","updated_at":"2024-07-16T02:35:13.329Z","packages_url":"https://packages.ecosyste.ms/api/v1/registries/npmjs.org/maintainers/vhain/packages"}]}