{"id":5312166,"name":"tokenizers","ecosystem":"hex","description":"Bindings to Hugging Face Tokenizers for Elixir","homepage":null,"licenses":"Apache-2.0","normalized_licenses":["Apache-2.0"],"repository_url":"https://github.com/elixir-nx/tokenizers","keywords_array":[],"namespace":null,"versions_count":10,"first_release_published_at":"2022-10-07T23:15:33.193Z","latest_release_published_at":"2024-10-02T18:08:44.623Z","latest_release_number":"0.5.1","last_synced_at":"2026-09-15T10:48:45.460Z","created_at":"2022-10-07T23:20:52.382Z","updated_at":"2026-09-16T06:08:13.679Z","registry_url":"https://hex.pm/packages/tokenizers/","install_command":"mix hex.package fetch tokenizers","documentation_url":"http://hexdocs.pm/tokenizers/","metadata":{},"repo_metadata":{"id":37101250,"uuid":"458383564","full_name":"elixir-nx/tokenizers","owner":"elixir-nx","description":"Elixir bindings for 🤗 Tokenizers","archived":false,"fork":false,"pushed_at":"2025-05-30T20:59:41.000Z","size":2824,"stargazers_count":102,"open_issues_count":2,"forks_count":20,"subscribers_count":10,"default_branch":"main","last_synced_at":"2026-09-15T08:33:02.071Z","etag":null,"topics":["elixir","machine-learning","rust","text-processing"],"latest_commit_sha":null,"homepage":"https://hexdocs.pm/tokenizers","language":"Elixir","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":"apache-2.0","status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/elixir-nx.png","metadata":{"files":{"readme":"README.md","changelog":"CHANGELOG.md","contributing":null,"funding":null,"license":"LICENSE","code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":"CODEOWNERS.md","security":null,"support":null,"governance":null,"roadmap":null,"authors":null,"dei":null,"publiccode":null,"codemeta":null,"zenodo":null}},"created_at":"2022-02-12T00:48:12.000Z","updated_at":"2026-08-20T07:09:53.000Z","dependencies_parsed_at":"2023-10-10T18:48:57.731Z","dependency_job_id":"f095b155-aef8-450b-80ba-2caebc0ea88d","html_url":"https://github.com/elixir-nx/tokenizers","commit_stats":{"total_commits":38,"total_committers":9,"mean_commits":4.222222222222222,"dds":0.7105263157894737,"last_synced_commit":"be0b12d10dd72ab3b4be5029d7828d4830e86ab9"},"previous_names":[],"tags_count":10,"template":false,"template_full_name":null,"purl":"pkg:github/elixir-nx/tokenizers","repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/elixir-nx%2Ftokenizers","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/elixir-nx%2Ftokenizers/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/elixir-nx%2Ftokenizers/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/elixir-nx%2Ftokenizers/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/elixir-nx","download_url":"https://codeload.github.com/elixir-nx/tokenizers/tar.gz/refs/heads/main","sbom_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/elixir-nx%2Ftokenizers/sbom","scorecard":null,"host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":341189360,"owners_count":37352326,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2026-08-22T15:14:58.755Z","status":"online","status_checked_at":"2026-09-16T02:00:06.638Z","response_time":96,"last_error":null,"robots_txt_status":"success","robots_txt_updated_at":"2025-07-24T06:49:26.215Z","robots_txt_url":"https://github.com/robots.txt","online":true,"can_crawl_api":true,"host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"},"owner_record":{"login":"elixir-nx","name":"Numerical Elixir (Nx)","uuid":"74903619","kind":"organization","description":"","email":null,"website":null,"location":null,"twitter":null,"company":null,"icon_url":"https://avatars.githubusercontent.com/u/74903619?v=4","repositories_count":13,"last_synced_at":"2023-03-03T21:48:44.285Z","metadata":{"has_sponsors_listing":false},"html_url":"https://github.com/elixir-nx","funding_links":[],"total_stars":null,"followers":null,"following":null,"created_at":"2022-11-14T05:48:36.304Z","updated_at":"2023-03-03T21:48:44.311Z","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/elixir-nx","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/elixir-nx/repositories"},"tags":[{"name":"v0.5.1","sha":"c4004dceba4562d3b8dd900d714282ca0cf0225c","kind":"commit","published_at":"2024-10-02T17:51:06.000Z","download_url":"https://codeload.github.com/elixir-nx/tokenizers/tar.gz/v0.5.1","html_url":"https://github.com/elixir-nx/tokenizers/releases/tag/v0.5.1","dependencies_parsed_at":null,"dependency_job_id":null,"purl":"pkg:github/elixir-nx/tokenizers@v0.5.1","tag_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/elixir-nx%2Ftokenizers/tags/v0.5.1","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/elixir-nx%2Ftokenizers/tags/v0.5.1/manifests"},{"name":"v0.5.0","sha":"09f3f25fc4792f7c3f8ac4dd8a43a26f76c6f85f","kind":"commit","published_at":"2024-04-24T11:35:19.000Z","download_url":"https://codeload.github.com/elixir-nx/tokenizers/tar.gz/v0.5.0","html_url":"https://github.com/elixir-nx/tokenizers/releases/tag/v0.5.0","dependencies_parsed_at":null,"dependency_job_id":null,"purl":"pkg:github/elixir-nx/tokenizers@v0.5.0","tag_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/elixir-nx%2Ftokenizers/tags/v0.5.0","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/elixir-nx%2Ftokenizers/tags/v0.5.0/manifests"},{"name":"v0.4.0","sha":"04e17532bcf9904424f8b46aae03a5040bf6259e","kind":"commit","published_at":"2023-08-09T10:42:19.000Z","download_url":"https://codeload.github.com/elixir-nx/tokenizers/tar.gz/v0.4.0","html_url":"https://github.com/elixir-nx/tokenizers/releases/tag/v0.4.0","dependencies_parsed_at":null,"dependency_job_id":null,"purl":"pkg:github/elixir-nx/tokenizers@v0.4.0","tag_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/elixir-nx%2Ftokenizers/tags/v0.4.0","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/elixir-nx%2Ftokenizers/tags/v0.4.0/manifests"},{"name":"v0.3.2","sha":"0190d0a4b80604af7d7a35dbb7544a886d9e2b02","kind":"commit","published_at":"2023-04-19T22:51:35.000Z","download_url":"https://codeload.github.com/elixir-nx/tokenizers/tar.gz/v0.3.2","html_url":"https://github.com/elixir-nx/tokenizers/releases/tag/v0.3.2","dependencies_parsed_at":null,"dependency_job_id":null,"purl":"pkg:github/elixir-nx/tokenizers@v0.3.2","tag_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/elixir-nx%2Ftokenizers/tags/v0.3.2","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/elixir-nx%2Ftokenizers/tags/v0.3.2/manifests"},{"name":"v0.3.1","sha":"752eaf15a7b65a376d5e257f996771fefb390a14","kind":"commit","published_at":"2023-04-06T22:17:51.000Z","download_url":"https://codeload.github.com/elixir-nx/tokenizers/tar.gz/v0.3.1","html_url":"https://github.com/elixir-nx/tokenizers/releases/tag/v0.3.1","dependencies_parsed_at":null,"dependency_job_id":null,"purl":"pkg:github/elixir-nx/tokenizers@v0.3.1","tag_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/elixir-nx%2Ftokenizers/tags/v0.3.1","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/elixir-nx%2Ftokenizers/tags/v0.3.1/manifests"},{"name":"v0.3.0","sha":"4ebc9aab2a3c512188b02216f23eb6a5bcb17992","kind":"commit","published_at":"2023-03-04T18:56:14.000Z","download_url":"https://codeload.github.com/elixir-nx/tokenizers/tar.gz/v0.3.0","html_url":"https://github.com/elixir-nx/tokenizers/releases/tag/v0.3.0","dependencies_parsed_at":null,"dependency_job_id":null,"purl":"pkg:github/elixir-nx/tokenizers@v0.3.0","tag_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/elixir-nx%2Ftokenizers/tags/v0.3.0","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/elixir-nx%2Ftokenizers/tags/v0.3.0/manifests"},{"name":"v0.2.0","sha":"eeb41853c3887627655e9d0577e85727be34ba8c","kind":"commit","published_at":"2022-12-07T04:11:49.000Z","download_url":"https://codeload.github.com/elixir-nx/tokenizers/tar.gz/v0.2.0","html_url":"https://github.com/elixir-nx/tokenizers/releases/tag/v0.2.0","dependencies_parsed_at":null,"dependency_job_id":null,"purl":"pkg:github/elixir-nx/tokenizers@v0.2.0","tag_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/elixir-nx%2Ftokenizers/tags/v0.2.0","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/elixir-nx%2Ftokenizers/tags/v0.2.0/manifests"},{"name":"v0.1.2","sha":"7ec2b83a6e3c234f653a563b0cb666b7d33c1df7","kind":"commit","published_at":"2022-10-13T01:46:38.000Z","download_url":"https://codeload.github.com/elixir-nx/tokenizers/tar.gz/v0.1.2","html_url":"https://github.com/elixir-nx/tokenizers/releases/tag/v0.1.2","dependencies_parsed_at":null,"dependency_job_id":null,"purl":"pkg:github/elixir-nx/tokenizers@v0.1.2","tag_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/elixir-nx%2Ftokenizers/tags/v0.1.2","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/elixir-nx%2Ftokenizers/tags/v0.1.2/manifests"},{"name":"v0.1.1","sha":"6650a8ac239f78443b860c70ead6889f44c001b8","kind":"commit","published_at":"2022-10-11T00:45:52.000Z","download_url":"https://codeload.github.com/elixir-nx/tokenizers/tar.gz/v0.1.1","html_url":"https://github.com/elixir-nx/tokenizers/releases/tag/v0.1.1","dependencies_parsed_at":null,"dependency_job_id":null,"purl":"pkg:github/elixir-nx/tokenizers@v0.1.1","tag_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/elixir-nx%2Ftokenizers/tags/v0.1.1","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/elixir-nx%2Ftokenizers/tags/v0.1.1/manifests"},{"name":"v0.1.0","sha":"f23a4fcd26383e744827eac769da55a28d307d66","kind":"tag","published_at":"2022-08-27T00:45:06.000Z","download_url":"https://codeload.github.com/elixir-nx/tokenizers/tar.gz/v0.1.0","html_url":"https://github.com/elixir-nx/tokenizers/releases/tag/v0.1.0","dependencies_parsed_at":null,"dependency_job_id":null,"purl":"pkg:github/elixir-nx/tokenizers@v0.1.0","tag_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/elixir-nx%2Ftokenizers/tags/v0.1.0","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/elixir-nx%2Ftokenizers/tags/v0.1.0/manifests"}]},"repo_metadata_updated_at":"2026-09-16T06:08:13.678Z","dependent_packages_count":1,"downloads":713866,"downloads_period":"total","dependent_repos_count":1,"rankings":{"downloads":8.890054415524814,"dependent_repos_count":18.652068445551695,"dependent_packages_count":10.640529731856029,"stargazers_count":8.181996984199829,"forks_count":9.362092703074804,"docker_downloads_count":null,"average":11.145348456041434},"purl":"pkg:hex/tokenizers","advisories":[],"docker_usage_url":"https://docker.ecosyste.ms/usage/hex/tokenizers","docker_dependents_count":null,"docker_downloads_count":null,"usage_url":"https://repos.ecosyste.ms/usage/hex/tokenizers","dependent_repositories_url":"https://repos.ecosyste.ms/api/v1/usage/hex/tokenizers/dependencies","status":null,"funding_links":[],"critical":null,"issue_metadata":{"last_synced_at":"2026-09-15T07:03:25.751Z","issues_count":10,"pull_requests_count":63,"avg_time_to_close_issue":893456.8888888889,"avg_time_to_close_pull_request":218624.7931034483,"issues_closed_count":9,"pull_requests_closed_count":58,"pull_request_authors_count":15,"issue_authors_count":10,"avg_comments_per_issue":3.1,"avg_comments_per_pull_request":1.3174603174603174,"merged_pull_requests_count":56,"bot_issues_count":0,"bot_pull_requests_count":0,"past_year_issues_count":1,"past_year_pull_requests_count":1,"past_year_avg_time_to_close_issue":92667.0,"past_year_avg_time_to_close_pull_request":null,"past_year_issues_closed_count":1,"past_year_pull_requests_closed_count":0,"past_year_pull_request_authors_count":1,"past_year_issue_authors_count":1,"past_year_avg_comments_per_issue":3.0,"past_year_avg_comments_per_pull_request":0.0,"past_year_bot_issues_count":0,"past_year_bot_pull_requests_count":0,"past_year_merged_pull_requests_count":0,"issues_url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/repositories/elixir-nx%2Ftokenizers/issues","maintainers":[{"login":"jonatanklosko","count":4,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/jonatanklosko"}],"active_maintainers":[]},"versions_url":"https://packages.ecosyste.ms/api/v1/registries/hex.pm/packages/tokenizers/versions","version_numbers_url":"https://packages.ecosyste.ms/api/v1/registries/hex.pm/packages/tokenizers/version_numbers","latest_version_url":"https://packages.ecosyste.ms/api/v1/registries/hex.pm/packages/tokenizers/latest_version","dependent_packages_url":"https://packages.ecosyste.ms/api/v1/registries/hex.pm/packages/tokenizers/dependent_packages","related_packages_url":"https://packages.ecosyste.ms/api/v1/registries/hex.pm/packages/tokenizers/related_packages","codemeta_url":"https://packages.ecosyste.ms/api/v1/registries/hex.pm/packages/tokenizers/codemeta","maintainers":[{"uuid":"elixir_nx","login":"elixir_nx","name":null,"email":null,"url":null,"packages_count":18,"html_url":"https://hex.pm/users/elixir_nx","role":null,"created_at":"2022-11-08T15:32:05.436Z","updated_at":"2022-11-08T15:32:05.436Z","packages_url":"https://packages.ecosyste.ms/api/v1/registries/hex.pm/maintainers/elixir_nx/packages"}]}