{"id":11971398,"name":"github.com/agentstation/tokenizer","ecosystem":"go","description":"Package tokenizer provides a collection of high-performance tokenizer implementations.","homepage":"https://github.com/agentstation/tokenizer","licenses":"MIT","normalized_licenses":["MIT"],"repository_url":"https://github.com/agentstation/tokenizer","keywords_array":[],"namespace":"github.com/agentstation","versions_count":8,"first_release_published_at":"2025-08-01T15:46:54.000Z","latest_release_published_at":"2025-09-06T02:41:54.000Z","latest_release_number":"v0.0.8","last_synced_at":"2026-10-06T14:49:21.260Z","created_at":"2025-08-02T08:45:26.240Z","updated_at":"2026-10-06T14:49:21.260Z","registry_url":"https://pkg.go.dev/github.com/agentstation/tokenizer","install_command":"go get github.com/agentstation/tokenizer","documentation_url":"https://pkg.go.dev/github.com/agentstation/tokenizer#section-documentation","metadata":null,"repo_metadata":{"id":307514555,"uuid":"1029484812","full_name":"agentstation/tokenizer","owner":"agentstation","description":"High-performance tokenizer implementations in Go with unified CLI. Features Llama 3 tokenizer with exact compatibility, streaming support, and comprehensive tooling.","archived":false,"fork":false,"pushed_at":"2026-09-10T23:04:02.000Z","size":5298,"stargazers_count":0,"open_issues_count":9,"forks_count":0,"subscribers_count":0,"default_branch":"master","last_synced_at":"2026-10-06T10:59:34.559Z","etag":null,"topics":["ai","bpe","cli","golang","gomarkdoc","goreleaser","llama3","llm","machine-learning","natural-language-processing","nlp","text-processing","tokenizer"],"latest_commit_sha":null,"homepage":"https://pkg.go.dev/github.com/agentstation/tokenizer","language":"Go","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":"mit","status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/agentstation.png","metadata":{"files":{"readme":"README.md","changelog":null,"contributing":null,"funding":null,"license":"LICENSE","code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":".github/CODEOWNERS","security":null,"support":null,"governance":null,"roadmap":null,"authors":null,"dei":null,"publiccode":null,"codemeta":null,"zenodo":null,"notice":null,"maintainers":null,"copyright":null,"agents":null,"dco":null,"cla":null}},"created_at":"2025-07-31T05:54:08.000Z","updated_at":"2026-09-10T23:04:07.000Z","dependencies_parsed_at":"2025-07-31T19:01:49.508Z","dependency_job_id":"052ab3bb-a4c8-4f02-9293-627610977da1","html_url":"https://github.com/agentstation/tokenizer","commit_stats":null,"previous_names":["agentstation/tokenizer"],"tags_count":8,"template":false,"template_full_name":null,"purl":"pkg:github/agentstation/tokenizer","repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/agentstation%2Ftokenizer","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/agentstation%2Ftokenizer/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/agentstation%2Ftokenizer/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/agentstation%2Ftokenizer/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/agentstation","download_url":"https://codeload.github.com/agentstation/tokenizer/tar.gz/refs/heads/master","sbom_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/agentstation%2Ftokenizer/sbom","scorecard":null,"host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":343170754,"owners_count":38088309,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2026-10-03T21:59:58.778Z","status":"online","status_checked_at":"2026-10-06T02:00:06.537Z","response_time":86,"last_error":null,"robots_txt_status":"success","robots_txt_updated_at":"2025-07-24T06:49:26.215Z","robots_txt_url":"https://github.com/robots.txt","online":true,"can_crawl_api":true,"host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"},"owner_record":{"login":"agentstation","name":"AgentStation","uuid":"165715156","kind":"organization","description":"","email":null,"website":"agentstation.ai","location":"United States of America","twitter":null,"company":null,"icon_url":"https://avatars.githubusercontent.com/u/165715156?v=4","repositories_count":1,"last_synced_at":"2024-09-15T07:18:24.955Z","metadata":{"has_sponsors_listing":false},"html_url":"https://github.com/agentstation","funding_links":[],"total_stars":0,"followers":0,"following":0,"created_at":"2024-09-15T07:18:25.158Z","updated_at":"2024-09-15T07:18:25.158Z","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/agentstation","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/agentstation/repositories"},"tags":[{"name":"v0.0.8","sha":"23b959390acf028392f68fe3ba49db7d839302d4","kind":"tag","published_at":"2025-09-06T02:42:43.000Z","download_url":"https://codeload.github.com/agentstation/tokenizer/tar.gz/v0.0.8","html_url":"https://github.com/agentstation/tokenizer/releases/tag/v0.0.8","dependencies_parsed_at":null,"dependency_job_id":null,"purl":"pkg:github/agentstation/tokenizer@v0.0.8","tag_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/agentstation%2Ftokenizer/tags/v0.0.8","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/agentstation%2Ftokenizer/tags/v0.0.8/manifests"},{"name":"v0.0.7","sha":"049655a27e6606ca1a08b1ea8c5a69ece706eeb0","kind":"tag","published_at":"2025-08-01T22:48:01.000Z","download_url":"https://codeload.github.com/agentstation/tokenizer/tar.gz/v0.0.7","html_url":"https://github.com/agentstation/tokenizer/releases/tag/v0.0.7","dependencies_parsed_at":null,"dependency_job_id":null,"purl":"pkg:github/agentstation/tokenizer@v0.0.7","tag_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/agentstation%2Ftokenizer/tags/v0.0.7","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/agentstation%2Ftokenizer/tags/v0.0.7/manifests"},{"name":"v0.0.6","sha":"b79ddbd917fc1979010bdb6def89eb384fd8c50d","kind":"tag","published_at":"2025-08-01T22:42:32.000Z","download_url":"https://codeload.github.com/agentstation/tokenizer/tar.gz/v0.0.6","html_url":"https://github.com/agentstation/tokenizer/releases/tag/v0.0.6","dependencies_parsed_at":null,"dependency_job_id":null,"purl":"pkg:github/agentstation/tokenizer@v0.0.6","tag_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/agentstation%2Ftokenizer/tags/v0.0.6","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/agentstation%2Ftokenizer/tags/v0.0.6/manifests"},{"name":"v0.0.5","sha":"c9228ed148bc248c1a0665f547bed25aff533688","kind":"tag","published_at":"2025-08-01T21:53:49.000Z","download_url":"https://codeload.github.com/agentstation/tokenizer/tar.gz/v0.0.5","html_url":"https://github.com/agentstation/tokenizer/releases/tag/v0.0.5","dependencies_parsed_at":null,"dependency_job_id":null,"purl":"pkg:github/agentstation/tokenizer@v0.0.5","tag_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/agentstation%2Ftokenizer/tags/v0.0.5","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/agentstation%2Ftokenizer/tags/v0.0.5/manifests"},{"name":"v0.0.4","sha":"54e03c9bf7a33073aaa98f53f85937f103891e96","kind":"tag","published_at":"2025-08-01T19:21:37.000Z","download_url":"https://codeload.github.com/agentstation/tokenizer/tar.gz/v0.0.4","html_url":"https://github.com/agentstation/tokenizer/releases/tag/v0.0.4","dependencies_parsed_at":null,"dependency_job_id":null,"purl":"pkg:github/agentstation/tokenizer@v0.0.4","tag_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/agentstation%2Ftokenizer/tags/v0.0.4","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/agentstation%2Ftokenizer/tags/v0.0.4/manifests"},{"name":"v0.0.3","sha":"0fcfc7666f98b9e099e8ee02fe4b76cabd12882d","kind":"tag","published_at":"2025-08-01T17:06:01.000Z","download_url":"https://codeload.github.com/agentstation/tokenizer/tar.gz/v0.0.3","html_url":"https://github.com/agentstation/tokenizer/releases/tag/v0.0.3","dependencies_parsed_at":null,"dependency_job_id":null,"purl":"pkg:github/agentstation/tokenizer@v0.0.3","tag_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/agentstation%2Ftokenizer/tags/v0.0.3","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/agentstation%2Ftokenizer/tags/v0.0.3/manifests"},{"name":"v0.0.2","sha":"c194a9e6fb71fe27776802a7ee1967a2af580657","kind":"tag","published_at":"2025-08-01T16:50:06.000Z","download_url":"https://codeload.github.com/agentstation/tokenizer/tar.gz/v0.0.2","html_url":"https://github.com/agentstation/tokenizer/releases/tag/v0.0.2","dependencies_parsed_at":null,"dependency_job_id":null,"purl":"pkg:github/agentstation/tokenizer@v0.0.2","tag_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/agentstation%2Ftokenizer/tags/v0.0.2","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/agentstation%2Ftokenizer/tags/v0.0.2/manifests"},{"name":"v0.0.1","sha":"ba5a228bbf7e2d3fe63e941a532e7fe8b4009066","kind":"tag","published_at":"2025-08-01T16:23:03.000Z","download_url":"https://codeload.github.com/agentstation/tokenizer/tar.gz/v0.0.1","html_url":"https://github.com/agentstation/tokenizer/releases/tag/v0.0.1","dependencies_parsed_at":null,"dependency_job_id":null,"purl":"pkg:github/agentstation/tokenizer@v0.0.1","tag_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/agentstation%2Ftokenizer/tags/v0.0.1","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/agentstation%2Ftokenizer/tags/v0.0.1/manifests"}]},"repo_metadata_updated_at":"2026-10-06T14:49:20.795Z","dependent_packages_count":0,"downloads":null,"downloads_period":null,"dependent_repos_count":0,"rankings":{"downloads":null,"dependent_repos_count":5.6078556279763525,"dependent_packages_count":5.254901351480829,"stargazers_count":null,"forks_count":null,"docker_downloads_count":null,"average":5.431378489728591},"purl":"pkg:golang/github.com/agentstation/tokenizer","advisories":[],"docker_usage_url":"https://docker.ecosyste.ms/usage/go/github.com/agentstation/tokenizer","docker_dependents_count":null,"docker_downloads_count":null,"usage_url":"https://repos.ecosyste.ms/usage/go/github.com/agentstation/tokenizer","dependent_repositories_url":"https://repos.ecosyste.ms/api/v1/usage/go/github.com/agentstation/tokenizer/dependencies","status":null,"funding_links":[],"critical":null,"issue_metadata":{"last_synced_at":"2026-10-06T06:19:50.940Z","issues_count":0,"pull_requests_count":30,"avg_time_to_close_issue":null,"avg_time_to_close_pull_request":2883896.947368421,"issues_closed_count":0,"pull_requests_closed_count":19,"pull_request_authors_count":3,"issue_authors_count":0,"avg_comments_per_issue":null,"avg_comments_per_pull_request":1.4666666666666668,"merged_pull_requests_count":5,"bot_issues_count":0,"bot_pull_requests_count":26,"past_year_issues_count":0,"past_year_pull_requests_count":16,"past_year_avg_time_to_close_issue":null,"past_year_avg_time_to_close_pull_request":1662586.5,"past_year_issues_closed_count":0,"past_year_pull_requests_closed_count":8,"past_year_pull_request_authors_count":1,"past_year_issue_authors_count":0,"past_year_avg_comments_per_issue":null,"past_year_avg_comments_per_pull_request":1.5,"past_year_bot_issues_count":0,"past_year_bot_pull_requests_count":16,"past_year_merged_pull_requests_count":0,"issues_url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/repositories/agentstation%2Ftokenizer/issues","maintainers":[{"login":"jackspirou","count":4,"url":"https://issues.ecosyste.ms/api/v1/hosts/GitHub/authors/jackspirou"}],"active_maintainers":[]},"versions_url":"https://packages.ecosyste.ms/api/v1/registries/proxy.golang.org/packages/github.com%2Fagentstation%2Ftokenizer/versions","version_numbers_url":"https://packages.ecosyste.ms/api/v1/registries/proxy.golang.org/packages/github.com%2Fagentstation%2Ftokenizer/version_numbers","latest_version_url":"https://packages.ecosyste.ms/api/v1/registries/proxy.golang.org/packages/github.com%2Fagentstation%2Ftokenizer/latest_version","dependent_packages_url":"https://packages.ecosyste.ms/api/v1/registries/proxy.golang.org/packages/github.com%2Fagentstation%2Ftokenizer/dependent_packages","related_packages_url":"https://packages.ecosyste.ms/api/v1/registries/proxy.golang.org/packages/github.com%2Fagentstation%2Ftokenizer/related_packages","codemeta_url":"https://packages.ecosyste.ms/api/v1/registries/proxy.golang.org/packages/github.com%2Fagentstation%2Ftokenizer/codemeta","maintainers":[]}