From ea5800b322ef2095a10dac0c35b2c250bb80d920 Mon Sep 17 00:00:00 2001 From: Richard Feldman Date: Tue, 16 Dec 2025 09:21:26 -0500 Subject: [PATCH] Revise comment about tiktoken --- crates/language_models/src/provider/anthropic.rs | 2 +- crates/language_models/src/provider/bedrock.rs | 2 +- crates/language_models/src/provider/google.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/language_models/src/provider/anthropic.rs b/crates/language_models/src/provider/anthropic.rs index 299e94484cf41f1bcb38e50656128c38d954d612..6be88460d26da1bfdebe5e01dbde2a55be875b1f 100644 --- a/crates/language_models/src/provider/anthropic.rs +++ b/crates/language_models/src/provider/anthropic.rs @@ -275,7 +275,7 @@ pub fn count_anthropic_tokens( } } - // Tiktoken doesn't yet support these models, so we manually use the + // Tiktoken doesn't support these models, so we manually use the // same tokenizer as GPT-4. tiktoken_rs::num_tokens_from_messages("gpt-4", &string_messages) .map(|tokens| (tokens + tokens_from_images) as u64) diff --git a/crates/language_models/src/provider/bedrock.rs b/crates/language_models/src/provider/bedrock.rs index b85a038bb235d97bd9de8614f19764ecabf7bbfe..bc2ad2f63f71c54beabc32de46833ac83807084b 100644 --- a/crates/language_models/src/provider/bedrock.rs +++ b/crates/language_models/src/provider/bedrock.rs @@ -878,7 +878,7 @@ pub fn get_bedrock_tokens( } } - // Tiktoken doesn't yet support these models, so we manually use the + // Tiktoken doesn't support these models, so we manually use the // same tokenizer as GPT-4. tiktoken_rs::num_tokens_from_messages("gpt-4", &string_messages) .map(|tokens| (tokens + tokens_from_images) as u64) diff --git a/crates/language_models/src/provider/google.rs b/crates/language_models/src/provider/google.rs index b3e03ce9d3a7be24996a074abff3f421d5bad0fd..af3e6406cae8d81cd6a92ca62aabbd5e94eec752 100644 --- a/crates/language_models/src/provider/google.rs +++ b/crates/language_models/src/provider/google.rs @@ -690,7 +690,7 @@ pub fn count_google_tokens( }) .collect::>(); - // Tiktoken doesn't yet support these models, so we manually use the + // Tiktoken doesn't support these models, so we manually use the // same tokenizer as GPT-4. tiktoken_rs::num_tokens_from_messages("gpt-4", &messages).map(|tokens| tokens as u64) })