Revise comment about tiktoken

Richard Feldman created 1 week ago

Change summary

crates/language_models/src/provider/anthropic.rs | 2 +-
crates/language_models/src/provider/bedrock.rs   | 2 +-
crates/language_models/src/provider/google.rs    | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)

Detailed changes

crates/language_models/src/provider/anthropic.rs

@@ -275,7 +275,7 @@ pub fn count_anthropic_tokens(
             }
         }
 
-        // Tiktoken doesn't yet support these models, so we manually use the
+        // Tiktoken doesn't support these models, so we manually use the
         // same tokenizer as GPT-4.
         tiktoken_rs::num_tokens_from_messages("gpt-4", &string_messages)
             .map(|tokens| (tokens + tokens_from_images) as u64)

crates/language_models/src/provider/bedrock.rs

@@ -878,7 +878,7 @@ pub fn get_bedrock_tokens(
                 }
             }
 
-            // Tiktoken doesn't yet support these models, so we manually use the
+            // Tiktoken doesn't support these models, so we manually use the
             // same tokenizer as GPT-4.
             tiktoken_rs::num_tokens_from_messages("gpt-4", &string_messages)
                 .map(|tokens| (tokens + tokens_from_images) as u64)

crates/language_models/src/provider/google.rs

@@ -690,7 +690,7 @@ pub fn count_google_tokens(
             })
             .collect::<Vec<_>>();
 
-        // Tiktoken doesn't yet support these models, so we manually use the
+        // Tiktoken doesn't support these models, so we manually use the
         // same tokenizer as GPT-4.
         tiktoken_rs::num_tokens_from_messages("gpt-4", &messages).map(|tokens| tokens as u64)
     })