Use gpt-4o tokenizer for gpt-5 for now

Richard Feldman created 5 months ago

Change summary

crates/language_models/src/provider/open_ai.rs | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)

Detailed changes

crates/language_models/src/provider/open_ai.rs

@@ -673,10 +673,11 @@ pub fn count_open_ai_tokens(
             | Model::O1
             | Model::O3
             | Model::O3Mini
-            | Model::O4Mini
-            | Model::Five
-            | Model::FiveMini
-            | Model::FiveNano => tiktoken_rs::num_tokens_from_messages(model.id(), &messages),
+            | Model::O4Mini => tiktoken_rs::num_tokens_from_messages(model.id(), &messages),
+            // GPT-5 models don't have tiktoken support yet; fall back on gpt-4o tokenizer
+            Model::Five | Model::FiveMini | Model::FiveNano => {
+                tiktoken_rs::num_tokens_from_messages("gpt-4o", &messages)
+            }
         }
         .map(|tokens| tokens as u64)
     })