From e2be6611dbf69e86fde684de16784b3df523aca3 Mon Sep 17 00:00:00 2001 From: Richard Feldman Date: Thu, 7 Aug 2025 14:39:58 -0400 Subject: [PATCH] Use gpt-4o tokenizer for gpt-5 for now --- crates/language_models/src/provider/open_ai.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/language_models/src/provider/open_ai.rs b/crates/language_models/src/provider/open_ai.rs index 04d8fd9f2bfd0253f1944f65f8fb801c155dc19b..ee74562687b5de4258328a1f7ffae082d5dc4931 100644 --- a/crates/language_models/src/provider/open_ai.rs +++ b/crates/language_models/src/provider/open_ai.rs @@ -673,10 +673,11 @@ pub fn count_open_ai_tokens( | Model::O1 | Model::O3 | Model::O3Mini - | Model::O4Mini - | Model::Five - | Model::FiveMini - | Model::FiveNano => tiktoken_rs::num_tokens_from_messages(model.id(), &messages), + | Model::O4Mini => tiktoken_rs::num_tokens_from_messages(model.id(), &messages), + // GPT-5 models don't have tiktoken support yet; fall back on gpt-4o tokenizer + Model::Five | Model::FiveMini | Model::FiveNano => { + tiktoken_rs::num_tokens_from_messages("gpt-4o", &messages) + } } .map(|tokens| tokens as u64) })