From a3c9f94d40de3ccc24a8fbf4ecda511c84b8e189 Mon Sep 17 00:00:00 2001 From: Roshan Padaki Date: Sat, 1 Feb 2025 12:08:44 -0500 Subject: [PATCH] assistant: Use GPT 4 tokenizer for `o3-mini` (#24068) Sorry to dump an unsolicited PR for a hot feature! I'm sure someone else was taking a look at this. I noticed that token counting was disabled and I was getting error logs of the form `[2025-01-31T22:59:01-05:00 ERROR assistant_context_editor] No tokenizer found for model o3-mini` when using the new model. To fix the issue, this PR registers the `gpt-4` tokenizer for this model. Release Notes: - openai: Fixed Assistant token counts for `o3-mini` models --- crates/language_models/src/provider/open_ai.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/language_models/src/provider/open_ai.rs b/crates/language_models/src/provider/open_ai.rs index 36742b0035f511b8552826a34ce26c4a1432300f..f5529757b5f064b1c208beb3a23a743c53a504b8 100644 --- a/crates/language_models/src/provider/open_ai.rs +++ b/crates/language_models/src/provider/open_ai.rs @@ -361,7 +361,10 @@ pub fn count_open_ai_tokens( .collect::>(); match model { - open_ai::Model::Custom { .. } | open_ai::Model::O1Mini | open_ai::Model::O1 => { + open_ai::Model::Custom { .. } + | open_ai::Model::O1Mini + | open_ai::Model::O1 + | open_ai::Model::O3Mini => { tiktoken_rs::num_tokens_from_messages("gpt-4", &messages) } _ => tiktoken_rs::num_tokens_from_messages(model.id(), &messages),