From b24e84f4917e401eb983b5ce81a30c0e8bf34e41 Mon Sep 17 00:00:00 2001 From: Roshan Padaki Date: Sat, 1 Feb 2025 12:08:44 -0500 Subject: [PATCH] assistant: Use GPT 4 tokenizer for `o3-mini` (#24068) Sorry to dump an unsolicited PR for a hot feature! I'm sure someone else was taking a look at this. I noticed that token counting was disabled and I was getting error logs of the form `[2025-01-31T22:59:01-05:00 ERROR assistant_context_editor] No tokenizer found for model o3-mini` when using the new model. To fix the issue, this PR registers the `gpt-4` tokenizer for this model. Release Notes: - openai: Fixed Assistant token counts for `o3-mini` models --- crates/language_models/src/provider/open_ai.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/language_models/src/provider/open_ai.rs b/crates/language_models/src/provider/open_ai.rs index 2e0c3cffab11ce0b51d606502640696291037677..c07bf7038c8d795e828c35e69d044f8e3107ec6a 100644 --- a/crates/language_models/src/provider/open_ai.rs +++ b/crates/language_models/src/provider/open_ai.rs @@ -362,7 +362,10 @@ pub fn count_open_ai_tokens( .collect::>(); match model { - open_ai::Model::Custom { .. } | open_ai::Model::O1Mini | open_ai::Model::O1 => { + open_ai::Model::Custom { .. } + | open_ai::Model::O1Mini + | open_ai::Model::O1 + | open_ai::Model::O3Mini => { tiktoken_rs::num_tokens_from_messages("gpt-4", &messages) } _ => tiktoken_rs::num_tokens_from_messages(model.id(), &messages),