From bd1ff476b987ed2570f90b76e8c028d7e7951543 Mon Sep 17 00:00:00 2001 From: Peter Tripp Date: Tue, 10 Sep 2024 15:38:27 -0400 Subject: [PATCH] Revert tokenizer for custom OpenAI models (#17660) Fix for custom openai models tokenizer settings. --- crates/language_model/src/provider/open_ai.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/language_model/src/provider/open_ai.rs b/crates/language_model/src/provider/open_ai.rs index 15d84f6cca2efc73f22c0337a5f4802f306aab6f..a7a962e9251671ced71bf8f988156b7ebeb5489e 100644 --- a/crates/language_model/src/provider/open_ai.rs +++ b/crates/language_model/src/provider/open_ai.rs @@ -370,7 +370,11 @@ pub fn count_open_ai_tokens( }) .collect::>(); - tiktoken_rs::num_tokens_from_messages(model.id(), &messages) + if let open_ai::Model::Custom { .. } = model { + tiktoken_rs::num_tokens_from_messages("gpt-4", &messages) + } else { + tiktoken_rs::num_tokens_from_messages(model.id(), &messages) + } }) .boxed() }