Revert tokenizer for custom OpenAI models (#17660)

Peter Tripp created 

Fix for custom openai models tokenizer settings.

Change summary

crates/language_model/src/provider/open_ai.rs | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)

Detailed changes

crates/language_model/src/provider/open_ai.rs 🔗 

@@ -370,7 +370,11 @@ pub fn count_open_ai_tokens(
                 })
                 .collect::<Vec<_>>();
 
-            tiktoken_rs::num_tokens_from_messages(model.id(), &messages)
+            if let open_ai::Model::Custom { .. } = model {
+                tiktoken_rs::num_tokens_from_messages("gpt-4", &messages)
+            } else {
+                tiktoken_rs::num_tokens_from_messages(model.id(), &messages)
+            }
         })
         .boxed()
 }