open_ai: Make the deltas optional (#39142)

Techy and Bennet Bo Fenner created 3 weeks ago

I am using an Azure OpenAI instance since that is what is provided at
work and with how they have it setup not all responses contain a delta,
which lead to errors and truncated responses. This is related to how
they are filtering potentially offensive requests and responses. I don't
believe this filter was made in-house, instead I believe it is provided
by Microsoft/Azure, so I suspect this fix may help other users.

Release Notes:

- N/A

Co-authored-by: Bennet Bo Fenner <[e-mail]>

Change summary

crates/language_models/src/provider/open_ai.rs | 30 ++++++++++----------
crates/open_ai/src/open_ai.rs                  |  2 
2 files changed, 16 insertions(+), 16 deletions(-)

Detailed changes

crates/language_models/src/provider/open_ai.rs

@@ -538,27 +538,27 @@ impl OpenAiEventMapper {
             return events;
         };
 
-        if let Some(content) = choice.delta.content.clone() {
-            if !content.is_empty() {
+        if let Some(delta) = choice.delta.as_ref() {
+            if let Some(content) = delta.content.clone() {
                 events.push(Ok(LanguageModelCompletionEvent::Text(content)));
             }
-        }
-
-        if let Some(tool_calls) = choice.delta.tool_calls.as_ref() {
-            for tool_call in tool_calls {
-                let entry = self.tool_calls_by_index.entry(tool_call.index).or_default();
 
-                if let Some(tool_id) = tool_call.id.clone() {
-                    entry.id = tool_id;
-                }
+            if let Some(tool_calls) = delta.tool_calls.as_ref() {
+                for tool_call in tool_calls {
+                    let entry = self.tool_calls_by_index.entry(tool_call.index).or_default();
 
-                if let Some(function) = tool_call.function.as_ref() {
-                    if let Some(name) = function.name.clone() {
-                        entry.name = name;
+                    if let Some(tool_id) = tool_call.id.clone() {
+                        entry.id = tool_id;
                     }
 
-                    if let Some(arguments) = function.arguments.clone() {
-                        entry.arguments.push_str(&arguments);
+                    if let Some(function) = tool_call.function.as_ref() {
+                        if let Some(name) = function.name.clone() {
+                            entry.name = name;
+                        }
+
+                        if let Some(arguments) = function.arguments.clone() {
+                            entry.arguments.push_str(&arguments);
+                        }
                     }
                 }
             }

crates/open_ai/src/open_ai.rs

@@ -420,7 +420,7 @@ pub struct Usage {
 #[derive(Serialize, Deserialize, Debug)]
 pub struct ChoiceDelta {
     pub index: u32,
-    pub delta: ResponseMessageDelta,
+    pub delta: Option<ResponseMessageDelta>,
     pub finish_reason: Option<String>,
 }