open_ai: Support structured OpenAI tool output content (#51832)

Ben Brandt created 3 weeks ago

Allow function call outputs to carry either plain text or a list
of input content items, so image tool results are serialized as
image content instead of a raw base64 string.

Release Notes:

- N/A

Change summary

crates/language_models/src/provider/open_ai.rs | 16 ++++++++++++----
crates/open_ai/src/responses.rs                |  9 ++++++++-
2 files changed, 20 insertions(+), 5 deletions(-)

Detailed changes

crates/language_models/src/provider/open_ai.rs

@@ -14,8 +14,8 @@ use language_model::{
 };
 use menu;
 use open_ai::responses::{
-    ResponseFunctionCallItem, ResponseFunctionCallOutputItem, ResponseInputContent,
-    ResponseInputItem, ResponseMessageItem,
+    ResponseFunctionCallItem, ResponseFunctionCallOutputContent, ResponseFunctionCallOutputItem,
+    ResponseInputContent, ResponseInputItem, ResponseMessageItem,
 };
 use open_ai::{
     ImageUrl, Model, OPEN_AI_API_URL, ReasoningEffort, ResponseStreamEvent,
@@ -647,8 +647,16 @@ fn append_message_to_response_items(
                     ResponseFunctionCallOutputItem {
                         call_id: tool_result.tool_use_id.to_string(),
                         output: match tool_result.content {
-                            LanguageModelToolResultContent::Text(text) => text.to_string(),
-                            LanguageModelToolResultContent::Image(image) => image.to_base64_url(),
+                            LanguageModelToolResultContent::Text(text) => {
+                                ResponseFunctionCallOutputContent::Text(text.to_string())
+                            }
+                            LanguageModelToolResultContent::Image(image) => {
+                                ResponseFunctionCallOutputContent::List(vec![
+                                    ResponseInputContent::Image {
+                                        image_url: image.to_base64_url(),
+                                    },
+                                ])
+                            }
                         },
                     },
                 ));

crates/open_ai/src/responses.rs

@@ -55,7 +55,14 @@ pub struct ResponseFunctionCallItem {
 #[derive(Debug, Serialize, Deserialize)]
 pub struct ResponseFunctionCallOutputItem {
     pub call_id: String,
-    pub output: String,
+    pub output: ResponseFunctionCallOutputContent,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(untagged)]
+pub enum ResponseFunctionCallOutputContent {
+    List(Vec<ResponseInputContent>),
+    Text(String),
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]