From 40af9dc982aefa2399a67b654401885c5adf6795 Mon Sep 17 00:00:00 2001 From: "zed-zippy[bot]" <234243425+zed-zippy[bot]@users.noreply.github.com> Date: Tue, 17 Mar 2026 22:26:34 +0000 Subject: [PATCH] open_ai: Fix tool output for /responses endpoint (#51789) (cherry-pick to stable) (#51792) Cherry-pick of #51789 to stable ---- We were sending the raw tool debug output as JSON to the model rather than whatever the tool intended as content for the model. Which meant we were sending unneeded information to the model, which matters in the edit tool case. Release Notes: - N/A Co-authored-by: Ben Brandt --- crates/language_model/src/request.rs | 2 + .../src/provider/copilot_chat.rs | 46 +++++++------------ .../language_models/src/provider/open_ai.rs | 31 ++++--------- 3 files changed, 29 insertions(+), 50 deletions(-) diff --git a/crates/language_model/src/request.rs b/crates/language_model/src/request.rs index 9be3002deae758ee99432842a31e3b90754ada0f..3e8408f8334a52c2c807f991535cbbbd79800c4b 100644 --- a/crates/language_model/src/request.rs +++ b/crates/language_model/src/request.rs @@ -234,7 +234,9 @@ pub struct LanguageModelToolResult { pub tool_use_id: LanguageModelToolUseId, pub tool_name: Arc, pub is_error: bool, + /// The tool output formatted for presenting to the model pub content: LanguageModelToolResultContent, + /// The raw tool output, if available, often for debugging or extra state for replay pub output: Option, } diff --git a/crates/language_models/src/provider/copilot_chat.rs b/crates/language_models/src/provider/copilot_chat.rs index 4363430f865de63ed5fec0d6b40b085d9413fc2a..6af42910afc676252c210e3cb15b4b75469b7038 100644 --- a/crates/language_models/src/provider/copilot_chat.rs +++ b/crates/language_models/src/provider/copilot_chat.rs @@ -940,38 +940,26 @@ fn into_copilot_responses( Role::User => { for content in &message.content { if let MessageContent::ToolResult(tool_result) = content { - let output = if let Some(out) = &tool_result.output { - match out { - serde_json::Value::String(s) => { - responses::ResponseFunctionOutput::Text(s.clone()) - } - serde_json::Value::Null => { - responses::ResponseFunctionOutput::Text(String::new()) - } - other => responses::ResponseFunctionOutput::Text(other.to_string()), + let output = match &tool_result.content { + LanguageModelToolResultContent::Text(text) => { + responses::ResponseFunctionOutput::Text(text.to_string()) } - } else { - match &tool_result.content { - LanguageModelToolResultContent::Text(text) => { - responses::ResponseFunctionOutput::Text(text.to_string()) - } - LanguageModelToolResultContent::Image(image) => { - if model.supports_vision() { - responses::ResponseFunctionOutput::Content(vec![ - responses::ResponseInputContent::InputImage { - image_url: Some(image.to_base64_url()), - detail: Default::default(), - }, - ]) - } else { - debug_panic!( - "This should be caught at {} level", - tool_result.tool_name - ); - responses::ResponseFunctionOutput::Text( + LanguageModelToolResultContent::Image(image) => { + if model.supports_vision() { + responses::ResponseFunctionOutput::Content(vec![ + responses::ResponseInputContent::InputImage { + image_url: Some(image.to_base64_url()), + detail: Default::default(), + }, + ]) + } else { + debug_panic!( + "This should be caught at {} level", + tool_result.tool_name + ); + responses::ResponseFunctionOutput::Text( "[Tool responded with an image, but this model does not support vision]".into(), ) - } } } }; diff --git a/crates/language_models/src/provider/open_ai.rs b/crates/language_models/src/provider/open_ai.rs index 24887dda575061505749d328c07669fe4d069a9f..44fd3f293fa4e787007d5351c4291235875222a8 100644 --- a/crates/language_models/src/provider/open_ai.rs +++ b/crates/language_models/src/provider/open_ai.rs @@ -9,9 +9,8 @@ use language_model::{ LanguageModelCompletionEvent, LanguageModelId, LanguageModelImage, LanguageModelName, LanguageModelProvider, LanguageModelProviderId, LanguageModelProviderName, LanguageModelProviderState, LanguageModelRequest, LanguageModelRequestMessage, - LanguageModelToolChoice, LanguageModelToolResult, LanguageModelToolResultContent, - LanguageModelToolUse, LanguageModelToolUseId, MessageContent, RateLimiter, Role, StopReason, - TokenUsage, env_var, + LanguageModelToolChoice, LanguageModelToolResultContent, LanguageModelToolUse, + LanguageModelToolUseId, MessageContent, RateLimiter, Role, StopReason, TokenUsage, env_var, }; use menu; use open_ai::responses::{ @@ -635,7 +634,10 @@ fn append_message_to_response_items( input_items.push(ResponseInputItem::FunctionCallOutput( ResponseFunctionCallOutputItem { call_id: tool_result.tool_use_id.to_string(), - output: tool_result_output(&tool_result), + output: match tool_result.content { + LanguageModelToolResultContent::Text(text) => text.to_string(), + LanguageModelToolResultContent::Image(image) => image.to_base64_url(), + }, }, )); } @@ -703,21 +705,6 @@ fn flush_response_parts( parts.clear(); } -fn tool_result_output(result: &LanguageModelToolResult) -> String { - if let Some(output) = &result.output { - match output { - serde_json::Value::String(text) => text.clone(), - serde_json::Value::Null => String::new(), - _ => output.to_string(), - } - } else { - match &result.content { - LanguageModelToolResultContent::Text(text) => text.to_string(), - LanguageModelToolResultContent::Image(image) => image.to_base64_url(), - } - } -} - fn add_message_content_part( new_part: open_ai::MessagePart, role: Role, @@ -1379,7 +1366,9 @@ impl Render for ConfigurationView { mod tests { use futures::{StreamExt, executor::block_on}; use gpui::TestAppContext; - use language_model::{LanguageModelRequestMessage, LanguageModelRequestTool}; + use language_model::{ + LanguageModelRequestMessage, LanguageModelRequestTool, LanguageModelToolResult, + }; use open_ai::responses::{ ResponseFunctionToolCall, ResponseOutputItem, ResponseOutputMessage, ResponseStatusDetails, ResponseSummary, ResponseUsage, StreamEvent as ResponsesStreamEvent, @@ -1624,7 +1613,7 @@ mod tests { { "type": "function_call_output", "call_id": "call-42", - "output": "{\"forecast\":\"Sunny\"}" + "output": "Sunny" } ], "stream": true,