From d671230edad79da92195a6bb2165e521b6726c69 Mon Sep 17 00:00:00 2001
From: Philip Zeyliger <philip@bold.dev>
Date: Sun, 11 Jan 2026 04:58:56 +0000
Subject: [PATCH] shelley: display image in read_image tool using base64 data
 from toolResult

Prompt: In a new worktree: have the read image tool display the image being read. It's in the client data already so we may as well display it. See the screenshot tool for an example.

The read_image tool already returns the image as base64 in the toolResult.
Use that data directly to render the image as a data URL instead of
requiring a separate Display field with a URL path.
---
 ui/src/components/Message.tsx       |  1 -
 ui/src/components/ReadImageTool.tsx | 28 +++++++---------------------
 2 files changed, 7 insertions(+), 22 deletions(-)

diff --git a/ui/src/components/Message.tsx b/ui/src/components/Message.tsx
index 2fb157743ac1862b8a9dab6ac15ce212b2c02029..dc59bdfe30b1da1adddcb335bd56d4697cd972ef 100644
--- a/ui/src/components/Message.tsx
+++ b/ui/src/components/Message.tsx
@@ -575,7 +575,6 @@ function Message({ message, onOpenDiffViewer, onCommentTextChange }: MessageProp
               toolResult={content.ToolResult}
               hasError={hasError}
               executionTime={executionTime}
-              display={content.Display}
             />
           );
         }
diff --git a/ui/src/components/ReadImageTool.tsx b/ui/src/components/ReadImageTool.tsx
index 14a39cc38923e8d9d309a3ce0f612a8e8ded2216..de96eb8438238121d3afac4822a9ebd97ebe9754 100644
--- a/ui/src/components/ReadImageTool.tsx
+++ b/ui/src/components/ReadImageTool.tsx
@@ -7,7 +7,6 @@ interface ReadImageToolProps {
   toolResult?: LLMContent[];
   hasError?: boolean;
   executionTime?: string;
-  display?: unknown; // Display data from the tool_result Content
 }
 
 function ReadImageTool({
@@ -16,7 +15,6 @@ function ReadImageTool({
   toolResult,
   hasError,
   executionTime,
-  display,
 }: ReadImageToolProps) {
   const [isExpanded, setIsExpanded] = useState(true); // Default to expanded
 
@@ -47,26 +45,14 @@ function ReadImageTool({
 
   const filename = getPath(toolInput) || getId(toolInput) || "image";
 
-  // Use display data passed as prop (from tool_result Content.Display)
-  const displayData = display;
-
-  // Construct image URL
+  // Build image URL from the base64 data in toolResult
+  // The read_image tool returns [text description, image content with Data and MediaType]
   let imageUrl: string | undefined = undefined;
-  if (displayData && typeof displayData === "object" && displayData !== null) {
-    const url =
-      "url" in displayData && typeof displayData.url === "string" ? displayData.url : undefined;
-    const path =
-      "path" in displayData && typeof displayData.path === "string" ? displayData.path : undefined;
-    const id =
-      "id" in displayData && typeof displayData.id === "string" ? displayData.id : undefined;
-
-    imageUrl =
-      url ||
-      (path
-        ? `/api/read?path=${encodeURIComponent(path)}`
-        : id
-          ? `/api/read?path=${encodeURIComponent(id)}`
-          : undefined);
+  if (toolResult && toolResult.length >= 2) {
+    const imageContent = toolResult[1];
+    if (imageContent?.Data && imageContent?.MediaType) {
+      imageUrl = `data:${imageContent.MediaType};base64,${imageContent.Data}`;
+    }
   }
 
   const isComplete = !isRunning && toolResult !== undefined;