From abd3b1cdda8e613f2971b619e0cd21eac17ebcef Mon Sep 17 00:00:00 2001 From: Philip Zeyliger Date: Mon, 19 Jan 2026 19:49:45 -0800 Subject: [PATCH] shelley: add output_iframe tool for sandboxed HTML visualization We'll see if this sticks and is found useful. You can't really have the LLM output HTML, because it ends up dangerous in the timeline. But you can embed an iframe. So let's give the LLM a way to do that! Prompt: I want Shelley to be able to produce HTML output that the user can see, but for security, I want this to be made safe by using an iframe. For example, if I ask Shelley to produce a vegalite visualization, I want the agent to be able to produce sandboxed HTML to do that. Create a new tool "output_iframe" that should be used for visual elements if the agent wants to display them, and should be expanded by default in the HTML tool output. Advise the tool description to only use this for visualizations like graphs and demonstrations of HTML, and to use regular messages for chatting with the user. Add a new tool 'output_iframe' that allows the agent to display HTML content to the user in a secure sandboxed iframe. This is useful for: - Vega-Lite and other chart library visualizations - HTML/CSS demonstrations - Interactive widgets - SVG graphics The tool: - Takes HTML content and optional title as input - Returns 'displayed' to the LLM (minimal response) - Passes HTML to UI via Display field for rendering - Uses sandbox="allow-scripts" for security (scripts run but can't access parent) The UI component: - Renders iframe with srcdoc for the HTML - Expanded by default (like screenshot tool) - Styled consistently with other tool components Tool description explicitly advises using this only for visualizations, not for regular chat messages. Co-authored-by: Shelley --- AGENTS.md | 11 +- claudetool/output_iframe.go | 87 +++++++ claudetool/output_iframe_test.go | 106 +++++++++ claudetool/toolset.go | 1 + ui/src/components/ChatInterface.tsx | 2 + ui/src/components/Message.tsx | 19 ++ ui/src/components/OutputIframeTool.tsx | 310 +++++++++++++++++++++++++ ui/src/styles.css | 161 +++++++++++++ 8 files changed, 692 insertions(+), 5 deletions(-) create mode 100644 claudetool/output_iframe.go create mode 100644 claudetool/output_iframe_test.go create mode 100644 ui/src/components/OutputIframeTool.tsx diff --git a/AGENTS.md b/AGENTS.md index 516e5923ef5c1a0c52c80dca5b3bde97e8392d55..9203f930028e00e54849f1963328950ee2901452 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,8 +8,9 @@ and can fabricate some inputs and outputs. To test things, launch shelley with the relevant flag to only expose this model, and use shelley with a browser. 6. Build the UI (`make ui` or `cd ui && pnpm install && pnpm run build`) before running Go tests so `ui/dist` exists for the embed. -7. Run Go unit tests with `go test ./server` (or narrower packages while iterating) once the UI bundle is built. -8. To programmatically type into the React message input (e.g., in browser automation), you must use React's internal setter: +7. Run TypeScript type checking with `cd ui && pnpm run type-check`. Run linting with `pnpm run lint`. +8. Run Go unit tests with `go test ./server` (or narrower packages while iterating) once the UI bundle is built. +9. To programmatically type into the React message input (e.g., in browser automation), you must use React's internal setter: ```javascript const input = document.querySelector('[data-testid="message-input"]'); const nativeInputValueSetter = Object.getOwnPropertyDescriptor(window.HTMLTextAreaElement.prototype, "value").set; @@ -17,10 +18,10 @@ input.dispatchEvent(new Event('input', { bubbles: true })); ``` Simply setting `input.value = '...'` won't work because React won't detect the change. -9. Commit your changes before finishing your turn. -10. If you are testing Shelley itself, be aware that you might be running "under" shelley, +10. Commit your changes before finishing your turn. +11. If you are testing Shelley itself, be aware that you might be running "under" shelley, and indiscriminately running pkill -f shelley may break things. -11. To test the Shelley UI in a separate instance, build with `make build`, then run on a +12. To test the Shelley UI in a separate instance, build with `make build`, then run on a different port with a separate database: ``` ./bin/shelley -config /exe.dev/shelley.json -db /tmp/shelley-test.db serve -port 8002 diff --git a/claudetool/output_iframe.go b/claudetool/output_iframe.go new file mode 100644 index 0000000000000000000000000000000000000000..ff7d3c8f09a59d8a0dcb3576f71674b8e4929259 --- /dev/null +++ b/claudetool/output_iframe.go @@ -0,0 +1,87 @@ +package claudetool + +import ( + "context" + "encoding/json" + + "shelley.exe.dev/llm" +) + +// OutputIframeTool displays sandboxed HTML content to the user. +var OutputIframeTool = &llm.Tool{ + Name: outputIframeName, + Description: outputIframeDescription, + InputSchema: llm.MustSchema(outputIframeInputSchema), + Run: outputIframeRun, +} + +const ( + outputIframeName = "output_iframe" + outputIframeDescription = `Display HTML content to the user in a sandboxed iframe. + +Use this tool for visualizations like charts, graphs, and HTML demos that the user should see. +The HTML will be rendered in a secure sandbox with scripts enabled but isolated from the parent page. + +Do NOT use this tool for: +- Regular text responses (use normal messages instead) +- File operations (use patch or bash) +- Simple data display (just describe it in text) + +Good uses: +- Vega-Lite or other chart library visualizations +- HTML/CSS demonstrations +- Interactive widgets or mini-apps +- SVG graphics + +The HTML should be self-contained. You can include inline +`; + +const MIN_HEIGHT = 100; +const MAX_HEIGHT = 600; + +function OutputIframeTool({ + toolInput, + isRunning, + toolResult, + hasError, + executionTime, + display, +}: OutputIframeToolProps) { + // Default to expanded for visual content + const [isExpanded, setIsExpanded] = useState(true); + const [iframeHeight, setIframeHeight] = useState(300); + const iframeRef = useRef(null); + + // Extract input data + const getTitle = (input: unknown): string | undefined => { + if ( + typeof input === "object" && + input !== null && + "title" in input && + typeof input.title === "string" + ) { + return input.title; + } + return undefined; + }; + + const getHtmlFromInput = (input: unknown): string | undefined => { + if ( + typeof input === "object" && + input !== null && + "html" in input && + typeof input.html === "string" + ) { + return input.html; + } + return undefined; + }; + + // Get display data - prefer from display prop, fall back to toolInput + const getDisplayData = (): { html?: string; title?: string } => { + // First try display prop (from tool result) + if (display && typeof display === "object" && display !== null) { + const d = display as { html?: string; title?: string }; + return { + html: typeof d.html === "string" ? d.html : undefined, + title: typeof d.title === "string" ? d.title : undefined, + }; + } + // Fall back to toolInput + return { + html: getHtmlFromInput(toolInput), + title: getTitle(toolInput), + }; + }; + + const displayData = getDisplayData(); + const title = displayData.title || "HTML Output"; + const html = displayData.html; + + // Inject height reporter script into HTML + const htmlWithHeightReporter = html + ? html.includes("") + ? html.replace("", HEIGHT_REPORTER_SCRIPT + "") + : html + HEIGHT_REPORTER_SCRIPT + : undefined; + + // Listen for height messages from iframe + const handleMessage = useCallback((event: MessageEvent) => { + if ( + event.data && + typeof event.data === "object" && + event.data.type === "iframe-height" && + typeof event.data.height === "number" + ) { + // Verify the message is from our iframe + if (iframeRef.current && event.source === iframeRef.current.contentWindow) { + const newHeight = Math.min(Math.max(event.data.height, MIN_HEIGHT), MAX_HEIGHT); + setIframeHeight(newHeight); + } + } + }, []); + + useEffect(() => { + window.addEventListener("message", handleMessage); + return () => window.removeEventListener("message", handleMessage); + }, [handleMessage]); + + // Escape HTML special characters for safe embedding + const escapeHtml = (str: string): string => { + return str + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """) + .replace(/'/g, "'"); + }; + + // Open HTML in new tab with sandbox protection + const handleOpenInNewTab = (e: React.MouseEvent) => { + e.stopPropagation(); + if (!html) return; + + // Create a wrapper HTML page that embeds the content in a sandboxed iframe + // This preserves security even when opened in a new tab + const escapedHtml = escapeHtml(html); + const escapedTitle = escapeHtml(title); + + const wrapperHtml = ` + + + + ${escapedTitle} + + + + + +`; + + const blob = new Blob([wrapperHtml], { type: "text/html" }); + const url = URL.createObjectURL(blob); + window.open(url, "_blank"); + // Clean up the URL after a delay + setTimeout(() => URL.revokeObjectURL(url), 1000); + }; + + const isComplete = !isRunning && toolResult !== undefined; + + return ( +
+
setIsExpanded(!isExpanded)}> +
+ ✨ + {title} + {isComplete && hasError && ✗} + {isComplete && !hasError && ✓} +
+
+ {isComplete && !hasError && html && ( + + )} + +
+
+ + {isExpanded && ( +
+ {isComplete && !hasError && htmlWithHeightReporter && ( +
+ {executionTime && ( +
+ Output: + {executionTime} +
+ )} +
+