imageutil: add HEIC to PNG conversion support

Philip Zeyliger and Shelley created 3 months ago

Prompt: make the screenshot tool work with heic?

Add IsHEIC() to detect HEIC/HEIF images by file magic (ftyp box brands).
Add ConvertHEICToPNG() using ImageMagick's convert command.
Update read_image tool to auto-convert HEIC images before processing.

This enables the read_image tool to handle iPhone photos uploaded in
HEIC format, which Go's standard image library doesn't support.

Co-authored-by: Shelley <[e-mail]>

Change summary

claudetool/browse/browse.go | 13 ++++++++
llm/imageutil/heic.go       | 40 +++++++++++++++++++++++++++
llm/imageutil/heic_test.go  | 57 +++++++++++++++++++++++++++++++++++++++
3 files changed, 110 insertions(+)

Detailed changes

claudetool/browse/browse.go

@@ -664,6 +664,16 @@ func (b *BrowseTools) readImageRun(ctx context.Context, m json.RawMessage) llm.T
 		return llm.ErrorfToolOut("failed to read image file: %w", err)
 	}
 
+	// Convert HEIC to PNG if needed (Go's image library doesn't support HEIC)
+	converted := false
+	if imageutil.IsHEIC(imageData) {
+		imageData, err = imageutil.ConvertHEICToPNG(imageData)
+		if err != nil {
+			return llm.ErrorfToolOut("failed to convert HEIC image: %w", err)
+		}
+		converted = true
+	}
+
 	detectedType := http.DetectContentType(imageData)
 	if !strings.HasPrefix(detectedType, "image/") {
 		return llm.ErrorfToolOut("file is not an image: %s", detectedType)
@@ -684,6 +694,9 @@ func (b *BrowseTools) readImageRun(ctx context.Context, m json.RawMessage) llm.T
 	mediaType := "image/" + format
 
 	description := fmt.Sprintf("Image from %s (type: %s)", input.Path, mediaType)
+	if converted {
+		description += " [converted from HEIC]"
+	}
 	if resized {
 		description += " [resized]"
 	}

llm/imageutil/heic.go

@@ -0,0 +1,40 @@
+package imageutil
+
+import (
+	"bytes"
+	"fmt"
+	"os/exec"
+)
+
+// IsHEIC checks if data is a HEIC/HEIF image based on file magic.
+// HEIC files are ISO Base Media File Format containers with specific brand codes.
+func IsHEIC(data []byte) bool {
+	if len(data) < 12 {
+		return false
+	}
+	// ftyp box starts at offset 4, brand at offset 8
+	// Common brands: heic, heix, hevc, hevx, mif1, msf1
+	if data[4] != 'f' || data[5] != 't' || data[6] != 'y' || data[7] != 'p' {
+		return false
+	}
+	brand := string(data[8:12])
+	switch brand {
+	case "heic", "heix", "hevc", "hevx", "mif1", "msf1", "avif":
+		return true
+	}
+	return false
+}
+
+// ConvertHEICToPNG converts HEIC image data to PNG using ImageMagick's convert command.
+// Returns the PNG data or an error if conversion fails.
+func ConvertHEICToPNG(data []byte) ([]byte, error) {
+	cmd := exec.Command("convert", "heic:-", "png:-")
+	cmd.Stdin = bytes.NewReader(data)
+	var stdout, stderr bytes.Buffer
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+	if err := cmd.Run(); err != nil {
+		return nil, fmt.Errorf("convert heic to png: %w: %s", err, stderr.String())
+	}
+	return stdout.Bytes(), nil
+}

llm/imageutil/heic_test.go

@@ -0,0 +1,57 @@
+package imageutil
+
+import (
+	"bytes"
+	"image/png"
+	"os"
+	"testing"
+)
+
+func TestIsHEIC(t *testing.T) {
+	tests := []struct {
+		name string
+		data []byte
+		want bool
+	}{
+		{"empty", []byte{}, false},
+		{"too short", []byte{0, 0, 0, 0, 'f', 't', 'y', 'p'}, false},
+		{"heic brand", []byte{0, 0, 0, 0, 'f', 't', 'y', 'p', 'h', 'e', 'i', 'c'}, true},
+		{"heix brand", []byte{0, 0, 0, 0, 'f', 't', 'y', 'p', 'h', 'e', 'i', 'x'}, true},
+		{"mif1 brand", []byte{0, 0, 0, 0, 'f', 't', 'y', 'p', 'm', 'i', 'f', '1'}, true},
+		{"avif brand", []byte{0, 0, 0, 0, 'f', 't', 'y', 'p', 'a', 'v', 'i', 'f'}, true},
+		{"not ftyp", []byte{0, 0, 0, 0, 'x', 'x', 'x', 'x', 'h', 'e', 'i', 'c'}, false},
+		{"unknown brand", []byte{0, 0, 0, 0, 'f', 't', 'y', 'p', 'i', 's', 'o', 'm'}, false},
+		{"png", []byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 0}, false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := IsHEIC(tt.data); got != tt.want {
+				t.Errorf("IsHEIC() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestConvertHEICToPNG(t *testing.T) {
+	// Skip if no real HEIC test file available
+	testFile := "/tmp/shelley-screenshots/upload_349d2aa15d2b3e4e.heic"
+	data, err := os.ReadFile(testFile)
+	if err != nil {
+		t.Skipf("test HEIC file not available: %v", err)
+	}
+
+	if !IsHEIC(data) {
+		t.Fatal("test file should be detected as HEIC")
+	}
+
+	pngData, err := ConvertHEICToPNG(data)
+	if err != nil {
+		t.Fatalf("ConvertHEICToPNG failed: %v", err)
+	}
+
+	// Verify it's valid PNG
+	_, err = png.Decode(bytes.NewReader(pngData))
+	if err != nil {
+		t.Fatalf("result is not valid PNG: %v", err)
+	}
+}