diff --git a/claudetool/browse/browse.go b/claudetool/browse/browse.go index 359bf869fd8ab978f00f848051e2ed92a1f1f31b..deb2f7f84c98a9ef2c080935d1ed774ae367a64f 100644 --- a/claudetool/browse/browse.go +++ b/claudetool/browse/browse.go @@ -664,6 +664,16 @@ func (b *BrowseTools) readImageRun(ctx context.Context, m json.RawMessage) llm.T return llm.ErrorfToolOut("failed to read image file: %w", err) } + // Convert HEIC to PNG if needed (Go's image library doesn't support HEIC) + converted := false + if imageutil.IsHEIC(imageData) { + imageData, err = imageutil.ConvertHEICToPNG(imageData) + if err != nil { + return llm.ErrorfToolOut("failed to convert HEIC image: %w", err) + } + converted = true + } + detectedType := http.DetectContentType(imageData) if !strings.HasPrefix(detectedType, "image/") { return llm.ErrorfToolOut("file is not an image: %s", detectedType) @@ -684,6 +694,9 @@ func (b *BrowseTools) readImageRun(ctx context.Context, m json.RawMessage) llm.T mediaType := "image/" + format description := fmt.Sprintf("Image from %s (type: %s)", input.Path, mediaType) + if converted { + description += " [converted from HEIC]" + } if resized { description += " [resized]" } diff --git a/llm/imageutil/heic.go b/llm/imageutil/heic.go new file mode 100644 index 0000000000000000000000000000000000000000..cb6e5b3d6ec8848c2198697f91daa7c202218998 --- /dev/null +++ b/llm/imageutil/heic.go @@ -0,0 +1,40 @@ +package imageutil + +import ( + "bytes" + "fmt" + "os/exec" +) + +// IsHEIC checks if data is a HEIC/HEIF image based on file magic. +// HEIC files are ISO Base Media File Format containers with specific brand codes. +func IsHEIC(data []byte) bool { + if len(data) < 12 { + return false + } + // ftyp box starts at offset 4, brand at offset 8 + // Common brands: heic, heix, hevc, hevx, mif1, msf1 + if data[4] != 'f' || data[5] != 't' || data[6] != 'y' || data[7] != 'p' { + return false + } + brand := string(data[8:12]) + switch brand { + case "heic", "heix", "hevc", "hevx", "mif1", "msf1", "avif": + return true + } + return false +} + +// ConvertHEICToPNG converts HEIC image data to PNG using ImageMagick's convert command. +// Returns the PNG data or an error if conversion fails. +func ConvertHEICToPNG(data []byte) ([]byte, error) { + cmd := exec.Command("convert", "heic:-", "png:-") + cmd.Stdin = bytes.NewReader(data) + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("convert heic to png: %w: %s", err, stderr.String()) + } + return stdout.Bytes(), nil +} diff --git a/llm/imageutil/heic_test.go b/llm/imageutil/heic_test.go new file mode 100644 index 0000000000000000000000000000000000000000..0fa361a2286f8002278b8e5ae53e3e94e4b44750 --- /dev/null +++ b/llm/imageutil/heic_test.go @@ -0,0 +1,57 @@ +package imageutil + +import ( + "bytes" + "image/png" + "os" + "testing" +) + +func TestIsHEIC(t *testing.T) { + tests := []struct { + name string + data []byte + want bool + }{ + {"empty", []byte{}, false}, + {"too short", []byte{0, 0, 0, 0, 'f', 't', 'y', 'p'}, false}, + {"heic brand", []byte{0, 0, 0, 0, 'f', 't', 'y', 'p', 'h', 'e', 'i', 'c'}, true}, + {"heix brand", []byte{0, 0, 0, 0, 'f', 't', 'y', 'p', 'h', 'e', 'i', 'x'}, true}, + {"mif1 brand", []byte{0, 0, 0, 0, 'f', 't', 'y', 'p', 'm', 'i', 'f', '1'}, true}, + {"avif brand", []byte{0, 0, 0, 0, 'f', 't', 'y', 'p', 'a', 'v', 'i', 'f'}, true}, + {"not ftyp", []byte{0, 0, 0, 0, 'x', 'x', 'x', 'x', 'h', 'e', 'i', 'c'}, false}, + {"unknown brand", []byte{0, 0, 0, 0, 'f', 't', 'y', 'p', 'i', 's', 'o', 'm'}, false}, + {"png", []byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 0}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := IsHEIC(tt.data); got != tt.want { + t.Errorf("IsHEIC() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestConvertHEICToPNG(t *testing.T) { + // Skip if no real HEIC test file available + testFile := "/tmp/shelley-screenshots/upload_349d2aa15d2b3e4e.heic" + data, err := os.ReadFile(testFile) + if err != nil { + t.Skipf("test HEIC file not available: %v", err) + } + + if !IsHEIC(data) { + t.Fatal("test file should be detected as HEIC") + } + + pngData, err := ConvertHEICToPNG(data) + if err != nil { + t.Fatalf("ConvertHEICToPNG failed: %v", err) + } + + // Verify it's valid PNG + _, err = png.Decode(bytes.NewReader(pngData)) + if err != nil { + t.Fatalf("result is not valid PNG: %v", err) + } +}