From c49d386efdf71ba2069d4bf9c7e90d2d909476ea Mon Sep 17 00:00:00 2001 From: Gerardo Rodriguez Date: Wed, 21 Oct 2026 16:03:28 +0000 Subject: [PATCH 1/4] media: decode HEIC photos with libheif so grid images aren't cropped --- Dockerfile | 2 +- pkg/connector/media.go | 48 ++++++++++++++++++++++-------------------- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/Dockerfile b/Dockerfile index a0847df9..5fbc7201 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,7 +26,7 @@ FROM ${DOCKER_HUB}/alpine:3.23 ENV UID=1337 \ GID=1337 -RUN apk add --no-cache ffmpeg su-exec ca-certificates bash jq curl yq-go olm +RUN apk add --no-cache ffmpeg libheif-tools su-exec ca-certificates bash jq curl yq-go olm COPY --from=builder /build/matrix-line /usr/bin/matrix-line COPY ./docker-run.sh /docker-run.sh diff --git a/pkg/connector/media.go b/pkg/connector/media.go index a65c3add..8082a740 100644 --- a/pkg/connector/media.go +++ b/pkg/connector/media.go @@ -46,44 +46,46 @@ func convertHEICToJPEG(ctx context.Context, imageData []byte) ([]byte, error) { if err := ctx.Err(); err != nil { return nil, err } - input, err := os.CreateTemp("", "line-image-*.heic") + dir, err := os.MkdirTemp("", "line-heic-*") if err != nil { - return nil, fmt.Errorf("failed to create temporary HEIC file: %w", err) + return nil, fmt.Errorf("failed to create temporary HEIC directory: %w", err) } - inputPath := input.Name() - defer os.Remove(inputPath) + defer os.RemoveAll(dir) - if _, err = input.Write(imageData); err != nil { - input.Close() + inputPath := filepath.Join(dir, "input.heic") + outputPath := filepath.Join(dir, "output.jpg") + if err = os.WriteFile(inputPath, imageData, 0o600); err != nil { return nil, fmt.Errorf("failed to write temporary HEIC file: %w", err) } - if err = input.Close(); err != nil { - return nil, fmt.Errorf("failed to close temporary HEIC file: %w", err) - } - var output bytes.Buffer - cmd := exec.CommandContext(ctx, "ffmpeg", - "-hide_banner", "-loglevel", "error", - "-i", inputPath, - "-frames:v", "1", "-q:v", "2", - "-f", "image2pipe", "-vcodec", "mjpeg", "pipe:1", - ) - cmd.Stdout = &output - err = cmd.Run() - if err != nil { + // iPhone photos are HEIF grids of tiles. The ffmpeg CLI only decodes the + // first tile, while libheif assembles the full image. + var stderr bytes.Buffer + cmd := exec.CommandContext(ctx, "heif-dec", "--quiet", inputPath, outputPath) + cmd.Stderr = &stderr + if err = cmd.Run(); err != nil { if ctxErr := ctx.Err(); ctxErr != nil { - return nil, fmt.Errorf("ffmpeg failed: %w", ctxErr) + return nil, fmt.Errorf("heif-dec failed: %w", ctxErr) } - return nil, fmt.Errorf("ffmpeg failed: %w", err) + return nil, fmt.Errorf("heif-dec failed: %w: %s", err, strings.TrimSpace(stderr.String())) } - jpegData := output.Bytes() + // heif-dec writes output-1.jpg, output-2.jpg, ... instead of output.jpg + // when the file holds several top-level images. Use the first one. + outputs, err := filepath.Glob(filepath.Join(dir, "output*.jpg")) + if err != nil || len(outputs) == 0 { + return nil, fmt.Errorf("heif-dec produced no output") + } + jpegData, err := os.ReadFile(outputs[0]) + if err != nil { + return nil, fmt.Errorf("failed to read converted JPEG: %w", err) + } config, format, err := image.DecodeConfig(bytes.NewReader(jpegData)) if err != nil { return nil, fmt.Errorf("failed to validate converted JPEG: %w", err) } if format != "jpeg" || config.Width <= 0 || config.Height <= 0 { - return nil, fmt.Errorf("ffmpeg produced invalid JPEG output") + return nil, fmt.Errorf("heif-dec produced invalid JPEG output") } return jpegData, nil } From 6a6cc20c7cd436ec4fa75bd26a8e891316a44da7 Mon Sep 17 00:00:00 2001 From: Gerardo Rodriguez Date: Wed, 21 Oct 2026 16:16:53 +0000 Subject: [PATCH 2/4] media: strip EXIF from converted HEIC photos --- pkg/connector/media.go | 29 ++++++++++++++++++++++++++++- pkg/connector/media_test.go | 31 +++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 pkg/connector/media_test.go diff --git a/pkg/connector/media.go b/pkg/connector/media.go index 8082a740..66c38cf5 100644 --- a/pkg/connector/media.go +++ b/pkg/connector/media.go @@ -87,7 +87,34 @@ func convertHEICToJPEG(ctx context.Context, imageData []byte) ([]byte, error) { if format != "jpeg" || config.Width <= 0 || config.Height <= 0 { return nil, fmt.Errorf("heif-dec produced invalid JPEG output") } - return jpegData, nil + return stripJPEGExif(jpegData), nil +} + +// stripJPEGExif drops APP1 Exif segments, which heif-dec copies from the HEIC +// (GPS location, device model). Rotation is already applied to the pixels. +func stripJPEGExif(data []byte) []byte { + if len(data) < 4 || data[0] != 0xFF || data[1] != 0xD8 { + return data + } + out := make([]byte, 0, len(data)) + out = append(out, data[:2]...) + pos := 2 + for pos+4 <= len(data) && data[pos] == 0xFF { + marker := data[pos+1] + if marker == 0xDA { + break + } + length := int(data[pos+2])<<8 | int(data[pos+3]) + end := pos + 2 + length + if length < 2 || end > len(data) { + return data + } + if marker != 0xE1 || !bytes.HasPrefix(data[pos+4:end], []byte("Exif\x00\x00")) { + out = append(out, data[pos:end]...) + } + pos = end + } + return append(out, data[pos:]...) } // AES-256-CTR diff --git a/pkg/connector/media_test.go b/pkg/connector/media_test.go new file mode 100644 index 00000000..1b136747 --- /dev/null +++ b/pkg/connector/media_test.go @@ -0,0 +1,31 @@ +package connector + +import ( + "bytes" + "testing" +) + +func TestStripJPEGExif(t *testing.T) { + segment := func(marker byte, payload string) []byte { + n := len(payload) + 2 + return append([]byte{0xFF, marker, byte(n >> 8), byte(n)}, payload...) + } + soi := []byte{0xFF, 0xD8} + jfif := segment(0xE0, "JFIF\x00") + exif := segment(0xE1, "Exif\x00\x00MM\x00*GPS") + xmp := segment(0xE1, "http://ns.adobe.com/xap/1.0/\x00") + icc := segment(0xE2, "ICC_PROFILE\x00") + scan := []byte{0xFF, 0xDA, 0x00, 0x02, 0x12, 0x34, 0xFF, 0xD9} + + join := func(parts ...[]byte) []byte { return bytes.Join(parts, nil) } + input := join(soi, jfif, exif, xmp, icc, scan) + want := join(soi, jfif, xmp, icc, scan) + if got := stripJPEGExif(input); !bytes.Equal(got, want) { + t.Fatalf("stripJPEGExif() = %x, want %x", got, want) + } + + truncated := join(soi, segment(0xE1, "Exif\x00\x00")[:5]) + if got := stripJPEGExif(truncated); !bytes.Equal(got, truncated) { + t.Fatalf("malformed input should be returned unchanged") + } +} From 473a989c681c6d14eb50ce5b338e7562b437a8b9 Mon Sep 17 00:00:00 2001 From: Gerardo Rodriguez Date: Wed, 21 Oct 2026 16:33:51 +0000 Subject: [PATCH 3/4] media: strip all APP1 metadata from converted HEIC photos --- pkg/connector/media.go | 11 ++++++----- pkg/connector/media_test.go | 17 +++++++++-------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/pkg/connector/media.go b/pkg/connector/media.go index 66c38cf5..0cca8b21 100644 --- a/pkg/connector/media.go +++ b/pkg/connector/media.go @@ -87,12 +87,13 @@ func convertHEICToJPEG(ctx context.Context, imageData []byte) ([]byte, error) { if format != "jpeg" || config.Width <= 0 || config.Height <= 0 { return nil, fmt.Errorf("heif-dec produced invalid JPEG output") } - return stripJPEGExif(jpegData), nil + return stripJPEGMetadata(jpegData), nil } -// stripJPEGExif drops APP1 Exif segments, which heif-dec copies from the HEIC -// (GPS location, device model). Rotation is already applied to the pixels. -func stripJPEGExif(data []byte) []byte { +// stripJPEGMetadata drops every APP1 segment. heif-dec copies the HEIC's EXIF +// (split over several APP1 segments when large) and XMP into them, including +// GPS location and device model. Rotation is already applied to the pixels. +func stripJPEGMetadata(data []byte) []byte { if len(data) < 4 || data[0] != 0xFF || data[1] != 0xD8 { return data } @@ -109,7 +110,7 @@ func stripJPEGExif(data []byte) []byte { if length < 2 || end > len(data) { return data } - if marker != 0xE1 || !bytes.HasPrefix(data[pos+4:end], []byte("Exif\x00\x00")) { + if marker != 0xE1 { out = append(out, data[pos:end]...) } pos = end diff --git a/pkg/connector/media_test.go b/pkg/connector/media_test.go index 1b136747..3f2b8c5f 100644 --- a/pkg/connector/media_test.go +++ b/pkg/connector/media_test.go @@ -5,27 +5,28 @@ import ( "testing" ) -func TestStripJPEGExif(t *testing.T) { +func TestStripJPEGMetadata(t *testing.T) { segment := func(marker byte, payload string) []byte { n := len(payload) + 2 return append([]byte{0xFF, marker, byte(n >> 8), byte(n)}, payload...) } soi := []byte{0xFF, 0xD8} jfif := segment(0xE0, "JFIF\x00") - exif := segment(0xE1, "Exif\x00\x00MM\x00*GPS") - xmp := segment(0xE1, "http://ns.adobe.com/xap/1.0/\x00") + exif := segment(0xE1, "Exif\x00\x00MM\x00*") + exifContinuation := segment(0xE1, "GPS") + xmp := segment(0xE1, "http://ns.adobe.com/xap/1.0/\x00exif:GPSLatitude") icc := segment(0xE2, "ICC_PROFILE\x00") scan := []byte{0xFF, 0xDA, 0x00, 0x02, 0x12, 0x34, 0xFF, 0xD9} join := func(parts ...[]byte) []byte { return bytes.Join(parts, nil) } - input := join(soi, jfif, exif, xmp, icc, scan) - want := join(soi, jfif, xmp, icc, scan) - if got := stripJPEGExif(input); !bytes.Equal(got, want) { - t.Fatalf("stripJPEGExif() = %x, want %x", got, want) + input := join(soi, jfif, exif, exifContinuation, icc, xmp, scan) + want := join(soi, jfif, icc, scan) + if got := stripJPEGMetadata(input); !bytes.Equal(got, want) { + t.Fatalf("stripJPEGMetadata() = %x, want %x", got, want) } truncated := join(soi, segment(0xE1, "Exif\x00\x00")[:5]) - if got := stripJPEGExif(truncated); !bytes.Equal(got, truncated) { + if got := stripJPEGMetadata(truncated); !bytes.Equal(got, truncated) { t.Fatalf("malformed input should be returned unchanged") } } From 80dd43af757b537f86b8437265a9111cbb345385 Mon Sep 17 00:00:00 2001 From: Gerardo Rodriguez Date: Sat, 24 Oct 2026 10:46:08 +0000 Subject: [PATCH 4/4] media: fall back to PNG when heif-dec can't write a JPEG over broken EXIF --- pkg/connector/media.go | 52 ++++++++++++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/pkg/connector/media.go b/pkg/connector/media.go index 0cca8b21..482236d0 100644 --- a/pkg/connector/media.go +++ b/pkg/connector/media.go @@ -13,7 +13,7 @@ import ( "image" _ "image/gif" "image/jpeg" - _ "image/png" + "image/png" "io" "os" "os/exec" @@ -53,17 +53,41 @@ func convertHEICToJPEG(ctx context.Context, imageData []byte) ([]byte, error) { defer os.RemoveAll(dir) inputPath := filepath.Join(dir, "input.heic") - outputPath := filepath.Join(dir, "output.jpg") if err = os.WriteFile(inputPath, imageData, 0o600); err != nil { return nil, fmt.Errorf("failed to write temporary HEIC file: %w", err) } // iPhone photos are HEIF grids of tiles. The ffmpeg CLI only decodes the // first tile, while libheif assembles the full image. + jpegData, err := runHEIFDec(ctx, dir, inputPath, "jpg") + if err == nil { + return stripJPEGMetadata(jpegData), nil + } else if ctx.Err() != nil { + return nil, err + } + + // libheif's JPEG writer gives up on a broken EXIF block, while its PNG + // writer skips the block and writes the image. + pngData, pngErr := runHEIFDec(ctx, dir, inputPath, "png") + if pngErr != nil { + return nil, err + } + img, imgErr := png.Decode(bytes.NewReader(pngData)) + if imgErr != nil { + return nil, fmt.Errorf("failed to decode converted PNG: %w", imgErr) + } + var buf bytes.Buffer + if err = jpeg.Encode(&buf, img, &jpeg.Options{Quality: 90}); err != nil { + return nil, fmt.Errorf("failed to encode converted image: %w", err) + } + return buf.Bytes(), nil +} + +func runHEIFDec(ctx context.Context, dir, inputPath, extension string) ([]byte, error) { var stderr bytes.Buffer - cmd := exec.CommandContext(ctx, "heif-dec", "--quiet", inputPath, outputPath) + cmd := exec.CommandContext(ctx, "heif-dec", "--quiet", inputPath, filepath.Join(dir, "output."+extension)) cmd.Stderr = &stderr - if err = cmd.Run(); err != nil { + if err := cmd.Run(); err != nil { if ctxErr := ctx.Err(); ctxErr != nil { return nil, fmt.Errorf("heif-dec failed: %w", ctxErr) } @@ -72,22 +96,20 @@ func convertHEICToJPEG(ctx context.Context, imageData []byte) ([]byte, error) { // heif-dec writes output-1.jpg, output-2.jpg, ... instead of output.jpg // when the file holds several top-level images. Use the first one. - outputs, err := filepath.Glob(filepath.Join(dir, "output*.jpg")) + outputs, err := filepath.Glob(filepath.Join(dir, "output*."+extension)) if err != nil || len(outputs) == 0 { - return nil, fmt.Errorf("heif-dec produced no output") - } - jpegData, err := os.ReadFile(outputs[0]) - if err != nil { - return nil, fmt.Errorf("failed to read converted JPEG: %w", err) + return nil, fmt.Errorf("heif-dec produced no output: %s", strings.TrimSpace(stderr.String())) } - config, format, err := image.DecodeConfig(bytes.NewReader(jpegData)) + data, err := os.ReadFile(outputs[0]) if err != nil { - return nil, fmt.Errorf("failed to validate converted JPEG: %w", err) + return nil, fmt.Errorf("failed to read converted image: %w", err) } - if format != "jpeg" || config.Width <= 0 || config.Height <= 0 { - return nil, fmt.Errorf("heif-dec produced invalid JPEG output") + // heif-dec exits 0 even when its writer fails, leaving an empty file. + config, _, err := image.DecodeConfig(bytes.NewReader(data)) + if err != nil || config.Width <= 0 || config.Height <= 0 { + return nil, fmt.Errorf("heif-dec produced invalid %s output: %s", extension, strings.TrimSpace(stderr.String())) } - return stripJPEGMetadata(jpegData), nil + return data, nil } // stripJPEGMetadata drops every APP1 segment. heif-dec copies the HEIC's EXIF