media: decode HEIC photos with libheif so grid images aren't cropped

Dockerfile

@@ -26,7 +26,7 @@ FROM ${DOCKER_HUB}/alpine:3.23
 ENV UID=1337 \
     GID=1337
 
-RUN apk add --no-cache ffmpeg su-exec ca-certificates bash jq curl yq-go olm
+RUN apk add --no-cache ffmpeg libheif-tools su-exec ca-certificates bash jq curl yq-go olm
 
 COPY --from=builder /build/matrix-line /usr/bin/matrix-line
 COPY ./docker-run.sh /docker-run.sh

pkg/connector/media.go

@@ -13,7 +13,7 @@ import (
 	"image"
 	_ "image/gif"
 	"image/jpeg"
-	_ "image/png"
+	"image/png"
 	"io"
 	"os"
 	"os/exec"
@@ -46,46 +46,98 @@ func convertHEICToJPEG(ctx context.Context, imageData []byte) ([]byte, error) {
 	if err := ctx.Err(); err != nil {
 		return nil, err
 	}
-	input, err := os.CreateTemp("", "line-image-*.heic")
+	dir, err := os.MkdirTemp("", "line-heic-*")
 	if err != nil {
-		return nil, fmt.Errorf("failed to create temporary HEIC file: %w", err)
+		return nil, fmt.Errorf("failed to create temporary HEIC directory: %w", err)
 	}
-	inputPath := input.Name()
-	defer os.Remove(inputPath)
+	defer os.RemoveAll(dir)
 
-	if _, err = input.Write(imageData); err != nil {
-		input.Close()
+	inputPath := filepath.Join(dir, "input.heic")
+	if err = os.WriteFile(inputPath, imageData, 0o600); err != nil {
 		return nil, fmt.Errorf("failed to write temporary HEIC file: %w", err)
 	}
-	if err = input.Close(); err != nil {
-		return nil, fmt.Errorf("failed to close temporary HEIC file: %w", err)
+
+	// iPhone photos are HEIF grids of tiles. The ffmpeg CLI only decodes the
+	// first tile, while libheif assembles the full image.
+	jpegData, err := runHEIFDec(ctx, dir, inputPath, "jpg")
+	if err == nil {
+		return stripJPEGMetadata(jpegData), nil
+	} else if ctx.Err() != nil {
+		return nil, err
 	}
 
-	var output bytes.Buffer
-	cmd := exec.CommandContext(ctx, "ffmpeg",
-		"-hide_banner", "-loglevel", "error",
-		"-i", inputPath,
-		"-frames:v", "1", "-q:v", "2",
-		"-f", "image2pipe", "-vcodec", "mjpeg", "pipe:1",
-	)
-	cmd.Stdout = &output
-	err = cmd.Run()
-	if err != nil {
+	// libheif's JPEG writer gives up on a broken EXIF block, while its PNG
+	// writer skips the block and writes the image.
+	pngData, pngErr := runHEIFDec(ctx, dir, inputPath, "png")
+	if pngErr != nil {
+		return nil, err
+	}
+	img, imgErr := png.Decode(bytes.NewReader(pngData))
+	if imgErr != nil {
+		return nil, fmt.Errorf("failed to decode converted PNG: %w", imgErr)
+	}
+	var buf bytes.Buffer
+	if err = jpeg.Encode(&buf, img, &jpeg.Options{Quality: 90}); err != nil {
+		return nil, fmt.Errorf("failed to encode converted image: %w", err)
+	}
+	return buf.Bytes(), nil
+}
+
+func runHEIFDec(ctx context.Context, dir, inputPath, extension string) ([]byte, error) {
+	var stderr bytes.Buffer
+	cmd := exec.CommandContext(ctx, "heif-dec", "--quiet", inputPath, filepath.Join(dir, "output."+extension))
+	cmd.Stderr = &stderr
+	if err := cmd.Run(); err != nil {
 		if ctxErr := ctx.Err(); ctxErr != nil {
-			return nil, fmt.Errorf("ffmpeg failed: %w", ctxErr)
+			return nil, fmt.Errorf("heif-dec failed: %w", ctxErr)
 		}
-		return nil, fmt.Errorf("ffmpeg failed: %w", err)
+		return nil, fmt.Errorf("heif-dec failed: %w: %s", err, strings.TrimSpace(stderr.String()))
 	}
 
-	jpegData := output.Bytes()
-	config, format, err := image.DecodeConfig(bytes.NewReader(jpegData))
+	// heif-dec writes output-1.jpg, output-2.jpg, ... instead of output.jpg
+	// when the file holds several top-level images. Use the first one.
+	outputs, err := filepath.Glob(filepath.Join(dir, "output*."+extension))
+	if err != nil || len(outputs) == 0 {
+		return nil, fmt.Errorf("heif-dec produced no output: %s", strings.TrimSpace(stderr.String()))
+	}
+	data, err := os.ReadFile(outputs[0])
 	if err != nil {
-		return nil, fmt.Errorf("failed to validate converted JPEG: %w", err)
+		return nil, fmt.Errorf("failed to read converted image: %w", err)
 	}
-	if format != "jpeg" || config.Width <= 0 || config.Height <= 0 {
-		return nil, fmt.Errorf("ffmpeg produced invalid JPEG output")
+	// heif-dec exits 0 even when its writer fails, leaving an empty file.
+	config, _, err := image.DecodeConfig(bytes.NewReader(data))
+	if err != nil || config.Width <= 0 || config.Height <= 0 {
+		return nil, fmt.Errorf("heif-dec produced invalid %s output: %s", extension, strings.TrimSpace(stderr.String()))
+	}
+	return data, nil
+}
+
+// stripJPEGMetadata drops every APP1 segment. heif-dec copies the HEIC's EXIF
+// (split over several APP1 segments when large) and XMP into them, including
+// GPS location and device model. Rotation is already applied to the pixels.
+func stripJPEGMetadata(data []byte) []byte {
+	if len(data) < 4 || data[0] != 0xFF || data[1] != 0xD8 {
+		return data
+	}
+	out := make([]byte, 0, len(data))
+	out = append(out, data[:2]...)
+	pos := 2
+	for pos+4 <= len(data) && data[pos] == 0xFF {
+		marker := data[pos+1]
+		if marker == 0xDA {
+			break
+		}
+		length := int(data[pos+2])<<8 | int(data[pos+3])
+		end := pos + 2 + length
+		if length < 2 || end > len(data) {
+			return data
+		}
+		if marker != 0xE1 {
+			out = append(out, data[pos:end]...)
+		}
+		pos = end
 	}
-	return jpegData, nil
+	return append(out, data[pos:]...)
 }
 
 // AES-256-CTR

pkg/connector/media_test.go

@@ -0,0 +1,32 @@
+package connector
+
+import (
+	"bytes"
+	"testing"
+)
+
+func TestStripJPEGMetadata(t *testing.T) {
+	segment := func(marker byte, payload string) []byte {
+		n := len(payload) + 2
+		return append([]byte{0xFF, marker, byte(n >> 8), byte(n)}, payload...)
+	}
+	soi := []byte{0xFF, 0xD8}
+	jfif := segment(0xE0, "JFIF\x00")
+	exif := segment(0xE1, "Exif\x00\x00MM\x00*")
+	exifContinuation := segment(0xE1, "GPS")
+	xmp := segment(0xE1, "http://ns.adobe.com/xap/1.0/\x00exif:GPSLatitude")
+	icc := segment(0xE2, "ICC_PROFILE\x00")
+	scan := []byte{0xFF, 0xDA, 0x00, 0x02, 0x12, 0x34, 0xFF, 0xD9}
+
+	join := func(parts ...[]byte) []byte { return bytes.Join(parts, nil) }
+	input := join(soi, jfif, exif, exifContinuation, icc, xmp, scan)
+	want := join(soi, jfif, icc, scan)
+	if got := stripJPEGMetadata(input); !bytes.Equal(got, want) {
+		t.Fatalf("stripJPEGMetadata() = %x, want %x", got, want)
+	}
+
+	truncated := join(soi, segment(0xE1, "Exif\x00\x00")[:5])
+	if got := stripJPEGMetadata(truncated); !bytes.Equal(got, truncated) {
+		t.Fatalf("malformed input should be returned unchanged")
+	}
+}