diff --git a/Dockerfile b/Dockerfile index a0847df9..5fbc7201 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,7 +26,7 @@ FROM ${DOCKER_HUB}/alpine:3.23 ENV UID=1337 \ GID=1337 -RUN apk add --no-cache ffmpeg su-exec ca-certificates bash jq curl yq-go olm +RUN apk add --no-cache ffmpeg libheif-tools su-exec ca-certificates bash jq curl yq-go olm COPY --from=builder /build/matrix-line /usr/bin/matrix-line COPY ./docker-run.sh /docker-run.sh diff --git a/pkg/connector/media.go b/pkg/connector/media.go index a65c3add..482236d0 100644 --- a/pkg/connector/media.go +++ b/pkg/connector/media.go @@ -13,7 +13,7 @@ import ( "image" _ "image/gif" "image/jpeg" - _ "image/png" + "image/png" "io" "os" "os/exec" @@ -46,46 +46,98 @@ func convertHEICToJPEG(ctx context.Context, imageData []byte) ([]byte, error) { if err := ctx.Err(); err != nil { return nil, err } - input, err := os.CreateTemp("", "line-image-*.heic") + dir, err := os.MkdirTemp("", "line-heic-*") if err != nil { - return nil, fmt.Errorf("failed to create temporary HEIC file: %w", err) + return nil, fmt.Errorf("failed to create temporary HEIC directory: %w", err) } - inputPath := input.Name() - defer os.Remove(inputPath) + defer os.RemoveAll(dir) - if _, err = input.Write(imageData); err != nil { - input.Close() + inputPath := filepath.Join(dir, "input.heic") + if err = os.WriteFile(inputPath, imageData, 0o600); err != nil { return nil, fmt.Errorf("failed to write temporary HEIC file: %w", err) } - if err = input.Close(); err != nil { - return nil, fmt.Errorf("failed to close temporary HEIC file: %w", err) + + // iPhone photos are HEIF grids of tiles. The ffmpeg CLI only decodes the + // first tile, while libheif assembles the full image. + jpegData, err := runHEIFDec(ctx, dir, inputPath, "jpg") + if err == nil { + return stripJPEGMetadata(jpegData), nil + } else if ctx.Err() != nil { + return nil, err } - var output bytes.Buffer - cmd := exec.CommandContext(ctx, "ffmpeg", - "-hide_banner", "-loglevel", "error", - "-i", inputPath, - "-frames:v", "1", "-q:v", "2", - "-f", "image2pipe", "-vcodec", "mjpeg", "pipe:1", - ) - cmd.Stdout = &output - err = cmd.Run() - if err != nil { + // libheif's JPEG writer gives up on a broken EXIF block, while its PNG + // writer skips the block and writes the image. + pngData, pngErr := runHEIFDec(ctx, dir, inputPath, "png") + if pngErr != nil { + return nil, err + } + img, imgErr := png.Decode(bytes.NewReader(pngData)) + if imgErr != nil { + return nil, fmt.Errorf("failed to decode converted PNG: %w", imgErr) + } + var buf bytes.Buffer + if err = jpeg.Encode(&buf, img, &jpeg.Options{Quality: 90}); err != nil { + return nil, fmt.Errorf("failed to encode converted image: %w", err) + } + return buf.Bytes(), nil +} + +func runHEIFDec(ctx context.Context, dir, inputPath, extension string) ([]byte, error) { + var stderr bytes.Buffer + cmd := exec.CommandContext(ctx, "heif-dec", "--quiet", inputPath, filepath.Join(dir, "output."+extension)) + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { if ctxErr := ctx.Err(); ctxErr != nil { - return nil, fmt.Errorf("ffmpeg failed: %w", ctxErr) + return nil, fmt.Errorf("heif-dec failed: %w", ctxErr) } - return nil, fmt.Errorf("ffmpeg failed: %w", err) + return nil, fmt.Errorf("heif-dec failed: %w: %s", err, strings.TrimSpace(stderr.String())) } - jpegData := output.Bytes() - config, format, err := image.DecodeConfig(bytes.NewReader(jpegData)) + // heif-dec writes output-1.jpg, output-2.jpg, ... instead of output.jpg + // when the file holds several top-level images. Use the first one. + outputs, err := filepath.Glob(filepath.Join(dir, "output*."+extension)) + if err != nil || len(outputs) == 0 { + return nil, fmt.Errorf("heif-dec produced no output: %s", strings.TrimSpace(stderr.String())) + } + data, err := os.ReadFile(outputs[0]) if err != nil { - return nil, fmt.Errorf("failed to validate converted JPEG: %w", err) + return nil, fmt.Errorf("failed to read converted image: %w", err) } - if format != "jpeg" || config.Width <= 0 || config.Height <= 0 { - return nil, fmt.Errorf("ffmpeg produced invalid JPEG output") + // heif-dec exits 0 even when its writer fails, leaving an empty file. + config, _, err := image.DecodeConfig(bytes.NewReader(data)) + if err != nil || config.Width <= 0 || config.Height <= 0 { + return nil, fmt.Errorf("heif-dec produced invalid %s output: %s", extension, strings.TrimSpace(stderr.String())) + } + return data, nil +} + +// stripJPEGMetadata drops every APP1 segment. heif-dec copies the HEIC's EXIF +// (split over several APP1 segments when large) and XMP into them, including +// GPS location and device model. Rotation is already applied to the pixels. +func stripJPEGMetadata(data []byte) []byte { + if len(data) < 4 || data[0] != 0xFF || data[1] != 0xD8 { + return data + } + out := make([]byte, 0, len(data)) + out = append(out, data[:2]...) + pos := 2 + for pos+4 <= len(data) && data[pos] == 0xFF { + marker := data[pos+1] + if marker == 0xDA { + break + } + length := int(data[pos+2])<<8 | int(data[pos+3]) + end := pos + 2 + length + if length < 2 || end > len(data) { + return data + } + if marker != 0xE1 { + out = append(out, data[pos:end]...) + } + pos = end } - return jpegData, nil + return append(out, data[pos:]...) } // AES-256-CTR diff --git a/pkg/connector/media_test.go b/pkg/connector/media_test.go new file mode 100644 index 00000000..3f2b8c5f --- /dev/null +++ b/pkg/connector/media_test.go @@ -0,0 +1,32 @@ +package connector + +import ( + "bytes" + "testing" +) + +func TestStripJPEGMetadata(t *testing.T) { + segment := func(marker byte, payload string) []byte { + n := len(payload) + 2 + return append([]byte{0xFF, marker, byte(n >> 8), byte(n)}, payload...) + } + soi := []byte{0xFF, 0xD8} + jfif := segment(0xE0, "JFIF\x00") + exif := segment(0xE1, "Exif\x00\x00MM\x00*") + exifContinuation := segment(0xE1, "GPS") + xmp := segment(0xE1, "http://ns.adobe.com/xap/1.0/\x00exif:GPSLatitude") + icc := segment(0xE2, "ICC_PROFILE\x00") + scan := []byte{0xFF, 0xDA, 0x00, 0x02, 0x12, 0x34, 0xFF, 0xD9} + + join := func(parts ...[]byte) []byte { return bytes.Join(parts, nil) } + input := join(soi, jfif, exif, exifContinuation, icc, xmp, scan) + want := join(soi, jfif, icc, scan) + if got := stripJPEGMetadata(input); !bytes.Equal(got, want) { + t.Fatalf("stripJPEGMetadata() = %x, want %x", got, want) + } + + truncated := join(soi, segment(0xE1, "Exif\x00\x00")[:5]) + if got := stripJPEGMetadata(truncated); !bytes.Equal(got, truncated) { + t.Fatalf("malformed input should be returned unchanged") + } +}