From 4c2b20ae3d756397737cb24003751b2d493b5f33 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 17 Sep 2026 15:36:36 +0000 Subject: [PATCH] fix(lens): survive a missing ML Kit face-detection native library MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A device that cannot load `libface_detector_v2_jni.so` took the whole app down. `FaceTracker` built ML Kit's bundled detector in a field initializer with no guard, and ML Kit runs that model load off the caller's thread — on its own worker, inside a GMS `Task`. `Task` funnels only `Exception` into `addOnFailureListener`, so the `UnsatisfiedLinkError` escaped the worker's `Runnable` and killed the process, which is why Crashlytics reported it as fatal rather than as a failed task. Wrapping `FaceDetection.getClient` would have caught nothing. So load the library first, on our own thread, where the failure is catchable. A load that succeeds makes ML Kit's own `System.loadLibrary` a no-op; one that throws means ML Kit would have crashed, so the detector is never built: `detector` is null, `analyze` closes the proxy and publishes nothing, `close` is a no-op, and a Crashlytics non-fatal records the population. The lenses go inert and capture, trim, save and gallery are untouched — the same trade `CameraManager.bindWithLensEffect` already makes when the analysis use case cannot bind, and the one `HandTracker` makes for MediaPipe. The proxy is still closed on every frame in that state: under KEEP_ONLY_LATEST an unclosed proxy stalls the stream the hand tracker shares, so "lenses are inert" would otherwise become "the analyzer is dead". Loading by name couples one string to the dependency's `jni/` folder, so the version-catalog pin and Lesson 040 now say to re-check it on an ML Kit bump — a rename would read as "absent" and take face lenses off every device. Fixes #195 Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01FBjSmDxvBq7XtFE9PA28Ko --- .idea/dictionaries/project.xml | 1 + .../openloop/camera/lens/FaceTracker.kt | 87 +++++++++++++++---- .../diagnostics/ReverseCrashlytics.kt | 21 +++++ cspell.json | 1 + ...ease-apk-when-a-native-dependency-lands.md | 9 ++ gradle/libs.versions.toml | 3 + 6 files changed, 104 insertions(+), 18 deletions(-) diff --git a/.idea/dictionaries/project.xml b/.idea/dictionaries/project.xml index 15c83bbc..9f7e4865 100644 --- a/.idea/dictionaries/project.xml +++ b/.idea/dictionaries/project.xml @@ -236,6 +236,7 @@ lavfi lensed letterboxing + libface libimage libmediapipe libwebp diff --git a/app/src/main/java/io/github/stozo04/openloop/camera/lens/FaceTracker.kt b/app/src/main/java/io/github/stozo04/openloop/camera/lens/FaceTracker.kt index 04dc8087..7925ed42 100644 --- a/app/src/main/java/io/github/stozo04/openloop/camera/lens/FaceTracker.kt +++ b/app/src/main/java/io/github/stozo04/openloop/camera/lens/FaceTracker.kt @@ -8,8 +8,10 @@ import androidx.camera.core.ImageProxy import com.google.mlkit.vision.common.InputImage import com.google.mlkit.vision.face.Face import com.google.mlkit.vision.face.FaceDetection +import com.google.mlkit.vision.face.FaceDetector import com.google.mlkit.vision.face.FaceDetectorOptions import com.google.mlkit.vision.face.FaceLandmark +import io.github.stozo04.openloop.diagnostics.ReverseCrashlytics import kotlin.math.hypot /** @@ -47,26 +49,20 @@ class FaceTracker(private val onFaces: (List) -> Unit) : ImageAnal @Volatile private var epoch = 0 - private val detector = FaceDetection.getClient( - FaceDetectorOptions.Builder() - // FAST over ACCURATE: this runs per preview frame, and a lens that lags is worse than - // a lens that is a pixel off. - .setPerformanceMode(FaceDetectorOptions.PERFORMANCE_MODE_FAST) - // Landmarks (not contours) — the eyes, MOUTH_LEFT/RIGHT and MOUTH_BOTTOM are the whole - // input to LensAnchor, and contour mode is several times the work per frame. - .setLandmarkMode(FaceDetectorOptions.LANDMARK_MODE_ALL) - .setContourMode(FaceDetectorOptions.CONTOUR_MODE_NONE) - .setMinFaceSize(MIN_FACE_SIZE) - // Tracking ids are what let a slot follow a person across frames, and what keys every - // per-face state downstream (FaceSnapshot.trackingId). - .enableTracking() - .build(), - ) + /** + * ML Kit's bundled detector, or `null` when its native library will not load on this device — + * see [createDetector]. Null means the lenses are inert and nothing else changes. + */ + private val detector: FaceDetector? = createDetector() @SuppressLint("UnsafeOptInUsageError") override fun analyze(imageProxy: ImageProxy) { + // No detector on this device (see [createDetector]). Close the proxy anyway and publish + // nothing: under KEEP_ONLY_LATEST an unclosed proxy stalls the stream the hand tracker + // rides on too, so "lenses are inert" would become "the analyzer is dead". + val detector = detector val mediaImage = imageProxy.image - if (mediaImage == null) { + if (detector == null || mediaImage == null) { imageProxy.close() return } @@ -128,9 +124,55 @@ class FaceTracker(private val onFaces: (List) -> Unit) : ImageAnal onFaces(emptyList()) } - /** Releases the detector. Call when the analyzer is unbound. */ + /** Releases the detector. Call when the analyzer is unbound. No-op when it never came up. */ fun close() { - detector.close() + detector?.close() + } + + /** + * Builds the ML Kit detector, or returns `null` when the bundled model's native library will + * not load on this device. + * + * That the reported crash was **fatal** is the tell. ML Kit loads + * `libface_detector_v2_jni.so` from the static initializer of its own + * `ThickFaceDetectorCreator`, and it runs that off the caller's thread — on its own model-load + * worker, inside a GMS `Task`. `Task` funnels only `Exception` into `addOnFailureListener`, so + * the `UnsatisfiedLinkError` escapes the worker's `Runnable` and kills the process instead of + * surfacing as a failed task. A `try` around [FaceDetection.getClient] would have caught + * nothing (Issue #195, Crashlytics `37081b24…`, first seen 1.0.52). + * + * So the library is loaded **here** first, on the caller's own thread, where the failure is + * catchable. A load that succeeds makes ML Kit's own `System.loadLibrary` a no-op; one that + * fails means ML Kit would have crashed, so the detector is never built. The lenses then go + * inert and the camera, capture, trim and save paths are untouched — the same trade + * [io.github.stozo04.openloop.camera.CameraManager] already makes when the analysis use case + * cannot bind, and the one [HandTracker] makes for MediaPipe (Lesson 040). + */ + private fun createDetector(): FaceDetector? = try { + System.loadLibrary(NATIVE_LIBRARY) + FaceDetection.getClient( + FaceDetectorOptions.Builder() + // FAST over ACCURATE: this runs per preview frame, and a lens that lags is worse + // than a lens that is a pixel off. + .setPerformanceMode(FaceDetectorOptions.PERFORMANCE_MODE_FAST) + // Landmarks (not contours) — the eyes, MOUTH_LEFT/RIGHT and MOUTH_BOTTOM are the + // whole input to LensAnchor, and contour mode is several times the work per frame. + .setLandmarkMode(FaceDetectorOptions.LANDMARK_MODE_ALL) + .setContourMode(FaceDetectorOptions.CONTOUR_MODE_NONE) + .setMinFaceSize(MIN_FACE_SIZE) + // Tracking ids are what let a slot follow a person across frames, and what keys + // every per-face state downstream (FaceSnapshot.trackingId). + .enableTracking() + .build(), + ) + } catch (error: LinkageError) { + // The JVM's own class for "the library could not come up": UnsatisfiedLinkError when the + // .so is missing for this ABI or the install lost its native split, and + // ExceptionInInitializerError / NoClassDefFoundError when a static initializer threw. + // Not a catch-all — a bug in our own code still propagates (Lesson 013). + Log.w(TAG, "ML Kit face detection unavailable; face lenses will be inert", error) + ReverseCrashlytics.reportFaceTrackerUnavailable(error) + null } /** @@ -212,6 +254,15 @@ class FaceTracker(private val onFaces: (List) -> Unit) : ImageAnal companion object { private const val TAG = "OpenLoopFaceTracker" + /** + * The bundled model's native library, loaded by [createDetector] before ML Kit can load it + * somewhere uncatchable. **Coupled to the `mlkit-face-detection` pin in + * `gradle/libs.versions.toml`** (16.1.7): re-check the name in the AAR's `jni/` folder when + * that pin moves, because a rename would read here as "absent" and take face lenses off + * every device. The lens loop of `scripts/run-verification-loops.py` is where that shows up. + */ + private const val NATIVE_LIBRARY = "face_detector_v2_jni" + /** * How many people can wear the lens at once — `docs/PRD-multi-face-lenses.md` D1. Two is a * selfie with a kid or a friend; three is a group photo. Everything downstream is keyed by diff --git a/app/src/main/java/io/github/stozo04/openloop/diagnostics/ReverseCrashlytics.kt b/app/src/main/java/io/github/stozo04/openloop/diagnostics/ReverseCrashlytics.kt index 3dace0d3..8456d597 100644 --- a/app/src/main/java/io/github/stozo04/openloop/diagnostics/ReverseCrashlytics.kt +++ b/app/src/main/java/io/github/stozo04/openloop/diagnostics/ReverseCrashlytics.kt @@ -156,6 +156,27 @@ internal object ReverseCrashlytics { } } + /** + * ML Kit's bundled face detector could not be created (`docs/PRD-camera-lenses.md`): its + * native library would not load, so the camera keeps working and only the lenses go inert. + * + * Non-fatal on purpose — it replaces the **fatal** `UnsatisfiedLinkError` this used to be + * (Issue #195). The population that hits it still has to be visible in aggregate: a jump here + * after an ML Kit bump means the library name in `FaceTracker` moved, not that devices broke. + */ + fun reportFaceTrackerUnavailable(cause: Throwable) { + val crashlytics = crashlyticsOrNull() ?: return + val keys = CustomKeysAndValues.Builder() + .putString("face_failure_kind", cause.javaClass.simpleName.take(1024)) + .build() + runCatching { + crashlytics.log("face_tracker_unavailable: ${cause.javaClass.simpleName}") + crashlytics.recordException(cause, keys) + }.onFailure { e -> + Log.w(TAG, "Crashlytics recordException failed", e) + } + } + /** * The MediaPipe hand landmarker could not be created (`docs/PRD-lens-hand-flick.md`): the lens * still works and only the hand verb is off, so this is non-fatal — but the population that diff --git a/cspell.json b/cspell.json index a16e0167..ddf14390 100644 --- a/cspell.json +++ b/cspell.json @@ -259,6 +259,7 @@ "lavfi", "lensed", "letterboxing", + "libface", "libimage", "libmediapipe", "libwebp", diff --git a/docs/lessons_learned/040-run-the-release-apk-when-a-native-dependency-lands.md b/docs/lessons_learned/040-run-the-release-apk-when-a-native-dependency-lands.md index ae1ababa..50fbd2d9 100644 --- a/docs/lessons_learned/040-run-the-release-apk-when-a-native-dependency-lands.md +++ b/docs/lessons_learned/040-run-the-release-apk-when-a-native-dependency-lands.md @@ -48,6 +48,15 @@ without an explicit `-dontwarn`. and `LinkageError` (`ExceptionInInitializerError`, `UnsatisfiedLinkError`, `NoClassDefFoundError` — the JVM's own class of "the library could not come up"), turns the verb off, and reports a Crashlytics non-fatal. Not a catch-all: a bug in our code still propagates (Lesson 013). +- **First check the failure is even reachable from a `try`.** MediaPipe loads its library inside + `HandLandmarker.createFromOptions`, on the thread that called it, so wrapping the call is enough. + ML Kit does not: `FaceDetection.getClient` hands the load to ML Kit's **own** worker thread + inside a GMS `Task`, and `Task` funnels only `Exception` into `addOnFailureListener` — an `Error` + escapes the worker's `Runnable` and kills the process. Wrapping `getClient` there catches + nothing. Where the load is out of reach, **load the library yourself first**, on your own thread, + and skip building the detector when that throws (`FaceTracker.createDetector`, Issue #195). That + couples one string to the dependency's `jni/` folder; say so at the version-catalog pin, because + a rename reads as "absent" and silently takes the feature off every device. ## Detection checklist diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 6c6e535f..2e9b3ccc 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -103,6 +103,9 @@ play-review-ktx = { group = "com.google.android.play", name = "review-ktx", vers # ML Kit face detection — powers camera lenses (docs/PRD-camera-lenses.md). Deliberately the # STABLE face-detection API, not the beta face-mesh one; landmark mode already gives the eyes and # mouth corners LensAnchor needs. Bundled model: no Play-services round trip on first use. +# Bumping this pin: re-check that the AAR still ships `jni/*/libface_detector_v2_jni.so` under that +# exact name. FaceTracker loads it by name before ML Kit can (Issue #195, Lesson 040), and a rename +# would read as "absent" and take face lenses off every device. mlkit-face-detection = { group = "com.google.mlkit", name = "face-detection", version.ref = "mlkitFaceDetection" } # MediaPipe Hand Landmarker — the hand that flicks a lens (docs/PRD-lens-hand-flick.md). Hands only;