diff --git a/.idea/dictionaries/project.xml b/.idea/dictionaries/project.xml index 15c83bbc..9f7e4865 100644 --- a/.idea/dictionaries/project.xml +++ b/.idea/dictionaries/project.xml @@ -236,6 +236,7 @@ lavfi lensed letterboxing + libface libimage libmediapipe libwebp diff --git a/app/src/main/java/io/github/stozo04/openloop/camera/lens/FaceTracker.kt b/app/src/main/java/io/github/stozo04/openloop/camera/lens/FaceTracker.kt index 04dc8087..7925ed42 100644 --- a/app/src/main/java/io/github/stozo04/openloop/camera/lens/FaceTracker.kt +++ b/app/src/main/java/io/github/stozo04/openloop/camera/lens/FaceTracker.kt @@ -8,8 +8,10 @@ import androidx.camera.core.ImageProxy import com.google.mlkit.vision.common.InputImage import com.google.mlkit.vision.face.Face import com.google.mlkit.vision.face.FaceDetection +import com.google.mlkit.vision.face.FaceDetector import com.google.mlkit.vision.face.FaceDetectorOptions import com.google.mlkit.vision.face.FaceLandmark +import io.github.stozo04.openloop.diagnostics.ReverseCrashlytics import kotlin.math.hypot /** @@ -47,26 +49,20 @@ class FaceTracker(private val onFaces: (List) -> Unit) : ImageAnal @Volatile private var epoch = 0 - private val detector = FaceDetection.getClient( - FaceDetectorOptions.Builder() - // FAST over ACCURATE: this runs per preview frame, and a lens that lags is worse than - // a lens that is a pixel off. - .setPerformanceMode(FaceDetectorOptions.PERFORMANCE_MODE_FAST) - // Landmarks (not contours) — the eyes, MOUTH_LEFT/RIGHT and MOUTH_BOTTOM are the whole - // input to LensAnchor, and contour mode is several times the work per frame. - .setLandmarkMode(FaceDetectorOptions.LANDMARK_MODE_ALL) - .setContourMode(FaceDetectorOptions.CONTOUR_MODE_NONE) - .setMinFaceSize(MIN_FACE_SIZE) - // Tracking ids are what let a slot follow a person across frames, and what keys every - // per-face state downstream (FaceSnapshot.trackingId). - .enableTracking() - .build(), - ) + /** + * ML Kit's bundled detector, or `null` when its native library will not load on this device — + * see [createDetector]. Null means the lenses are inert and nothing else changes. + */ + private val detector: FaceDetector? = createDetector() @SuppressLint("UnsafeOptInUsageError") override fun analyze(imageProxy: ImageProxy) { + // No detector on this device (see [createDetector]). Close the proxy anyway and publish + // nothing: under KEEP_ONLY_LATEST an unclosed proxy stalls the stream the hand tracker + // rides on too, so "lenses are inert" would become "the analyzer is dead". + val detector = detector val mediaImage = imageProxy.image - if (mediaImage == null) { + if (detector == null || mediaImage == null) { imageProxy.close() return } @@ -128,9 +124,55 @@ class FaceTracker(private val onFaces: (List) -> Unit) : ImageAnal onFaces(emptyList()) } - /** Releases the detector. Call when the analyzer is unbound. */ + /** Releases the detector. Call when the analyzer is unbound. No-op when it never came up. */ fun close() { - detector.close() + detector?.close() + } + + /** + * Builds the ML Kit detector, or returns `null` when the bundled model's native library will + * not load on this device. + * + * That the reported crash was **fatal** is the tell. ML Kit loads + * `libface_detector_v2_jni.so` from the static initializer of its own + * `ThickFaceDetectorCreator`, and it runs that off the caller's thread — on its own model-load + * worker, inside a GMS `Task`. `Task` funnels only `Exception` into `addOnFailureListener`, so + * the `UnsatisfiedLinkError` escapes the worker's `Runnable` and kills the process instead of + * surfacing as a failed task. A `try` around [FaceDetection.getClient] would have caught + * nothing (Issue #195, Crashlytics `37081b24…`, first seen 1.0.52). + * + * So the library is loaded **here** first, on the caller's own thread, where the failure is + * catchable. A load that succeeds makes ML Kit's own `System.loadLibrary` a no-op; one that + * fails means ML Kit would have crashed, so the detector is never built. The lenses then go + * inert and the camera, capture, trim and save paths are untouched — the same trade + * [io.github.stozo04.openloop.camera.CameraManager] already makes when the analysis use case + * cannot bind, and the one [HandTracker] makes for MediaPipe (Lesson 040). + */ + private fun createDetector(): FaceDetector? = try { + System.loadLibrary(NATIVE_LIBRARY) + FaceDetection.getClient( + FaceDetectorOptions.Builder() + // FAST over ACCURATE: this runs per preview frame, and a lens that lags is worse + // than a lens that is a pixel off. + .setPerformanceMode(FaceDetectorOptions.PERFORMANCE_MODE_FAST) + // Landmarks (not contours) — the eyes, MOUTH_LEFT/RIGHT and MOUTH_BOTTOM are the + // whole input to LensAnchor, and contour mode is several times the work per frame. + .setLandmarkMode(FaceDetectorOptions.LANDMARK_MODE_ALL) + .setContourMode(FaceDetectorOptions.CONTOUR_MODE_NONE) + .setMinFaceSize(MIN_FACE_SIZE) + // Tracking ids are what let a slot follow a person across frames, and what keys + // every per-face state downstream (FaceSnapshot.trackingId). + .enableTracking() + .build(), + ) + } catch (error: LinkageError) { + // The JVM's own class for "the library could not come up": UnsatisfiedLinkError when the + // .so is missing for this ABI or the install lost its native split, and + // ExceptionInInitializerError / NoClassDefFoundError when a static initializer threw. + // Not a catch-all — a bug in our own code still propagates (Lesson 013). + Log.w(TAG, "ML Kit face detection unavailable; face lenses will be inert", error) + ReverseCrashlytics.reportFaceTrackerUnavailable(error) + null } /** @@ -212,6 +254,15 @@ class FaceTracker(private val onFaces: (List) -> Unit) : ImageAnal companion object { private const val TAG = "OpenLoopFaceTracker" + /** + * The bundled model's native library, loaded by [createDetector] before ML Kit can load it + * somewhere uncatchable. **Coupled to the `mlkit-face-detection` pin in + * `gradle/libs.versions.toml`** (16.1.7): re-check the name in the AAR's `jni/` folder when + * that pin moves, because a rename would read here as "absent" and take face lenses off + * every device. The lens loop of `scripts/run-verification-loops.py` is where that shows up. + */ + private const val NATIVE_LIBRARY = "face_detector_v2_jni" + /** * How many people can wear the lens at once — `docs/PRD-multi-face-lenses.md` D1. Two is a * selfie with a kid or a friend; three is a group photo. Everything downstream is keyed by diff --git a/app/src/main/java/io/github/stozo04/openloop/diagnostics/ReverseCrashlytics.kt b/app/src/main/java/io/github/stozo04/openloop/diagnostics/ReverseCrashlytics.kt index 3dace0d3..8456d597 100644 --- a/app/src/main/java/io/github/stozo04/openloop/diagnostics/ReverseCrashlytics.kt +++ b/app/src/main/java/io/github/stozo04/openloop/diagnostics/ReverseCrashlytics.kt @@ -156,6 +156,27 @@ internal object ReverseCrashlytics { } } + /** + * ML Kit's bundled face detector could not be created (`docs/PRD-camera-lenses.md`): its + * native library would not load, so the camera keeps working and only the lenses go inert. + * + * Non-fatal on purpose — it replaces the **fatal** `UnsatisfiedLinkError` this used to be + * (Issue #195). The population that hits it still has to be visible in aggregate: a jump here + * after an ML Kit bump means the library name in `FaceTracker` moved, not that devices broke. + */ + fun reportFaceTrackerUnavailable(cause: Throwable) { + val crashlytics = crashlyticsOrNull() ?: return + val keys = CustomKeysAndValues.Builder() + .putString("face_failure_kind", cause.javaClass.simpleName.take(1024)) + .build() + runCatching { + crashlytics.log("face_tracker_unavailable: ${cause.javaClass.simpleName}") + crashlytics.recordException(cause, keys) + }.onFailure { e -> + Log.w(TAG, "Crashlytics recordException failed", e) + } + } + /** * The MediaPipe hand landmarker could not be created (`docs/PRD-lens-hand-flick.md`): the lens * still works and only the hand verb is off, so this is non-fatal — but the population that diff --git a/cspell.json b/cspell.json index a16e0167..ddf14390 100644 --- a/cspell.json +++ b/cspell.json @@ -259,6 +259,7 @@ "lavfi", "lensed", "letterboxing", + "libface", "libimage", "libmediapipe", "libwebp", diff --git a/docs/lessons_learned/040-run-the-release-apk-when-a-native-dependency-lands.md b/docs/lessons_learned/040-run-the-release-apk-when-a-native-dependency-lands.md index ae1ababa..50fbd2d9 100644 --- a/docs/lessons_learned/040-run-the-release-apk-when-a-native-dependency-lands.md +++ b/docs/lessons_learned/040-run-the-release-apk-when-a-native-dependency-lands.md @@ -48,6 +48,15 @@ without an explicit `-dontwarn`. and `LinkageError` (`ExceptionInInitializerError`, `UnsatisfiedLinkError`, `NoClassDefFoundError` — the JVM's own class of "the library could not come up"), turns the verb off, and reports a Crashlytics non-fatal. Not a catch-all: a bug in our code still propagates (Lesson 013). +- **First check the failure is even reachable from a `try`.** MediaPipe loads its library inside + `HandLandmarker.createFromOptions`, on the thread that called it, so wrapping the call is enough. + ML Kit does not: `FaceDetection.getClient` hands the load to ML Kit's **own** worker thread + inside a GMS `Task`, and `Task` funnels only `Exception` into `addOnFailureListener` — an `Error` + escapes the worker's `Runnable` and kills the process. Wrapping `getClient` there catches + nothing. Where the load is out of reach, **load the library yourself first**, on your own thread, + and skip building the detector when that throws (`FaceTracker.createDetector`, Issue #195). That + couples one string to the dependency's `jni/` folder; say so at the version-catalog pin, because + a rename reads as "absent" and silently takes the feature off every device. ## Detection checklist diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 6c6e535f..2e9b3ccc 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -103,6 +103,9 @@ play-review-ktx = { group = "com.google.android.play", name = "review-ktx", vers # ML Kit face detection — powers camera lenses (docs/PRD-camera-lenses.md). Deliberately the # STABLE face-detection API, not the beta face-mesh one; landmark mode already gives the eyes and # mouth corners LensAnchor needs. Bundled model: no Play-services round trip on first use. +# Bumping this pin: re-check that the AAR still ships `jni/*/libface_detector_v2_jni.so` under that +# exact name. FaceTracker loads it by name before ML Kit can (Issue #195, Lesson 040), and a rename +# would read as "absent" and take face lenses off every device. mlkit-face-detection = { group = "com.google.mlkit", name = "face-detection", version.ref = "mlkitFaceDetection" } # MediaPipe Hand Landmarker — the hand that flicks a lens (docs/PRD-lens-hand-flick.md). Hands only;