[MS-1540] Enable age and gender estimation in face capture processing

face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackViewModel.kt

@@ -299,6 +299,14 @@ internal class LiveFeedbackViewModel @Inject constructor(
     }
 
     private suspend fun sendEventsAndFinish(attemptNumber: Int) {
+        // Freeze frame processing so `process()` can't concurrently mutate `userCaptures`/
+        // `fallbackCapture` while we enrich captures and write events below.
+        emit(phase = LiveFeedbackState.Phase.VALIDATING)
+
+        // Age/gender estimation is extra native processing, so it's only run here on the final,
+        // accepted set of captures.
+        enrichCapturesWithAgeAndGender()
+
         sortedQualifyingCaptures = userCaptures
             .filter { isAutoCapture || it.hasValidStatus() } // Auto-capture images are pre-qualified
             .sortedByDescending { it.face?.quality }
@@ -436,6 +444,19 @@ internal class LiveFeedbackViewModel @Inject constructor(
             .awaitAll()
     }
 
+    private suspend fun enrichCapturesWithAgeAndGender() = withContext(bgDispatcher) {
+        userCaptures.forEachIndexed { index, faceDetection ->
+            userCaptures[index] = enrichWithAgeAndGender(faceDetection) ?: faceDetection
+        }
+        fallbackCapture = enrichWithAgeAndGender(fallbackCapture)
+    }
+
+    private fun enrichWithAgeAndGender(faceDetection: FaceDetection?): FaceDetection? {
+        val face = faceDetection?.face ?: return faceDetection
+        val ageAndGender = faceDetector.analyze(faceDetection.bitmap, estimateAgeAndGender = true) ?: return faceDetection
+        return faceDetection.copy(face = face.copy(age = ageAndGender.age, gender = ageAndGender.gender))
+    }
+
     private suspend fun sendCaptureEvent(
         faceDetection: FaceDetection?,
         attemptNumber: Int,

face/capture/src/main/java/com/simprints/face/capture/usecases/SimpleCaptureEventReporter.kt

@@ -117,6 +117,13 @@ internal class SimpleCaptureEventReporter @Inject constructor(
             format = it.format,
             spoofScore = faceDetection.spoofCheckResult?.score,
             spoofSkipReason = mapSpoofReason(faceDetection.spoofCheckResult?.skipReason),
+            age = it.age,
+            gender = it.gender?.let { gender ->
+                FaceCapturePayload.Gender(
+                    male = gender.maleProbability,
+                    female = gender.femaleProbability,
+                )
+            },
         )
     }
 

face/capture/src/test/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackViewModelTest.kt

@@ -409,6 +409,7 @@ internal class LiveFeedbackViewModelTest {
     fun `event saving - captured samples are stored as non-fallback with one event per sample plus fallback`() = runTest {
         val validFace = getFace()
         every { faceDetector.analyze(frame) } returns validFace
+        every { faceDetector.analyze(any(), estimateAgeAndGender = true) } returns null
 
         viewModel.initAutoCapture()
         viewModel.initCapture(ModalitySdkType.SIM_FACE, 2, 0)
@@ -429,6 +430,58 @@ internal class LiveFeedbackViewModelTest {
         coVerify(exactly = 1) { eventReporter.addFallbackCaptureEvent(any(), any()) }
     }
 
+    @Test
+    fun `event saving - enriches only the final accepted captures with age and gender`() = runTest {
+        val validFace = getFace()
+        val enrichedFace = getFace().copy(age = 34f, gender = Face.Gender(0.2f, 0.8f))
+        every { faceDetector.analyze(frame) } returns validFace
+        every { faceDetector.analyze(any(), estimateAgeAndGender = true) } returns enrichedFace
+
+        viewModel.initAutoCapture()
+        viewModel.initCapture(ModalitySdkType.RANK_ONE, 1, 0)
+        viewModel.process(frame, frame) // fallback frame before start
+        viewModel.startCapture()
+        viewModel.process(frame, frame) // captured sample -> finishes
+
+        with(viewModel.sortedQualifyingCaptures) {
+            assertThat(this).hasSize(1)
+            assertThat(first().face?.age).isEqualTo(34f)
+            assertThat(first().face?.gender).isEqualTo(Face.Gender(0.2f, 0.8f))
+        }
+        // Once for the captured sample and once for the fallback capture.
+        verify(exactly = 2) { faceDetector.analyze(any(), estimateAgeAndGender = true) }
+    }
+
+    @Test
+    fun `event saving - age and gender estimation is not repeated on every failed spoof-check retry`() = runTest {
+        every { getSpoofCheckConfiguration.invoke(any(), any()) } returns spoofConfig(FaceConfiguration.SpoofCheckMode.ENFORCED)
+        every { faceDetector.analyze(frame) } returns getFace()
+        every { faceDetector.analyze(any(), estimateAgeAndGender = true) } returns getFace()
+        coEvery { faceDetector.spoofCheck(any(), any()) } returns SpoofCheckResult(score = 0.9f) // always fails
+
+        viewModel.initAutoCapture()
+        viewModel.initCapture(ModalitySdkType.SIM_FACE, 1, 0)
+
+        // Attempt 1 fails and gets discarded.
+        viewModel.process(frame, frame)
+        viewModel.startCapture()
+        viewModel.process(frame, frame)
+        advanceUntilIdle()
+        assertThat(viewModel.state.value.phase).isEqualTo(LiveFeedbackState.Phase.NOT_STARTED)
+        verify(exactly = 0) { faceDetector.analyze(any(), estimateAgeAndGender = true) }
+
+        // Attempt 2 reaches maxAttempts and finishes despite still failing spoof check.
+        viewModel.process(frame, frame)
+        viewModel.startCapture()
+        viewModel.process(frame, frame)
+        advanceUntilIdle()
+        assertThat(viewModel.state.value.phase).isEqualTo(LiveFeedbackState.Phase.FINISHED)
+
+        // Enrichment only runs once, for the final (accepted) attempt's captures + fallback -
+        // never for the discarded first attempt.
+        verify(exactly = 2) { faceDetector.analyze(any(), estimateAgeAndGender = true) }
+    }
+
     @Test
     fun `event saving - falls back to the fallback capture when no captured sample qualifies`() = runTest {
         every { faceDetector.analyze(frame) } returnsMany listOf(

face/capture/src/test/java/com/simprints/face/capture/usecases/SimpleCaptureEventReporterTest.kt

@@ -305,7 +305,10 @@ class SimpleCaptureEventReporterTest {
         Timestamp(1L),
     )
 
-    private fun getFace() = Face(
+    private fun getFace(
+        age: Float? = null,
+        gender: Face.Gender? = null,
+    ) = Face(
         100,
         100,
         Rect(0, 0, 0, 0),
@@ -314,5 +317,41 @@ class SimpleCaptureEventReporterTest {
         0f,
         byteArrayOf(),
         "",
+        age,
+        gender,
     )
+
+    @Test
+    fun `Adds capture event with age and gender when available`() = runTest {
+        val detection = getDetection(FaceDetection.Status.VALID).copy(
+            face = getFace(age = 25f, gender = Face.Gender(maleProbability = 0.7f, femaleProbability = 0.3f)),
+        )
+
+        reporter.addCaptureEvents(detection, 1, 0.5f, SpoofCheckConfiguration.DISABLED)
+
+        coVerify {
+            eventRepository.addOrUpdateEvent(
+                match<FaceCaptureEvent> {
+                    it.payload.face?.age == 25f &&
+                        it.payload.face?.gender?.male == 0.7f &&
+                        it.payload.face?.gender?.female == 0.3f
+                },
+            )
+        }
+    }
+
+    @Test
+    fun `Adds capture event with null age and gender when not available`() = runTest {
+        val detection = getDetection(FaceDetection.Status.VALID).copy(face = getFace(age = null, gender = null))
+
+        reporter.addCaptureEvents(detection, 1, 0.5f, SpoofCheckConfiguration.DISABLED)
+
+        coVerify {
+            eventRepository.addOrUpdateEvent(
+                match<FaceCaptureEvent> {
+                    it.payload.face?.age == null && it.payload.face?.gender == null
+                },
+            )
+        }
+    }
 }

face/infra/base-bio-sdk/src/main/java/com/simprints/face/infra/basebiosdk/detection/Face.kt

@@ -14,6 +14,8 @@ import android.graphics.RectF
  * @property quality image quality
  * @property template
  * @property format
+ * @property age estimated age of the person, if available from the template extraction
+ * @property gender estimated gender probabilities of the person, if available from the template extraction
  *
  */
 data class Face(
@@ -25,6 +27,8 @@ data class Face(
     val quality: Float,
     val template: ByteArray,
     val format: String,
+    val age: Float? = null,
+    val gender: Gender? = null,
 ) {
     // Relative = coordinates are fractions of the source image dimensions
     val relativeBoundingBox
@@ -34,4 +38,9 @@ data class Face(
             absoluteBoundingBox.right.toFloat() / sourceWidth,
             absoluteBoundingBox.bottom.toFloat() / sourceHeight,
         )
+
+    data class Gender(
+        val maleProbability: Float,
+        val femaleProbability: Float,
+    )
 }

face/infra/base-bio-sdk/src/main/java/com/simprints/face/infra/basebiosdk/detection/FaceDetector.kt

@@ -1,15 +1,23 @@
 package com.simprints.face.infra.basebiosdk.detection
 
 import android.graphics.Bitmap
+import com.simprints.core.ExcludedFromGeneratedTestCoverageReports
 
+@ExcludedFromGeneratedTestCoverageReports("No need to test the interface")
 interface FaceDetector {
     /**
      * Analyze an ARGB_8888 bitmap and return the detected face data
      *
      * @param bitmap ARGB_8888 formatted
+     * @param estimateAgeAndGender whether to also request age/gender estimation. This is extra native
+     * processing on top of face detection/template extraction, so it should only be requested for
+     * the final selected capture, not on every live-preview frame.
      * @return Face object or null if no face is detected
      */
-    fun analyze(bitmap: Bitmap): Face?
+    fun analyze(
+        bitmap: Bitmap,
+        estimateAgeAndGender: Boolean = false,
+    ): Face?
 
     /**
      * Perform a spoof check on an ARGB_8888 bitmap

face/infra/base-bio-sdk/src/test/java/com/simprints/infra/facebiosdk/detection/FaceTest.kt

@@ -1,8 +1,8 @@
 package com.simprints.infra.facebiosdk.detection
 
 import android.graphics.Rect
-import androidx.test.ext.junit.runners.AndroidJUnit4
-import com.google.common.truth.Truth.assertThat
+import androidx.test.ext.junit.runners.*
+import com.google.common.truth.Truth.*
 import com.simprints.face.infra.basebiosdk.detection.Face
 import org.junit.Test
 import org.junit.runner.RunWith
@@ -21,6 +21,8 @@ class FaceTest {
             template = byteArrayOf(0),
             format = "format",
             absoluteBoundingBox = Rect(0, 0, 50, 100),
+            age = 0f,
+            gender = Face.Gender(0.5f, 0.5f),
         )
         // when
         val relativeBoundingBox = face.relativeBoundingBox

face/infra/roc-v1/src/main/java/com/simprints/face/infra/rocv1/detection/RocV1Detector.kt

@@ -10,6 +10,7 @@ import io.rankone.rocsdk.embedded.SWIGTYPE_p_float
 import io.rankone.rocsdk.embedded.SWIGTYPE_p_unsigned_char
 import io.rankone.rocsdk.embedded.roc
 import io.rankone.rocsdk.embedded.roc_detection
+import io.rankone.rocsdk.embedded.roc_embedded_gender
 import io.rankone.rocsdk.embedded.roc_embedded_landmark
 import io.rankone.rocsdk.embedded.roc_image
 import java.nio.ByteBuffer
@@ -38,12 +39,16 @@ class RocV1Detector @Inject constructor() : FaceDetector {
         var template: SWIGTYPE_p_unsigned_char,
         var yaw: SWIGTYPE_p_float,
         var quality: SWIGTYPE_p_float,
+        var age: SWIGTYPE_p_float?,
+        var gender: roc_embedded_gender?,
     ) {
         fun cleanup() {
             face.delete()
             roc.delete_uint8_t_array(template)
             roc.delete_float(yaw)
             roc.delete_float(quality)
+            age?.let { roc.delete_float(it) }
+            gender?.delete()
         }
     }
 
@@ -52,7 +57,10 @@ class RocV1Detector @Inject constructor() : FaceDetector {
         configuredMaxSize: Int,
     ) = SpoofCheckResult(0f, SpoofCheckResult.SkipReason.NOT_AVAILABLE)
 
-    override fun analyze(bitmap: Bitmap): Face? {
+    override fun analyze(
+        bitmap: Bitmap,
+        estimateAgeAndGender: Boolean,
+    ): Face? {
         val rocColorImage = roc_image()
         val rocGrayImage = roc_image()
 
@@ -70,7 +78,7 @@ class RocV1Detector @Inject constructor() : FaceDetector {
 
         roc.roc_free_image(rocColorImage)
 
-        return analyze(rocGrayImage, bitmap.width, bitmap.height)
+        return analyze(rocGrayImage, bitmap.width, bitmap.height, estimateAgeAndGender)
     }
 
     /**
@@ -80,12 +88,15 @@ class RocV1Detector @Inject constructor() : FaceDetector {
         rocImage: roc_image,
         imageWidth: Int,
         imageHeight: Int,
+        estimateAgeAndGender: Boolean,
     ): Face? {
         val rocFace = ROCFace(
             roc_detection(),
             roc.new_uint8_t_array(roc.ROC_FAST_FV_SIZE.toInt()),
             roc.new_float(),
             roc.new_float(),
+            if (estimateAgeAndGender) roc.new_float() else null,
+            if (estimateAgeAndGender) roc_embedded_gender() else null,
         )
 
         val faceDetected = getRocTemplateFromImage(rocImage, rocFace)
@@ -100,6 +111,8 @@ class RocV1Detector @Inject constructor() : FaceDetector {
 
         val qualityValue = roc.float_value(rocFace.quality)
 
+        val ageValue = rocFace.age?.let { roc.float_value(it) }
+
         val face = Face(
             imageWidth,
             imageHeight,
@@ -114,6 +127,13 @@ class RocV1Detector @Inject constructor() : FaceDetector {
             qualityValue,
             roc.cdata(roc.roc_cast(rocFace.template), roc.ROC_FAST_FV_SIZE.toInt()),
             RANK_ONE_TEMPLATE_FORMAT_1_23,
+            age = ageValue,
+            gender = rocFace.gender?.let {
+                Face.Gender(
+                    maleProbability = it.male,
+                    femaleProbability = it.female,
+                )
+            },
         )
 
         // Free all resources after getting the face
@@ -188,9 +208,9 @@ class RocV1Detector @Inject constructor() : FaceDetector {
                     chin,
                     rocFace.template,
                     rocFace.quality,
+                    rocFace.age,
                     null,
-                    null,
-                    null,
+                    rocFace.gender,
                     null,
                     null,
                     null,