Android 0.6.8: playback sharing, live drawing and encrypted images

app/build.gradle.kts

@@ -37,8 +37,8 @@ android {
         minSdk = 33
         targetSdk = 35
         // Remain newer than the 0.6.3 rendezvous candidate (code 26).
-        versionCode = 30
-        versionName = "0.6.7"
+        versionCode = 31
+        versionName = "0.6.8"
 
         testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"
 
@@ -80,6 +80,7 @@ android {
     }
 
     sourceSets.getByName("androidTest").assets.srcDir("../protocol/src/test/resources")
+    sourceSets.getByName("androidTest").assets.srcDir("src/test/resources/attachments")
     sourceSets.getByName("androidTest").assets.srcDir(layout.buildDirectory.dir("browser-call-interop"))
     // The reviewed Link bundle is unpacked into build/link-bridge only after
     // its archive and per-file manifest have been verified. Never copy these

app/src/androidTest/AndroidManifest.xml

@@ -5,5 +5,7 @@
       the real NIP-55 intent/result boundary without introducing a signer into
       the production application.
     -->
-    <application />
+    <application android:allowAudioPlaybackCapture="true">
+        <activity android:name="dev.forgesworn.kithmoot.media.PlaybackToneActivity" android:exported="true" />
+    </application>
 </manifest>

app/src/androidTest/kotlin/dev/forgesworn/kithmoot/media/PlaybackCaptureTest.kt

@@ -0,0 +1,124 @@
+package dev.forgesworn.kithmoot.media
+
+import android.app.Activity
+import android.content.ComponentName
+import android.content.Intent
+import android.media.AudioAttributes
+import android.media.AudioFormat
+import android.media.AudioTrack
+import android.media.projection.MediaProjectionManager
+import android.os.Bundle
+import android.os.SystemClock
+import android.view.accessibility.AccessibilityNodeInfo
+import android.widget.TextView
+import androidx.activity.result.contract.ActivityResultContracts
+import androidx.test.core.app.ActivityScenario
+import androidx.test.platform.app.InstrumentationRegistry
+import dev.forgesworn.kithmoot.MainActivity
+import dev.forgesworn.kithmoot.service.ScreenShareService
+import kotlinx.coroutines.*
+import kotlinx.coroutines.flow.first
+import org.junit.Assert.*
+import org.junit.Assume.assumeTrue
+import org.junit.Test
+import org.webrtc.EglBase
+import org.webrtc.PeerConnectionFactory
+import org.webrtc.audio.JavaAudioDeviceModule
+import java.nio.ByteBuffer
+import java.nio.ByteOrder
+import java.util.concurrent.CountDownLatch
+import java.util.concurrent.TimeUnit
+import kotlin.math.sin
+
+/** Separate instrumentation-package UID: captured media, never a person's app. */
+class PlaybackToneActivity : Activity() {
+    private var tone: AudioTrack? = null
+    override fun onCreate(state: Bundle?) {
+        super.onCreate(state)
+        setContentView(TextView(this).apply { text = "Synthetic playback capture test" })
+        val samples = ShortArray(48_000) { (sin(it * 2 * Math.PI * 440 / 48_000) * 8_000).toInt().toShort() }
+        tone = AudioTrack.Builder().setAudioAttributes(AudioAttributes.Builder()
+            .setUsage(AudioAttributes.USAGE_MEDIA).setAllowedCapturePolicy(AudioAttributes.ALLOW_CAPTURE_BY_ALL).build())
+            .setAudioFormat(AudioFormat.Builder().setSampleRate(48_000).setEncoding(AudioFormat.ENCODING_PCM_16BIT)
+                .setChannelMask(AudioFormat.CHANNEL_OUT_MONO).build())
+            .setTransferMode(AudioTrack.MODE_STATIC).setBufferSizeInBytes(samples.size * 2).build().also {
+                it.write(samples, 0, samples.size); it.setLoopPoints(0, samples.size, -1); it.play()
+            }
+    }
+    override fun onNewIntent(intent: Intent) { super.onNewIntent(intent); if (intent.getBooleanExtra("stop", false)) finish() }
+    override fun onDestroy() { tone?.stop(); tone?.release(); super.onDestroy() }
+}
+
+class PlaybackCaptureTest {
+    @Test fun playback_enters_the_audio_buffer_and_stops_with_screen_capture() = runBlocking {
+        // The automatic consent click is confined to a disposable emulator.
+        assumeTrue(android.os.Build.HARDWARE in setOf("ranchu", "goldfish"))
+        val instrumentation = InstrumentationRegistry.getInstrumentation()
+        val context = instrumentation.targetContext
+        val automation = instrumentation.uiAutomation
+        automation.grantRuntimePermission(context.packageName, android.Manifest.permission.RECORD_AUDIO)
+        automation.grantRuntimePermission(context.packageName, android.Manifest.permission.POST_NOTIFICATIONS)
+        var permission: Intent? = null
+        val returned = CountDownLatch(1)
+        ActivityScenario.launch(MainActivity::class.java).use { scenario ->
+            scenario.onActivity { activity ->
+                val launcher = activity.activityResultRegistry.register("synthetic-projection", ActivityResultContracts.StartActivityForResult()) { result ->
+                    if (result.resultCode == Activity.RESULT_OK) permission = result.data
+                    returned.countDown()
+                }
+                val manager = context.getSystemService(MediaProjectionManager::class.java)
+                launcher.launch(if (android.os.Build.VERSION.SDK_INT >= 34) manager.createScreenCaptureIntent(
+                    android.media.projection.MediaProjectionConfig.createConfigForDefaultDisplay()) else manager.createScreenCaptureIntent())
+            }
+            val deadline = SystemClock.uptimeMillis() + 15_000
+            while (returned.count > 0 && SystemClock.uptimeMillis() < deadline) {
+                val root = automation.rootInActiveWindow
+                for (label in listOf("Entire screen", "Start now", "Start recording", "Share screen", "Start")) {
+                    root?.findAccessibilityNodeInfosByText(label)?.firstOrNull { it.isClickable }
+                        ?.performAction(AccessibilityNodeInfo.ACTION_CLICK)
+                }
+                SystemClock.sleep(200)
+            }
+            assertTrue("Screen capture consent did not complete", returned.await(1, TimeUnit.SECONDS))
+            assertNotNull(permission)
+            instrumentation.runOnMainSync { ScreenShareService.start(context) }
+            withTimeout(5_000) { ScreenShareService.running.first { it } }
+            PeerConnectionFactory.initialize(PeerConnectionFactory.InitializationOptions.builder(context).createInitializationOptions())
+            val playback = PlaybackAudio()
+            val callbackPeak = java.util.concurrent.atomic.AtomicInteger(0)
+            val module = JavaAudioDeviceModule.builder(context).setSampleRate(48_000)
+                .setAudioBufferCallback { buffer, format, channels, rate, bytes, time ->
+                    playback.fill(buffer, format, channels, rate, bytes)
+                    val pcm = buffer.duplicate().order(ByteOrder.LITTLE_ENDIAN)
+                    callbackPeak.set((0 until bytes / 2).maxOf { kotlin.math.abs(pcm.getShort(it * 2).toInt()) })
+                    time
+                }
+                .createAudioDeviceModule()
+            val factory = PeerConnectionFactory.builder().setAudioDeviceModule(module).createPeerConnectionFactory()
+            val egl = EglBase.create()
+            val media = LocalMedia(context, factory, egl, playback)
+            try {
+                media.startScreenShare(permission!!)
+                assertTrue(playback.active)
+                assertFalse(playback.microphone)
+                val tone = Intent().setComponent(ComponentName(instrumentation.context.packageName, PlaybackToneActivity::class.java.name))
+                    .addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)
+                context.startActivity(tone)
+                val buffer = ByteBuffer.allocateDirect(960).order(ByteOrder.LITTLE_ENDIAN)
+                module.requestStartRecording()
+                withTimeout(10_000) { while (callbackPeak.get() < 100) delay(10) }
+                assertTrue("Another app's sound must enter WebRTC's input callback", callbackPeak.get() > 100)
+                assertEquals(listOf("screen-audio", "screen"), media.tracks.value.map { it.role })
+                media.stopScreenShare()
+                assertFalse(playback.active)
+                assertTrue(media.tracks.value.isEmpty())
+                playback.fill(buffer, AudioFormat.ENCODING_PCM_16BIT, 1, 48_000, 960)
+                assertTrue((0 until 960).all { buffer.get(it) == 0.toByte() })
+            } finally {
+                module.requestStopRecording(); media.releaseAll(); factory.dispose(); module.release(); egl.release(); ScreenShareService.stop(context)
+                context.startActivity(Intent().setComponent(ComponentName(instrumentation.context.packageName, PlaybackToneActivity::class.java.name))
+                    .addFlags(Intent.FLAG_ACTIVITY_NEW_TASK or Intent.FLAG_ACTIVITY_CLEAR_TOP or Intent.FLAG_ACTIVITY_SINGLE_TOP).putExtra("stop", true))
+            }
+        }
+    }
+}

app/src/androidTest/kotlin/dev/forgesworn/kithmoot/ui/AttachmentViewerTest.kt

@@ -0,0 +1,65 @@
+package dev.forgesworn.kithmoot.ui
+
+import android.graphics.Bitmap
+import androidx.activity.compose.setContent
+import androidx.compose.foundation.layout.fillMaxSize
+import androidx.compose.runtime.*
+import androidx.compose.ui.Modifier
+import androidx.compose.ui.test.*
+import androidx.compose.ui.test.junit4.createEmptyComposeRule
+import androidx.test.core.app.ActivityScenario
+import androidx.test.platform.app.InstrumentationRegistry
+import dev.forgesworn.kithmoot.MainActivity
+import dev.forgesworn.kithmoot.session.ChatAttachment
+import dev.forgesworn.kithmoot.session.ChatMessage
+import dev.forgesworn.kithmoot.ui.room.ChatPane
+import dev.forgesworn.kithmoot.ui.theme.KithMootTheme
+import kotlinx.serialization.json.*
+import org.junit.Assert.*
+import org.junit.Assume.assumeTrue
+import org.junit.Rule
+import org.junit.Test
+import java.io.File
+
+class AttachmentViewerTest {
+    @get:Rule val ui = createEmptyComposeRule()
+    @Test fun opens_a_web_encrypted_image_fits_zooms_and_closes_on_retraction() {
+        val url = InstrumentationRegistry.getArguments().getString("attachmentFixtureUrl")
+        assumeTrue("Supply an HTTPS host serving the exact synthetic web-image.enc fixture", url?.startsWith("https://") == true)
+        val instrumentation = InstrumentationRegistry.getInstrumentation()
+        val meta = Json.parseToJsonElement(instrumentation.context.assets.open("web-image.json").bufferedReader().readText()).jsonObject
+        val attachment = ChatAttachment(url!!, meta.getValue("sha256").jsonPrimitive.content, meta.getValue("key").jsonPrimitive.content, "picture.png")
+        val message = ChatMessage("image", "01".repeat(32), "02".repeat(32), "A synthetic image", 1_800_000_000, attachments = listOf(attachment))
+        var messages by mutableStateOf(listOf(message))
+        ActivityScenario.launch(MainActivity::class.java).use { scenario ->
+            scenario.onActivity { it.setContent { KithMootTheme { ChatPane(messages, "03".repeat(32), {}, Modifier.fillMaxSize()) } } }
+            ui.onNodeWithText("Open attachment: picture.png").performClick()
+            ui.waitUntil(35_000) { ui.onNodeWithContentDescription("picture.png").isDisplayed() }
+            ui.onNodeWithText("Actual size").performClick()
+            ui.onNodeWithText("Fit").performClick()
+            ui.onNodeWithContentDescription("picture.png").performTouchInput {
+                pinch(center - androidx.compose.ui.geometry.Offset(30f, 0f), center + androidx.compose.ui.geometry.Offset(30f, 0f),
+                    center - androidx.compose.ui.geometry.Offset(100f, 0f), center + androidx.compose.ui.geometry.Offset(100f, 0f))
+            }
+            ui.onNodeWithText("Fit").performClick()
+            val bounds = ui.onNodeWithContentDescription("picture.png").fetchSemanticsNode().boundsInWindow
+            ui.waitUntil(5_000) {
+                val frame = instrumentation.uiAutomation.takeScreenshot()
+                val margin = frame.getPixel(bounds.center.x.toInt(), (bounds.top + 5).toInt())
+                frame.recycle()
+                android.graphics.Color.red(margin) > 240 && android.graphics.Color.blue(margin) > 240
+            }
+            val bitmap = instrumentation.uiAutomation.takeScreenshot()
+            val left = bitmap.getPixel((bounds.left + bounds.width / 3).toInt(), bounds.center.y.toInt())
+            val right = bitmap.getPixel((bounds.left + bounds.width * 2 / 3).toInt(), bounds.center.y.toInt())
+            assertTrue(android.graphics.Color.red(left) > 180)
+            assertTrue(android.graphics.Color.blue(right) > 180)
+            File(instrumentation.targetContext.getExternalFilesDir(null), "attachment-viewer.png").outputStream().use {
+                bitmap.compress(Bitmap.CompressFormat.PNG, 100, it)
+            }
+            ui.runOnIdle { messages = messages + message.copy(id = "retraction", body = "Retracted", sentAt = message.sentAt + 1, attachments = emptyList(), retracts = message.id) }
+            ui.waitUntil(5_000) { !ui.onNodeWithText("Close image").isDisplayed() }
+            ui.onNodeWithText("Message retracted").assertIsDisplayed()
+        }
+    }
+}

app/src/androidTest/kotlin/dev/forgesworn/kithmoot/ui/ChatAndShareUiTest.kt

@@ -51,6 +51,7 @@ class ChatAndShareUiTest {
             for (i in 0 until buffer.dataV.capacity()) buffer.dataV.put(i, 150.toByte())
             val frame = VideoFrame(buffer, 0, System.nanoTime()); source.capturerObserver.onFrameCaptured(frame); frame.release(); frameNumber++
         }, 0, 100, TimeUnit.MILLISECONDS)
+        val strokes = mutableListOf<dev.forgesworn.kithmoot.protocol.ScreenAnnotation>()
         var phase by mutableIntStateOf(0)
         var live by mutableStateOf<VideoTrack?>(track)
         var pip by mutableStateOf(false)
@@ -70,7 +71,7 @@ class ChatAndShareUiTest {
                                     onReact = { target, emoji -> messages = messages + first.copy(id = "reaction-${messages.size}", participant = self, reaction = toggleReaction(messages, target, self, emoji)) })
                                 1 -> ScreenShareViewer(live, egl, "Synthetic workshop presentation", pip,
                                     onPopOut = { assertTrue(activity.enterPictureInPictureMode(PictureInPictureParams.Builder().setAspectRatio(Rational(16, 9)).build())) },
-                                    onClose = { phase = 2 })
+                                    onClose = { phase = 2 }, shareId = "synthetic-presentation", onAnnotation = strokes::add)
                                 else -> Text("Viewer closed; call track retained")
                             }
                         }
@@ -118,6 +119,12 @@ class ChatAndShareUiTest {
                         android.graphics.Color.red(left) > android.graphics.Color.red(right) + 30
                 }
                 assertNotEquals("The shared picture must render, not just negotiate", left, right)
+                ui.onNodeWithText("Draw").performClick()
+                val shared = ui.onNodeWithContentDescription("Shared screen, 100 percent zoom. Pinch to zoom and drag to pan.")
+                shared.performTouchInput { down(center); moveBy(androidx.compose.ui.geometry.Offset(40f, 20f), delayMillis = 80) }
+                ui.runOnIdle { assertTrue("A stroke must leave while the finger is still down", strokes.isNotEmpty()) }
+                shared.performTouchInput { up() }
+                ui.onNodeWithText("Draw").performClick()
                 ui.onNodeWithText("Pop out").performClick()
                 ui.waitUntil(20_000) { pip }
                 ui.onNodeWithText("Synthetic workshop presentation").assertDoesNotExist()

app/src/main/AndroidManifest.xml

@@ -82,6 +82,11 @@
             </intent-filter>
         </activity>
 
+        <provider android:name="androidx.core.content.FileProvider"
+            android:authorities="${applicationId}.images" android:exported="false" android:grantUriPermissions="true">
+            <meta-data android:name="android.support.FILE_PROVIDER_PATHS" android:resource="@xml/image_paths" />
+        </provider>
+
         <service
             android:name=".service.ScreenShareService"
             android:exported="false"