diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 2935219b..4104998d 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -37,8 +37,8 @@ android { minSdk = 33 targetSdk = 35 // Remain newer than the 0.6.3 rendezvous candidate (code 26). - versionCode = 30 - versionName = "0.6.7" + versionCode = 31 + versionName = "0.6.8" testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" @@ -80,6 +80,7 @@ android { } sourceSets.getByName("androidTest").assets.srcDir("../protocol/src/test/resources") + sourceSets.getByName("androidTest").assets.srcDir("src/test/resources/attachments") sourceSets.getByName("androidTest").assets.srcDir(layout.buildDirectory.dir("browser-call-interop")) // The reviewed Link bundle is unpacked into build/link-bridge only after // its archive and per-file manifest have been verified. Never copy these diff --git a/app/src/androidTest/AndroidManifest.xml b/app/src/androidTest/AndroidManifest.xml index ad051e5c..de401553 100644 --- a/app/src/androidTest/AndroidManifest.xml +++ b/app/src/androidTest/AndroidManifest.xml @@ -5,5 +5,7 @@ the real NIP-55 intent/result boundary without introducing a signer into the production application. --> - + + + diff --git a/app/src/androidTest/kotlin/dev/forgesworn/kithmoot/media/PlaybackCaptureTest.kt b/app/src/androidTest/kotlin/dev/forgesworn/kithmoot/media/PlaybackCaptureTest.kt new file mode 100644 index 00000000..bce47452 --- /dev/null +++ b/app/src/androidTest/kotlin/dev/forgesworn/kithmoot/media/PlaybackCaptureTest.kt @@ -0,0 +1,124 @@ +package dev.forgesworn.kithmoot.media + +import android.app.Activity +import android.content.ComponentName +import android.content.Intent +import android.media.AudioAttributes +import android.media.AudioFormat +import android.media.AudioTrack +import android.media.projection.MediaProjectionManager +import android.os.Bundle +import android.os.SystemClock +import android.view.accessibility.AccessibilityNodeInfo +import android.widget.TextView +import androidx.activity.result.contract.ActivityResultContracts +import androidx.test.core.app.ActivityScenario +import androidx.test.platform.app.InstrumentationRegistry +import dev.forgesworn.kithmoot.MainActivity +import dev.forgesworn.kithmoot.service.ScreenShareService +import kotlinx.coroutines.* +import kotlinx.coroutines.flow.first +import org.junit.Assert.* +import org.junit.Assume.assumeTrue +import org.junit.Test +import org.webrtc.EglBase +import org.webrtc.PeerConnectionFactory +import org.webrtc.audio.JavaAudioDeviceModule +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import kotlin.math.sin + +/** Separate instrumentation-package UID: captured media, never a person's app. */ +class PlaybackToneActivity : Activity() { + private var tone: AudioTrack? = null + override fun onCreate(state: Bundle?) { + super.onCreate(state) + setContentView(TextView(this).apply { text = "Synthetic playback capture test" }) + val samples = ShortArray(48_000) { (sin(it * 2 * Math.PI * 440 / 48_000) * 8_000).toInt().toShort() } + tone = AudioTrack.Builder().setAudioAttributes(AudioAttributes.Builder() + .setUsage(AudioAttributes.USAGE_MEDIA).setAllowedCapturePolicy(AudioAttributes.ALLOW_CAPTURE_BY_ALL).build()) + .setAudioFormat(AudioFormat.Builder().setSampleRate(48_000).setEncoding(AudioFormat.ENCODING_PCM_16BIT) + .setChannelMask(AudioFormat.CHANNEL_OUT_MONO).build()) + .setTransferMode(AudioTrack.MODE_STATIC).setBufferSizeInBytes(samples.size * 2).build().also { + it.write(samples, 0, samples.size); it.setLoopPoints(0, samples.size, -1); it.play() + } + } + override fun onNewIntent(intent: Intent) { super.onNewIntent(intent); if (intent.getBooleanExtra("stop", false)) finish() } + override fun onDestroy() { tone?.stop(); tone?.release(); super.onDestroy() } +} + +class PlaybackCaptureTest { + @Test fun playback_enters_the_audio_buffer_and_stops_with_screen_capture() = runBlocking { + // The automatic consent click is confined to a disposable emulator. + assumeTrue(android.os.Build.HARDWARE in setOf("ranchu", "goldfish")) + val instrumentation = InstrumentationRegistry.getInstrumentation() + val context = instrumentation.targetContext + val automation = instrumentation.uiAutomation + automation.grantRuntimePermission(context.packageName, android.Manifest.permission.RECORD_AUDIO) + automation.grantRuntimePermission(context.packageName, android.Manifest.permission.POST_NOTIFICATIONS) + var permission: Intent? = null + val returned = CountDownLatch(1) + ActivityScenario.launch(MainActivity::class.java).use { scenario -> + scenario.onActivity { activity -> + val launcher = activity.activityResultRegistry.register("synthetic-projection", ActivityResultContracts.StartActivityForResult()) { result -> + if (result.resultCode == Activity.RESULT_OK) permission = result.data + returned.countDown() + } + val manager = context.getSystemService(MediaProjectionManager::class.java) + launcher.launch(if (android.os.Build.VERSION.SDK_INT >= 34) manager.createScreenCaptureIntent( + android.media.projection.MediaProjectionConfig.createConfigForDefaultDisplay()) else manager.createScreenCaptureIntent()) + } + val deadline = SystemClock.uptimeMillis() + 15_000 + while (returned.count > 0 && SystemClock.uptimeMillis() < deadline) { + val root = automation.rootInActiveWindow + for (label in listOf("Entire screen", "Start now", "Start recording", "Share screen", "Start")) { + root?.findAccessibilityNodeInfosByText(label)?.firstOrNull { it.isClickable } + ?.performAction(AccessibilityNodeInfo.ACTION_CLICK) + } + SystemClock.sleep(200) + } + assertTrue("Screen capture consent did not complete", returned.await(1, TimeUnit.SECONDS)) + assertNotNull(permission) + instrumentation.runOnMainSync { ScreenShareService.start(context) } + withTimeout(5_000) { ScreenShareService.running.first { it } } + PeerConnectionFactory.initialize(PeerConnectionFactory.InitializationOptions.builder(context).createInitializationOptions()) + val playback = PlaybackAudio() + val callbackPeak = java.util.concurrent.atomic.AtomicInteger(0) + val module = JavaAudioDeviceModule.builder(context).setSampleRate(48_000) + .setAudioBufferCallback { buffer, format, channels, rate, bytes, time -> + playback.fill(buffer, format, channels, rate, bytes) + val pcm = buffer.duplicate().order(ByteOrder.LITTLE_ENDIAN) + callbackPeak.set((0 until bytes / 2).maxOf { kotlin.math.abs(pcm.getShort(it * 2).toInt()) }) + time + } + .createAudioDeviceModule() + val factory = PeerConnectionFactory.builder().setAudioDeviceModule(module).createPeerConnectionFactory() + val egl = EglBase.create() + val media = LocalMedia(context, factory, egl, playback) + try { + media.startScreenShare(permission!!) + assertTrue(playback.active) + assertFalse(playback.microphone) + val tone = Intent().setComponent(ComponentName(instrumentation.context.packageName, PlaybackToneActivity::class.java.name)) + .addFlags(Intent.FLAG_ACTIVITY_NEW_TASK) + context.startActivity(tone) + val buffer = ByteBuffer.allocateDirect(960).order(ByteOrder.LITTLE_ENDIAN) + module.requestStartRecording() + withTimeout(10_000) { while (callbackPeak.get() < 100) delay(10) } + assertTrue("Another app's sound must enter WebRTC's input callback", callbackPeak.get() > 100) + assertEquals(listOf("screen-audio", "screen"), media.tracks.value.map { it.role }) + media.stopScreenShare() + assertFalse(playback.active) + assertTrue(media.tracks.value.isEmpty()) + playback.fill(buffer, AudioFormat.ENCODING_PCM_16BIT, 1, 48_000, 960) + assertTrue((0 until 960).all { buffer.get(it) == 0.toByte() }) + } finally { + module.requestStopRecording(); media.releaseAll(); factory.dispose(); module.release(); egl.release(); ScreenShareService.stop(context) + context.startActivity(Intent().setComponent(ComponentName(instrumentation.context.packageName, PlaybackToneActivity::class.java.name)) + .addFlags(Intent.FLAG_ACTIVITY_NEW_TASK or Intent.FLAG_ACTIVITY_CLEAR_TOP or Intent.FLAG_ACTIVITY_SINGLE_TOP).putExtra("stop", true)) + } + } + } +} diff --git a/app/src/androidTest/kotlin/dev/forgesworn/kithmoot/ui/AttachmentViewerTest.kt b/app/src/androidTest/kotlin/dev/forgesworn/kithmoot/ui/AttachmentViewerTest.kt new file mode 100644 index 00000000..935a4b2f --- /dev/null +++ b/app/src/androidTest/kotlin/dev/forgesworn/kithmoot/ui/AttachmentViewerTest.kt @@ -0,0 +1,65 @@ +package dev.forgesworn.kithmoot.ui + +import android.graphics.Bitmap +import androidx.activity.compose.setContent +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.runtime.* +import androidx.compose.ui.Modifier +import androidx.compose.ui.test.* +import androidx.compose.ui.test.junit4.createEmptyComposeRule +import androidx.test.core.app.ActivityScenario +import androidx.test.platform.app.InstrumentationRegistry +import dev.forgesworn.kithmoot.MainActivity +import dev.forgesworn.kithmoot.session.ChatAttachment +import dev.forgesworn.kithmoot.session.ChatMessage +import dev.forgesworn.kithmoot.ui.room.ChatPane +import dev.forgesworn.kithmoot.ui.theme.KithMootTheme +import kotlinx.serialization.json.* +import org.junit.Assert.* +import org.junit.Assume.assumeTrue +import org.junit.Rule +import org.junit.Test +import java.io.File + +class AttachmentViewerTest { + @get:Rule val ui = createEmptyComposeRule() + @Test fun opens_a_web_encrypted_image_fits_zooms_and_closes_on_retraction() { + val url = InstrumentationRegistry.getArguments().getString("attachmentFixtureUrl") + assumeTrue("Supply an HTTPS host serving the exact synthetic web-image.enc fixture", url?.startsWith("https://") == true) + val instrumentation = InstrumentationRegistry.getInstrumentation() + val meta = Json.parseToJsonElement(instrumentation.context.assets.open("web-image.json").bufferedReader().readText()).jsonObject + val attachment = ChatAttachment(url!!, meta.getValue("sha256").jsonPrimitive.content, meta.getValue("key").jsonPrimitive.content, "picture.png") + val message = ChatMessage("image", "01".repeat(32), "02".repeat(32), "A synthetic image", 1_800_000_000, attachments = listOf(attachment)) + var messages by mutableStateOf(listOf(message)) + ActivityScenario.launch(MainActivity::class.java).use { scenario -> + scenario.onActivity { it.setContent { KithMootTheme { ChatPane(messages, "03".repeat(32), {}, Modifier.fillMaxSize()) } } } + ui.onNodeWithText("Open attachment: picture.png").performClick() + ui.waitUntil(35_000) { ui.onNodeWithContentDescription("picture.png").isDisplayed() } + ui.onNodeWithText("Actual size").performClick() + ui.onNodeWithText("Fit").performClick() + ui.onNodeWithContentDescription("picture.png").performTouchInput { + pinch(center - androidx.compose.ui.geometry.Offset(30f, 0f), center + androidx.compose.ui.geometry.Offset(30f, 0f), + center - androidx.compose.ui.geometry.Offset(100f, 0f), center + androidx.compose.ui.geometry.Offset(100f, 0f)) + } + ui.onNodeWithText("Fit").performClick() + val bounds = ui.onNodeWithContentDescription("picture.png").fetchSemanticsNode().boundsInWindow + ui.waitUntil(5_000) { + val frame = instrumentation.uiAutomation.takeScreenshot() + val margin = frame.getPixel(bounds.center.x.toInt(), (bounds.top + 5).toInt()) + frame.recycle() + android.graphics.Color.red(margin) > 240 && android.graphics.Color.blue(margin) > 240 + } + val bitmap = instrumentation.uiAutomation.takeScreenshot() + val left = bitmap.getPixel((bounds.left + bounds.width / 3).toInt(), bounds.center.y.toInt()) + val right = bitmap.getPixel((bounds.left + bounds.width * 2 / 3).toInt(), bounds.center.y.toInt()) + assertTrue(android.graphics.Color.red(left) > 180) + assertTrue(android.graphics.Color.blue(right) > 180) + File(instrumentation.targetContext.getExternalFilesDir(null), "attachment-viewer.png").outputStream().use { + bitmap.compress(Bitmap.CompressFormat.PNG, 100, it) + } + ui.runOnIdle { messages = messages + message.copy(id = "retraction", body = "Retracted", sentAt = message.sentAt + 1, attachments = emptyList(), retracts = message.id) } + ui.waitUntil(5_000) { !ui.onNodeWithText("Close image").isDisplayed() } + ui.onNodeWithText("Message retracted").assertIsDisplayed() + } + } +} diff --git a/app/src/androidTest/kotlin/dev/forgesworn/kithmoot/ui/ChatAndShareUiTest.kt b/app/src/androidTest/kotlin/dev/forgesworn/kithmoot/ui/ChatAndShareUiTest.kt index f61a878c..dd20e9f5 100644 --- a/app/src/androidTest/kotlin/dev/forgesworn/kithmoot/ui/ChatAndShareUiTest.kt +++ b/app/src/androidTest/kotlin/dev/forgesworn/kithmoot/ui/ChatAndShareUiTest.kt @@ -51,6 +51,7 @@ class ChatAndShareUiTest { for (i in 0 until buffer.dataV.capacity()) buffer.dataV.put(i, 150.toByte()) val frame = VideoFrame(buffer, 0, System.nanoTime()); source.capturerObserver.onFrameCaptured(frame); frame.release(); frameNumber++ }, 0, 100, TimeUnit.MILLISECONDS) + val strokes = mutableListOf() var phase by mutableIntStateOf(0) var live by mutableStateOf(track) var pip by mutableStateOf(false) @@ -70,7 +71,7 @@ class ChatAndShareUiTest { onReact = { target, emoji -> messages = messages + first.copy(id = "reaction-${messages.size}", participant = self, reaction = toggleReaction(messages, target, self, emoji)) }) 1 -> ScreenShareViewer(live, egl, "Synthetic workshop presentation", pip, onPopOut = { assertTrue(activity.enterPictureInPictureMode(PictureInPictureParams.Builder().setAspectRatio(Rational(16, 9)).build())) }, - onClose = { phase = 2 }) + onClose = { phase = 2 }, shareId = "synthetic-presentation", onAnnotation = strokes::add) else -> Text("Viewer closed; call track retained") } } @@ -118,6 +119,12 @@ class ChatAndShareUiTest { android.graphics.Color.red(left) > android.graphics.Color.red(right) + 30 } assertNotEquals("The shared picture must render, not just negotiate", left, right) + ui.onNodeWithText("Draw").performClick() + val shared = ui.onNodeWithContentDescription("Shared screen, 100 percent zoom. Pinch to zoom and drag to pan.") + shared.performTouchInput { down(center); moveBy(androidx.compose.ui.geometry.Offset(40f, 20f), delayMillis = 80) } + ui.runOnIdle { assertTrue("A stroke must leave while the finger is still down", strokes.isNotEmpty()) } + shared.performTouchInput { up() } + ui.onNodeWithText("Draw").performClick() ui.onNodeWithText("Pop out").performClick() ui.waitUntil(20_000) { pip } ui.onNodeWithText("Synthetic workshop presentation").assertDoesNotExist() diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 2cd55aa8..cd8b2bab 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -82,6 +82,11 @@ + + + + >(emptyList()) val tracks: StateFlow> = _tracks.asStateFlow() + private var microphoneRequested = false private var audioSource: AudioSource? = null private var audioTrack: AudioTrack? = null @@ -129,12 +133,22 @@ class LocalMedia( override fun onCameraClosed() = Unit } - val microphoneTrack: AudioTrack? get() = audioTrack + val microphoneTrack: AudioTrack? get() = audioTrack?.takeIf { microphoneRequested } val localCameraTrack: VideoTrack? get() = cameraTrack val localScreenTrack: VideoTrack? get() = screenTrack @Synchronized fun startMicrophone(): AudioTrack? { + microphoneRequested = true + micMuted = false + playbackAudio.microphone = true + val track = ensureAudioTrack() + track.setEnabled(true) + publish() + return track + } + + private fun ensureAudioTrack(): AudioTrack { audioTrack?.let { return it } val constraints = MediaConstraints().apply { // Left to the platform's hardware canceller where there is one; the @@ -147,15 +161,19 @@ class LocalMedia( val track = factory.createAudioTrack(trackId(Roles.MIC), source) audioSource = source audioTrack = track - // A microphone that has just been started is not muted, whatever the - // last one was. - micMuted = false - publish() return track } @Synchronized fun stopMicrophone() { + microphoneRequested = false + playbackAudio.microphone = false + micMuted = false + if (!playbackAudio.active) releaseAudioTrack() + publish() + } + + private fun releaseAudioTrack() { audioTrack?.let { runCatching { it.setEnabled(false) } } audioTrack = null audioSource?.let { runCatching { it.dispose() } } @@ -263,7 +281,7 @@ class LocalMedia( * create the projection at all. */ @Synchronized - fun startScreenShare(permission: Intent): VideoTrack? { + fun startScreenShare(permission: Intent, shareAudio: Boolean = true): VideoTrack? { screenTrack?.let { return it } val callback = object : MediaProjection.Callback() { override fun onStop() { @@ -275,21 +293,33 @@ class LocalMedia( // isScreencast = true, so the encoder favours sharpness over frame rate. // Text on a shared slide is unreadable otherwise. val source = factory.createVideoSource(true) - capturer.initialize(helper, context, source.capturerObserver) - val size = screenSize() - capturer.startCapture(size.first, size.second, SCREEN_FPS) - val track = factory.createVideoTrack(trackId(Roles.SCREEN), source) - screenCapturer = capturer screenHelper = helper screenSource = source + val track = try { + capturer.initialize(helper, context, source.capturerObserver) + val size = screenSize() + capturer.startCapture(size.first, size.second, SCREEN_FPS) + factory.createVideoTrack(trackId(Roles.SCREEN), source) + } catch (failure: Exception) { stopScreenShare(); throw failure } screenTrack = track + if (shareAudio && androidx.core.content.ContextCompat.checkSelfPermission(context, android.Manifest.permission.RECORD_AUDIO) == android.content.pm.PackageManager.PERMISSION_GRANTED) { + try { + playbackAudio.start(checkNotNull(capturer.mediaProjection)) + ensureAudioTrack().setEnabled(true) + } catch (_: Exception) { + playbackAudio.stop() + onBackgroundTrouble?.invoke("Screen shared without sound: Android could not start app audio capture.") + } + } else if (shareAudio) onBackgroundTrouble?.invoke("Screen shared without sound. Allow audio recording to share app sound.") publish() return track } @Synchronized fun stopScreenShare() { + playbackAudio.stop() + if (!microphoneRequested) releaseAudioTrack() else audioTrack?.setEnabled(!micMuted) runCatching { screenCapturer?.stopCapture() } runCatching { screenCapturer?.dispose() } runCatching { screenHelper?.dispose() } @@ -318,10 +348,11 @@ class LocalMedia( */ @Synchronized fun setMicrophoneMuted(muted: Boolean): Boolean { - val track = audioTrack ?: return false + val track = microphoneTrack ?: return false if (micMuted == muted) return true micMuted = muted - runCatching { track.setEnabled(!muted) } + playbackAudio.microphone = !muted + runCatching { track.setEnabled(playbackAudio.active || !muted) } // Republished at once, so the roster says so on this mute rather than // on the next thing that happens to change. publish() @@ -335,7 +366,10 @@ class LocalMedia( private fun publish() { _tracks.value = buildList { - audioTrack?.let { add(LocalTrack(it, Roles.MIC, muted = micMuted)) } + audioTrack?.let { add(LocalTrack(it, + if (playbackAudio.active) Roles.SCREEN_AUDIO else Roles.MIC, + muted = micMuted && !playbackAudio.active, + microphoneOn = microphoneRequested, microphoneMuted = micMuted)) } cameraTrack?.let { add(LocalTrack(it, Roles.CAMERA)) } screenTrack?.let { add(LocalTrack(it, Roles.SCREEN)) } } diff --git a/app/src/main/kotlin/dev/forgesworn/kithmoot/media/PlaybackAudio.kt b/app/src/main/kotlin/dev/forgesworn/kithmoot/media/PlaybackAudio.kt new file mode 100644 index 00000000..73778d5c --- /dev/null +++ b/app/src/main/kotlin/dev/forgesworn/kithmoot/media/PlaybackAudio.kt @@ -0,0 +1,67 @@ +package dev.forgesworn.kithmoot.media + +import android.annotation.SuppressLint +import android.media.AudioAttributes +import android.media.AudioFormat +import android.media.AudioPlaybackCaptureConfiguration +import android.media.AudioRecord +import android.media.projection.MediaProjection +import android.os.Process +import java.nio.ByteBuffer +import java.nio.ByteOrder + +/** One audio input for WebRTC: optional microphone plus consented app playback. + * KithMoot's own output is excluded so listeners are never looped back. + */ +class PlaybackAudio { + @Volatile var microphone = false + private var recorder: AudioRecord? = null + private val playback = ByteBuffer.allocateDirect(960).order(ByteOrder.LITTLE_ENDIAN) + val active: Boolean @Synchronized get() = recorder != null + + @SuppressLint("MissingPermission") // Caller checks RECORD_AUDIO before starting. + @Synchronized + fun start(projection: MediaProjection) { + if (recorder != null) return + val config = AudioPlaybackCaptureConfiguration.Builder(projection) + .addMatchingUsage(AudioAttributes.USAGE_MEDIA) + .addMatchingUsage(AudioAttributes.USAGE_GAME) + .addMatchingUsage(AudioAttributes.USAGE_UNKNOWN) + .excludeUid(Process.myUid()).build() + val format = AudioFormat.Builder().setSampleRate(48_000) + .setEncoding(AudioFormat.ENCODING_PCM_16BIT).setChannelMask(AudioFormat.CHANNEL_IN_MONO).build() + val made = AudioRecord.Builder().setAudioFormat(format) + .setBufferSizeInBytes(maxOf(9_600, AudioRecord.getMinBufferSize(48_000, + AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT))) + .setAudioPlaybackCaptureConfig(config).build() + try { + check(made.state == AudioRecord.STATE_INITIALIZED) { "Android could not capture app sound." } + made.startRecording() + check(made.recordingState == AudioRecord.RECORDSTATE_RECORDING) + recorder = made + } catch (error: Exception) { made.release(); throw error } + } + + @Synchronized + fun stop() { recorder?.let { runCatching { it.stop() }; it.release() }; recorder = null } + + /** Always overwrite the input when the microphone is off, even on failure. */ + @Synchronized + fun fill(buffer: ByteBuffer, format: Int, channels: Int, rate: Int, bytes: Int) { + val length = minOf(bytes, buffer.capacity()) + val input = buffer.duplicate().order(ByteOrder.LITTLE_ENDIAN) + val mic = microphone + if (!mic) for (i in 0 until length) input.put(i, 0) + if (format != AudioFormat.ENCODING_PCM_16BIT || channels != 1 || rate != 48_000 || length > playback.capacity()) return + val record = recorder ?: return + playback.clear() + val read = runCatching { record.read(playback, length, AudioRecord.READ_NON_BLOCKING) }.getOrDefault(0) + for (i in 0 until (read.coerceAtLeast(0) / 2)) { + val offset = i * 2 + input.putShort(offset, mixPcm(input.getShort(offset), playback.getShort(offset))) + } + } +} + +internal fun mixPcm(microphone: Short, playback: Short): Short = + (microphone.toInt() + playback.toInt()).coerceIn(Short.MIN_VALUE.toInt(), Short.MAX_VALUE.toInt()).toShort() diff --git a/app/src/main/kotlin/dev/forgesworn/kithmoot/media/WebRtcEngine.kt b/app/src/main/kotlin/dev/forgesworn/kithmoot/media/WebRtcEngine.kt index 9de572c7..38f23e3c 100644 --- a/app/src/main/kotlin/dev/forgesworn/kithmoot/media/WebRtcEngine.kt +++ b/app/src/main/kotlin/dev/forgesworn/kithmoot/media/WebRtcEngine.kt @@ -92,6 +92,7 @@ class WebRtcEngine( * it. It owns an `AudioRecord`, an `AudioTrack` and their threads, and only * [dispose] gives them back. */ + private val playbackAudio = PlaybackAudio() private val audioDevice: JavaAudioDeviceModule val localMedia: LocalMedia val audioRouting = CallAudioRouting(context) @@ -134,6 +135,11 @@ class WebRtcEngine( .createInitializationOptions(), ) audioDevice = JavaAudioDeviceModule.builder(context.applicationContext) + .setSampleRate(48_000) + .setAudioBufferCallback { buffer, format, channels, rate, bytes, timestamp -> + playbackAudio.fill(buffer, format, channels, rate, bytes) + timestamp + } .setUseHardwareAcousticEchoCanceler(true) .setUseHardwareNoiseSuppressor(true) .createAudioDeviceModule() @@ -142,7 +148,7 @@ class WebRtcEngine( .setVideoEncoderFactory(DefaultVideoEncoderFactory(eglBase.eglBaseContext, true, true)) .setVideoDecoderFactory(DefaultVideoDecoderFactory(eglBase.eglBaseContext)) .createPeerConnectionFactory() - localMedia = LocalMedia(context.applicationContext, factory, eglBase) + localMedia = LocalMedia(context.applicationContext, factory, eglBase, playbackAudio) } @Volatile var callActive: Boolean = true diff --git a/app/src/main/kotlin/dev/forgesworn/kithmoot/service/ScreenShareService.kt b/app/src/main/kotlin/dev/forgesworn/kithmoot/service/ScreenShareService.kt index dc70bd70..eb18047f 100644 --- a/app/src/main/kotlin/dev/forgesworn/kithmoot/service/ScreenShareService.kt +++ b/app/src/main/kotlin/dev/forgesworn/kithmoot/service/ScreenShareService.kt @@ -98,7 +98,7 @@ class ScreenShareService : Service() { ) return NotificationCompat.Builder(this, CHANNEL_ID) .setContentTitle("Sharing your screen") - .setContentText("Everyone in the room can see this device's screen.") + .setContentText("Your selected screen or app is being shared.") .setSmallIcon(R.drawable.ic_screen_share) .setContentIntent(open) .setOngoing(true) diff --git a/app/src/main/kotlin/dev/forgesworn/kithmoot/session/Chat.kt b/app/src/main/kotlin/dev/forgesworn/kithmoot/session/Chat.kt index ea50743d..77f67568 100644 --- a/app/src/main/kotlin/dev/forgesworn/kithmoot/session/Chat.kt +++ b/app/src/main/kotlin/dev/forgesworn/kithmoot/session/Chat.kt @@ -81,6 +81,7 @@ data class ChatMessage( val lane: Lane? = null, /** Verified participant signature, bound to this room and outer device. */ val assignment: NostrEvent? = null, + val attachments: List = emptyList(), ) fun encodeChatEvent( @@ -182,6 +183,9 @@ fun decodeChatEvent( // thing about one other message, and a payload carrying two of // them, or one beside conversation it has no business carrying, // is refused whole. Mirrors `decodeChatEvent` in src/chat.ts. + val rawAttachments = json["attachments"] as? kotlinx.serialization.json.JsonArray + require(rawAttachments == null || rawAttachments.size <= 4) + val attachments = rawAttachments?.mapNotNull(::parseAttachment).orEmpty() val statements = listOf("reaction", "replaces", "retracts", "invite", "assignment").count { json.containsKey(it) } val conversationKeys = listOf("kind", "attachments", "reply", "thread", "mentions") val hasStatementAlone = json.containsKey("reaction") || json.containsKey("retracts") || json.containsKey("invite") || json.containsKey("assignment") @@ -231,6 +235,7 @@ fun decodeChatEvent( mentions = mentions, invite = invite, assignment = assignment, + attachments = attachments, ) } } diff --git a/app/src/main/kotlin/dev/forgesworn/kithmoot/session/ChatAttachment.kt b/app/src/main/kotlin/dev/forgesworn/kithmoot/session/ChatAttachment.kt new file mode 100644 index 00000000..5fbba755 --- /dev/null +++ b/app/src/main/kotlin/dev/forgesworn/kithmoot/session/ChatAttachment.kt @@ -0,0 +1,94 @@ +package dev.forgesworn.kithmoot.session + +import dev.forgesworn.kithmoot.crypto.Digests +import dev.forgesworn.kithmoot.crypto.toHex +import kotlinx.serialization.json.* +import java.net.URI +import java.nio.ByteBuffer +import java.nio.charset.CodingErrorAction +import java.text.Normalizer +import javax.crypto.Cipher +import javax.crypto.spec.GCMParameterSpec +import javax.crypto.spec.SecretKeySpec + +data class ChatAttachment(val url: String, val sha256: String, val key: String, + val name: String? = null, val type: String? = null, val size: Long? = null) + +private val hex64 = Regex("[0-9a-fA-F]{64}") +fun parseAttachment(value: JsonElement): ChatAttachment? = runCatching { + val obj = value.jsonObject + fun string(key: String) = (obj[key] as? JsonPrimitive)?.takeIf { it.isString }?.content + val url = string("url") ?: return null + require(url.length in 1..2048 && URI(url).scheme.equals("https", true) && URI(url).host != null) + val hash = string("sha256")?.takeIf(hex64::matches) ?: return null + val key = string("key")?.takeIf(hex64::matches) ?: return null + require(obj["event"] == null || string("event")?.matches(hex64) == true) + ChatAttachment(url, hash.lowercase(), key.lowercase(), + string("name")?.filterNot { Character.isISOControl(it) || Character.getType(it) == Character.FORMAT.toInt() }?.take(255), + string("type")?.takeIf { it.length <= 128 && it.matches(Regex("[\\w.+-]+/[\\w.+-]+")) }?.lowercase(), + (obj["size"] as? JsonPrimitive)?.longOrNull?.takeIf { it >= 0 }) +}.getOrNull() + +const val MAX_IMAGE_ENVELOPE_BYTES = 32 * 1024 * 1024 +data class OpenedAttachment(val name: String, val type: String, val bytes: ByteArray) + +/** Compatible with the shared FSWNENC2 reader. No plaintext escapes before all + * records authenticate and the complete ciphertext hash matches the message. */ +fun openAttachment(envelope: ByteArray, attachment: ChatAttachment): OpenedAttachment { + require(envelope.size in 41..MAX_IMAGE_ENVELOPE_BYTES) { "Image attachment exceeds the 32 MiB viewing limit." } + require(Digests.sha256(envelope).toHex() == attachment.sha256) { "The download does not match the message." } + require(attachment.key.matches(hex64)) + val magic = envelope.copyOfRange(0, 8).toString(Charsets.US_ASCII) + val headerSize = when (magic) { "FSWNENC2" -> 56; "FSWNENC1", "WBLMENC1" -> 24; else -> error("Unknown encrypted file format.") } + require(envelope.size > headerSize) + val header = envelope.copyOfRange(0, headerSize) + val chunk = ByteBuffer.wrap(header).getInt(8) + val records = ByteBuffer.wrap(header).getInt(12) + require(chunk == 1_048_576 && records in 1..32) + val plainSize = envelope.size - headerSize - records * 16 + require(plainSize > (records - 1) * chunk && plainSize <= records * chunk) + val recovery = attachment.key.chunked(2).map { it.toInt(16).toByte() }.toByteArray() + val key = if (headerSize == 56) Digests.hkdfSha256(recovery, header.copyOfRange(24, 56), + "forgesworn-aes-256-gcm-chunked/v2".toByteArray(), 32) else recovery + val plaintext = ByteArray(plainSize) + try { + for (counter in 0 until records) { + val index = ByteBuffer.allocate(4).putInt(counter).array() + val nonce = header.copyOfRange(16, 24) + index + val cipher = Cipher.getInstance("AES/GCM/NoPadding") + cipher.init(Cipher.DECRYPT_MODE, SecretKeySpec(key, "AES"), GCMParameterSpec(128, nonce)) + cipher.updateAAD(header + index) + val start = headerSize + counter * (chunk + 16) + val end = minOf(envelope.size, start + chunk + 16) + val part = cipher.doFinal(envelope, start, end - start) + part.copyInto(plaintext, counter * chunk); part.fill(0) + } + val metadataSize = ByteBuffer.wrap(plaintext).int + require(metadataSize in 2..4096 && metadataSize + 4 < plaintext.size) + val text = Charsets.UTF_8.newDecoder().onMalformedInput(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(plaintext, 4, metadataSize)).toString() + val meta = Json.parseToJsonElement(text).jsonObject + require(meta.keys == setOf("name", "size", "type")) + val name = meta.getValue("name").jsonPrimitive.also { require(it.isString) }.content + val type = meta.getValue("type").jsonPrimitive.also { require(it.isString) }.content + val size = meta.getValue("size").jsonPrimitive.int + require(size in 1..MAX_IMAGE_ENVELOPE_BYTES) + require(name == canonicalAttachmentName(name)) + require(type.isNotEmpty() && type.length <= 255 && type == type.lowercase() && type.none { it.code < 32 || it.code == 127 }) + val canonical = buildJsonObject { put("name", name); put("size", size); put("type", type) }.toString() + require(text == canonical) { "Invalid encrypted metadata." } + val length = 4 + metadataSize + size + val padded = if (length <= chunk) { var n = 65_536; while (n < length) n *= 2; n } else ((length + chunk - 1) / chunk) * chunk + require(padded == plaintext.size && length <= plaintext.size) + return OpenedAttachment(name, type, plaintext.copyOfRange(4 + metadataSize, length)) + } finally { plaintext.fill(0); recovery.fill(0); key.fill(0) } +} + +private fun canonicalAttachmentName(value: String): String { + val leaf = value.split('/', '\\').last() + val cleaned = Normalizer.normalize(leaf, Normalizer.Form.NFC) + .filterNot { it.code < 32 || it.code == 127 }.replace(Regex("[<>:\"|?*]"), "_") + .trimStart('.').trim().ifEmpty { "blob.bin" } + val end = if (cleaned.length > 180 && cleaned[179].isHighSurrogate() && cleaned[180].isLowSurrogate()) 179 else 180 + return cleaned.take(end) +} diff --git a/app/src/main/kotlin/dev/forgesworn/kithmoot/session/RoomSession.kt b/app/src/main/kotlin/dev/forgesworn/kithmoot/session/RoomSession.kt index 28e8602f..5172fa2b 100644 --- a/app/src/main/kotlin/dev/forgesworn/kithmoot/session/RoomSession.kt +++ b/app/src/main/kotlin/dev/forgesworn/kithmoot/session/RoomSession.kt @@ -194,6 +194,7 @@ class RoomSession( * applied inside [unwrapSignal]. */ private val signalGuard = SignalGuard() + private val annotationGuard = SignalGuard(480) private val chatSeen = mutableSetOf() private val chatLog = mutableListOf() private val chatSenderTimes = linkedMapOf>() @@ -496,7 +497,14 @@ class RoomSession( * anything having to agree first. */ fun claim(role: String) { - synchronized(lock) { claims = claims + (role to now()) } + synchronized(lock) { + val at = now() + val latest = maxOf(claims[role] ?: 0L, roster.values + .filter { it.participant == identity.participant && it.updatedAt >= at - timing.presenceTtlSeconds } + .maxOfOrNull { it.claims[role] ?: 0L } ?: 0L) + check(latest < at + 60) { "Device clocks disagree. Wait a moment and try the handover again." } + claims = claims + (role to maxOf(at, latest + 1)) + } announce() } @@ -669,13 +677,14 @@ class RoomSession( // Rate limiting against the *sending device* rather than the wrap's // pubkey: every wrap is signed by a fresh ephemeral key, so the only // stable identity a budget can be held against is the one inside. - if (!signalGuard.admitEvent("inner:${signal.id}") || !signalGuard.admitSender(signal.from, at)) return + if (!signalGuard.admitEvent("inner:${signal.id}")) return val sender = synchronized(lock) { roster[signal.from] } // The roster authenticates sibling devices too. Only our exact local // device is excluded; paired cameras need ordinary negotiation. if (sender == null || signal.from == identity.devicePubkey) return if (signal.body.type == "annotation") { + if (!annotationGuard.admitSender(signal.from, at)) return // Shape-checked here, after the same roster and rate-limit checks // every other signal passes, and never falls through to ordinary // negotiation: an old reader that does not know this branch @@ -686,6 +695,7 @@ class RoomSession( } return } + if (!signalGuard.admitSender(signal.from, at)) return _signals.tryEmit(signal) } diff --git a/app/src/main/kotlin/dev/forgesworn/kithmoot/ui/KithMootApp.kt b/app/src/main/kotlin/dev/forgesworn/kithmoot/ui/KithMootApp.kt index 913544cf..4a80bee9 100644 --- a/app/src/main/kotlin/dev/forgesworn/kithmoot/ui/KithMootApp.kt +++ b/app/src/main/kotlin/dev/forgesworn/kithmoot/ui/KithMootApp.kt @@ -1,5 +1,8 @@ package dev.forgesworn.kithmoot.ui +import androidx.compose.ui.semantics.semantics +import androidx.compose.ui.semantics.contentDescription + import android.Manifest import android.app.Activity import android.media.projection.MediaProjectionManager @@ -66,6 +69,9 @@ fun KithMootApp(model: RoomViewModel, inPictureInPicture: Boolean = false, onPop val snackbars = remember { SnackbarHostState() } val roomUiState = rememberSaveableStateHolder() var searchOpen by remember { mutableStateOf(false) } + var shareOptions by remember { mutableStateOf(false) } + var shareDeviceSound by remember { mutableStateOf(true) } + var requestedScreenAudio by remember { mutableStateOf(true) } var cardsOpen by remember { mutableStateOf(false) } var expandedScreen by remember { mutableStateOf(null) } @@ -82,15 +88,16 @@ fun KithMootApp(model: RoomViewModel, inPictureInPicture: Boolean = false, onPop ) { result -> val data = result.data if (result.resultCode == Activity.RESULT_OK && data != null) { - model.startScreenShare(data) + model.startScreenShare(data, requestedScreenAudio) } else { model.screenShareDeclined() } } /** Asks Android for screen-capture consent. Notifications first, or the service cannot show one. */ - fun requestScreenShare() { - val launch = { + fun beginScreenShare() { + requestedScreenAudio = shareDeviceSound + val capture = { val manager = context.getSystemService(MediaProjectionManager::class.java) if (manager == null) { model.showNotice("This device has no screen capture.") @@ -98,6 +105,15 @@ fun KithMootApp(model: RoomViewModel, inPictureInPicture: Boolean = false, onPop projection.launch(manager.createScreenCaptureIntent()) } } + val launch = { + if (!requestedScreenAudio) capture() else asker.ask(PermissionAsk( + permission = Manifest.permission.RECORD_AUDIO, + title = "Share app sound", + why = "Android requires audio recording permission to share sound from apps that allow capture. Your microphone stays under its own call control.", + refused = "Audio permission was refused. Allow it in Settings to share your screen with app sound.", + onGranted = capture, + )) + } if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { asker.ask( PermissionAsk( @@ -114,6 +130,23 @@ fun KithMootApp(model: RoomViewModel, inPictureInPicture: Boolean = false, onPop } } + fun requestScreenShare() { shareOptions = true } + if (shareOptions && stage == Stage.ROOM) { + androidx.compose.material3.AlertDialog( + onDismissRequest = { shareOptions = false }, + title = { androidx.compose.material3.Text("Share your screen") }, + text = { + androidx.compose.foundation.layout.Column { + androidx.compose.material3.Text("Include device sound from apps that allow capture. This can include other apps' sound when you share a single app. Your microphone has its own call control.") + androidx.compose.material3.Switch(checked = shareDeviceSound, onCheckedChange = { shareDeviceSound = it }, + modifier = Modifier.semantics { contentDescription = "Share device sound" }) + } + }, + confirmButton = { androidx.compose.material3.TextButton(onClick = { shareOptions = false; beginScreenShare() }) { androidx.compose.material3.Text("Choose screen or app") } }, + dismissButton = { androidx.compose.material3.TextButton(onClick = { shareOptions = false }) { androidx.compose.material3.Text("Cancel") } }, + ) + } + LaunchedEffect(stage) { if (stage != Stage.ROOM) { expandedScreen = null; searchOpen = false } } val expanded = expandedScreen if (expanded != null && stage == Stage.ROOM) { @@ -123,6 +156,8 @@ fun KithMootApp(model: RoomViewModel, inPictureInPicture: Boolean = false, onPop track = meta?.let { videos["${it.device}|${it.role}"] }, eglBase = model.eglBase, title = if (tile?.isSelf == true) "Your screen" else "${dev.forgesworn.kithmoot.ui.room.shortId(expanded.participant)}’s screen", inPictureInPicture = inPictureInPicture, onPopOut = onPopOut, + shareId = meta?.trackId, marks = roomState.shareMarks[meta?.trackId].orEmpty(), + onAnnotation = model::drawOnShare, onClose = { expandedScreen = null }, ) return diff --git a/app/src/main/kotlin/dev/forgesworn/kithmoot/ui/RoomViewModel.kt b/app/src/main/kotlin/dev/forgesworn/kithmoot/ui/RoomViewModel.kt index 587062ad..391fac9b 100644 --- a/app/src/main/kotlin/dev/forgesworn/kithmoot/ui/RoomViewModel.kt +++ b/app/src/main/kotlin/dev/forgesworn/kithmoot/ui/RoomViewModel.kt @@ -2828,7 +2828,7 @@ class RoomViewModel(application: Application) : AndroidViewModel(application) { } // This device plays the room's audio unless one of your others takes it // over. Claiming rather than assuming is what lets that handover happen. - live.claim(Roles.MONITOR) + if (live.localRoles.value.monitorDevice == null) live.claim(Roles.MONITOR) notifications.begin(record.id, record.name, who.participant, epochSeconds()) scope.launch { @@ -2973,7 +2973,7 @@ class RoomViewModel(application: Application) : AndroidViewModel(application) { val media = engine ?: return@launch if (remembered) { Log.i(JOIN_LOG, "remembered join carried out now that media exists") - live.claim(Roles.MONITOR) + if (live.localRoles.value.monitorDevice == null) live.claim(Roles.MONITOR) adoptRoomCall() } launch { media.connections.collect { connections -> _room.update { if (session === live) it.copy(mediaConnections = connections) else it } } } @@ -3012,7 +3012,7 @@ class RoomViewModel(application: Application) : AndroidViewModel(application) { combine(media.localMedia.tracks, media.remoteTracks, live.localRoles) { local, remote, roles -> val listeningHere = media.callActive && (roles.monitorDevice == null || roles.holdsMonitor) _room.update { if (session === live) it.copy(listeningHere = listeningHere) else it } - local.any { it.role == Roles.MIC } || (listeningHere && remote.any { it.track is AudioTrack }) + local.any { it.microphoneOn } || (listeningHere && remote.any { it.track is AudioTrack }) }.distinctUntilChanged().collect { active -> media.audioRouting.setActive(active && media.callActive) } } launch { @@ -3067,6 +3067,23 @@ class RoomViewModel(application: Application) : AndroidViewModel(application) { } } + fun drawOnShare(annotation: dev.forgesworn.kithmoot.protocol.ScreenAnnotation) { + val live = session ?: return + val scope = sessionScope ?: return + if (!dev.forgesworn.kithmoot.protocol.isValidScreenAnnotation(annotation) || + _room.value.tiles.none { tile -> tile.videos.any { it.role == Roles.SCREEN && it.trackId == annotation.shareId } }) return + shareMarks.remember(annotation, MarkAuthor(live.identity.participant, "You")) + pushShareMarks() + ensureMarksTicking(scope) + scope.launch(Dispatchers.Default) { + for (device in live.remoteDevices.value) { + if (session !== live) break + runCatching { live.sendSignal(device, dev.forgesworn.kithmoot.protocol.SignalBody( + type = "annotation", roomId = live.room.roomId, annotation = annotation)) } + } + } + } + /** Recomputes every share's live marks and publishes them, so a fade in * progress is visible without waiting for the next stroke to arrive. */ private fun pushShareMarks() { @@ -3397,7 +3414,7 @@ class RoomViewModel(application: Application) : AndroidViewModel(application) { _room.update { it.copy(onCall = true, mediaRunning = true) } media.setCallActive(true) adoptRoomCall() - live.claim(Roles.MONITOR) + if (live.localRoles.value.monitorDevice == null) live.claim(Roles.MONITOR) } fun listenOnThisDevice() { if (_room.value.mediaRunning) session?.claim(Roles.MONITOR) } @@ -3517,7 +3534,7 @@ class RoomViewModel(application: Application) : AndroidViewModel(application) { * projection at all unless a `mediaProjection` service is already running, * and the failure is a `SecurityException` rather than a null. */ - fun startScreenShare(permission: Intent) { + fun startScreenShare(permission: Intent, shareAudio: Boolean = true) { if (!_room.value.mediaRunning) return val media = engine?.localMedia ?: return note("Screen sharing needs the media stack.") val scope = sessionScope ?: return @@ -3530,7 +3547,7 @@ class RoomViewModel(application: Application) : AndroidViewModel(application) { } val started = withContext(Dispatchers.Default) { synchronized(mediaControlLock) { runCatching { check(_room.value.mediaRunning && engine?.localMedia === media) { "The call has ended." } - media.startScreenShare(permission) + media.startScreenShare(permission, shareAudio) } } } if (started.getOrNull() == null) { ScreenShareService.stop(getApplication()) @@ -4357,8 +4374,8 @@ class RoomViewModel(application: Application) : AndroidViewModel(application) { private fun onLocalTracks(tracks: List) { _room.update { it.copy( - micOn = tracks.any { it.role == Roles.MIC }, - micMuted = tracks.any { it.role == Roles.MIC && it.muted }, + micOn = tracks.any { it.microphoneOn }, + micMuted = tracks.any { it.microphoneOn && it.microphoneMuted }, cameraOn = tracks.any { it.role == Roles.CAMERA }, screenOn = tracks.any { it.role == Roles.SCREEN }, ) } diff --git a/app/src/main/kotlin/dev/forgesworn/kithmoot/ui/room/AttachmentViewer.kt b/app/src/main/kotlin/dev/forgesworn/kithmoot/ui/room/AttachmentViewer.kt new file mode 100644 index 00000000..83fe1db5 --- /dev/null +++ b/app/src/main/kotlin/dev/forgesworn/kithmoot/ui/room/AttachmentViewer.kt @@ -0,0 +1,135 @@ +package dev.forgesworn.kithmoot.ui.room + +import android.content.ClipData +import android.content.Intent +import android.graphics.BitmapFactory +import androidx.compose.foundation.Image +import androidx.compose.foundation.gestures.detectTransformGestures +import androidx.compose.foundation.layout.* +import androidx.compose.material3.* +import androidx.compose.runtime.* +import androidx.compose.ui.Modifier +import androidx.compose.ui.geometry.Offset +import androidx.compose.ui.graphics.asImageBitmap +import androidx.compose.ui.graphics.graphicsLayer +import androidx.compose.ui.draw.clipToBounds +import androidx.compose.ui.input.pointer.pointerInput +import androidx.compose.ui.layout.onSizeChanged +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.unit.IntSize +import androidx.compose.ui.unit.dp +import androidx.compose.ui.window.Dialog +import androidx.compose.ui.window.DialogProperties +import androidx.core.content.FileProvider +import dev.forgesworn.kithmoot.session.* +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext +import okhttp3.OkHttpClient +import okhttp3.Request +import java.io.ByteArrayOutputStream +import java.io.File +import java.util.UUID +import java.util.concurrent.TimeUnit + +private val attachmentHttp = OkHttpClient.Builder().followRedirects(false).followSslRedirects(false) + .callTimeout(30, TimeUnit.SECONDS).build() + +/** Downloads happen only after a tap. No room key, cookies or identity header + * is sent to the file host, and decrypted files stay in memory until Open in. */ +@OptIn(ExperimentalLayoutApi::class) +@Composable +fun AttachmentViewer(attachment: ChatAttachment, onClose: () -> Unit) { + val context = LocalContext.current + val scope = rememberCoroutineScope() + var opened by remember(attachment) { mutableStateOf(null) } + var bitmap by remember(attachment) { mutableStateOf(null) } + var error by remember(attachment) { mutableStateOf(null) } + var zoom by remember(attachment) { mutableFloatStateOf(1f) } + var pan by remember(attachment) { mutableStateOf(Offset.Zero) } + var size by remember { mutableStateOf(IntSize.Zero) } + val call = remember(attachment) { attachmentHttp.newCall(Request.Builder().url(attachment.url).build()) } + DisposableEffect(attachment) { onDispose { call.cancel(); opened?.bytes?.fill(0) } } + LaunchedEffect(attachment) { + try { + val result = withContext(Dispatchers.IO) { + require(attachment.size == null || attachment.size <= MAX_IMAGE_ENVELOPE_BYTES) { "This attachment exceeds the 32 MiB image viewing limit." } + val bytes = call.execute().use { response -> + check(response.isSuccessful) { "The attachment host could not supply the image." } + val body = checkNotNull(response.body) + require(body.contentLength() <= MAX_IMAGE_ENVELOPE_BYTES) + body.byteStream().use { input -> + val output = ByteArrayOutputStream() + val buffer = ByteArray(8192) + while (true) { + val read = input.read(buffer); if (read < 0) break + require(output.size() + read <= MAX_IMAGE_ENVELOPE_BYTES) { "Image attachment exceeds 32 MiB." } + output.write(buffer, 0, read) + } + output.toByteArray() + } + } + val file = openAttachment(bytes, attachment) + try { + require(file.type in setOf("image/png", "image/jpeg", "image/webp", "image/gif")) { "This file type cannot be displayed in the Android image viewer." } + val bounds = BitmapFactory.Options().apply { inJustDecodeBounds = true } + BitmapFactory.decodeByteArray(file.bytes, 0, file.bytes.size, bounds) + require(bounds.outWidth > 0 && bounds.outHeight > 0 && bounds.outWidth.toLong() * bounds.outHeight <= 16_000_000) { "This image is too large for the Android viewer (16 megapixels maximum)." } + file to checkNotNull(BitmapFactory.decodeByteArray(file.bytes, 0, file.bytes.size)) { "The image could not be decoded." } + } catch (failure: Exception) { file.bytes.fill(0); throw failure } + } + opened = result.first; bitmap = result.second + } catch (cancelled: CancellationException) { throw cancelled } + catch (failure: Exception) { error = failure.message ?: "The image could not be opened." } + } + fun constrain(offset: Offset, scale: Float): Offset { + val image = bitmap ?: return Offset.Zero + val fit = minOf(size.width.toFloat() / image.width, size.height.toFloat() / image.height) + val x = ((image.width * fit * scale - size.width) / 2).coerceAtLeast(0f) + val y = ((image.height * fit * scale - size.height) / 2).coerceAtLeast(0f) + return Offset(offset.x.coerceIn(-x, x), offset.y.coerceIn(-y, y)) + } + Dialog(onDismissRequest = onClose, properties = DialogProperties(usePlatformDefaultWidth = false)) { + Surface(Modifier.fillMaxSize()) { + Column(Modifier.fillMaxSize().systemBarsPadding().padding(12.dp)) { + Text(opened?.name ?: attachment.name ?: "Image attachment", style = MaterialTheme.typography.titleMedium, maxLines = 2) + FlowRow { + TextButton(onClick = { zoom = 1f; pan = Offset.Zero }) { Text("Fit") } + TextButton(enabled = bitmap != null && size.width > 0 && size.height > 0, onClick = { + bitmap?.let { zoom = maxOf(1f, maxOf(it.width.toFloat() / size.width, it.height.toFloat() / size.height)); pan = Offset.Zero } + }) { Text("Actual size") } + TextButton(enabled = opened != null, onClick = { + val file = opened ?: return@TextButton + scope.launch { + runCatching { + val path = withContext(Dispatchers.IO) { + val directory = File(context.cacheDir, "opened-images").apply { mkdirs() } + directory.listFiles()?.filter { System.currentTimeMillis() - it.lastModified() > 3_600_000 }?.forEach { it.delete() } + File(directory, UUID.randomUUID().toString()).also { it.writeBytes(file.bytes) } + } + val uri = FileProvider.getUriForFile(context, "${context.packageName}.images", path) + val intent = Intent(Intent.ACTION_VIEW).setDataAndType(uri, file.type) + .addFlags(Intent.FLAG_GRANT_READ_URI_PERMISSION) + intent.clipData = ClipData.newRawUri("Image", uri) + context.startActivity(Intent.createChooser(intent, "Open image in")) + }.onFailure { error = "No image app could open this file." } + } + }) { Text("Open in…") } + TextButton(onClick = onClose) { Text("Close image") } + } + error?.let { Text(it, color = MaterialTheme.colorScheme.error) } + Box(Modifier.weight(1f).fillMaxWidth().clipToBounds().onSizeChanged { size = it; pan = constrain(pan, zoom) } + .pointerInput(bitmap, size) { + detectTransformGestures { _, movement, scale, _ -> + zoom = (zoom * scale).coerceIn(1f, 16f); pan = constrain(pan + movement, zoom) + } + }, contentAlignment = androidx.compose.ui.Alignment.Center) { + bitmap?.let { Image(it.asImageBitmap(), contentDescription = opened?.name ?: "Attached image", + modifier = Modifier.fillMaxSize().graphicsLayer { scaleX = zoom; scaleY = zoom; translationX = pan.x; translationY = pan.y }) } + if (bitmap == null && error == null) CircularProgressIndicator() + } + } + } + } +} diff --git a/app/src/main/kotlin/dev/forgesworn/kithmoot/ui/room/ChatPane.kt b/app/src/main/kotlin/dev/forgesworn/kithmoot/ui/room/ChatPane.kt index 86c49ef6..350b1388 100644 --- a/app/src/main/kotlin/dev/forgesworn/kithmoot/ui/room/ChatPane.kt +++ b/app/src/main/kotlin/dev/forgesworn/kithmoot/ui/room/ChatPane.kt @@ -67,6 +67,7 @@ fun ChatPane( onCloseSearch: () -> Unit = {}, onReadingChanged: (Boolean) -> Unit = {}, ) { + var expandedImage by remember { mutableStateOf(null) } var draft by rememberSaveable(stateSaver = TextFieldValue.Saver) { mutableStateOf(TextFieldValue("")) } var query by rememberSaveable { mutableStateOf("") } var emojiOpen by remember { mutableStateOf(false) } @@ -200,6 +201,11 @@ fun ChatPane( if (addressed) Text("Mentioned you", style = MaterialTheme.typography.labelSmall, color = MaterialTheme.colorScheme.primary) Text(if (r.retracted) "Message retracted" else message.body, style = MaterialTheme.typography.bodyLarge, color = if (r.retracted) MaterialTheme.colorScheme.onSurfaceVariant else MaterialTheme.colorScheme.onSurface) + if (!r.retracted) message.attachments.forEach { attachment -> + TextButton(onClick = { expandedImage = attachment }) { + Text("Open attachment: ${attachment.name ?: "Image"}") + } + } val meta = listOfNotNull(messageClock(message.sentAt), r.original.lane?.chip, if (r.edited && !r.retracted) "edited" else null, if (nested || r.orphan) "reply" else null) Text(meta.joinToString(" · "), Modifier.align(Alignment.End).padding(top = 3.dp), style = MaterialTheme.typography.labelSmall, @@ -236,6 +242,11 @@ fun ChatPane( style = MaterialTheme.typography.bodySmall, color = MaterialTheme.colorScheme.onSurfaceVariant) sendError?.let { Text(it, Modifier.padding(horizontal = 20.dp), color = MaterialTheme.colorScheme.error, style = MaterialTheme.typography.bodySmall) } } + expandedImage?.let { attachment -> + if (messages.any { it.attachments.contains(attachment) } && resolved.stream.flatMap { listOf(it) + it.replies }.any { !it.retracted && it.shown.attachments.contains(attachment) }) { + AttachmentViewer(attachment, onClose = { expandedImage = null }) + } else LaunchedEffect(attachment) { expandedImage = null } + } if (emojiOpen) EmojiDialog(onDismiss = { emojiOpen = false }) { emoji -> val start = draft.selection.min; val end = draft.selection.max val text = draft.text.replaceRange(start, end, emoji) diff --git a/app/src/main/kotlin/dev/forgesworn/kithmoot/ui/room/ScreenShareViewer.kt b/app/src/main/kotlin/dev/forgesworn/kithmoot/ui/room/ScreenShareViewer.kt index a376fe27..dd2f0848 100644 --- a/app/src/main/kotlin/dev/forgesworn/kithmoot/ui/room/ScreenShareViewer.kt +++ b/app/src/main/kotlin/dev/forgesworn/kithmoot/ui/room/ScreenShareViewer.kt @@ -5,6 +5,11 @@ import android.view.TextureView import androidx.activity.compose.BackHandler import androidx.compose.foundation.background import androidx.compose.foundation.gestures.detectTransformGestures +import androidx.compose.foundation.gestures.awaitEachGesture +import androidx.compose.foundation.gestures.awaitFirstDown +import dev.forgesworn.kithmoot.protocol.AnnotationPoint +import dev.forgesworn.kithmoot.protocol.ScreenAnnotation +import java.util.UUID import androidx.compose.foundation.layout.* import androidx.compose.material3.* import androidx.compose.runtime.* @@ -33,7 +38,10 @@ data class SharedScreen(val participant: String, val device: String) fun ScreenShareViewer( track: VideoTrack?, eglBase: EglBase?, title: String, inPictureInPicture: Boolean, onPopOut: (() -> Unit)?, onClose: () -> Unit, + shareId: String? = null, marks: List = emptyList(), + onAnnotation: (ScreenAnnotation) -> Unit = {}, ) { + var drawing by remember(shareId) { mutableStateOf(false) } var zoom by remember { mutableFloatStateOf(1f) } var pan by remember { mutableStateOf(Offset.Zero) } var aspect by remember { mutableFloatStateOf(16f / 9f) } @@ -58,6 +66,7 @@ fun ScreenShareViewer( Text("${(zoom * 100).toInt()}%", Modifier.padding(vertical = 14.dp)) TextButton(onClick = { setZoom(zoom * 1.25f) }, enabled = track != null && zoom < 8f, modifier = Modifier.semantics { contentDescription = "Zoom in" }) { Text("+") } TextButton(onClick = { setZoom(1f); pan = Offset.Zero }) { Text("Fit to screen") } + FilterChip(selected = drawing, onClick = { drawing = !drawing }, enabled = shareId != null && track != null, label = { Text("Draw") }) if (onPopOut != null) TextButton(onClick = onPopOut) { Text("Pop out") } TextButton(onClick = onClose) { Text("Close viewer") } } @@ -66,17 +75,38 @@ fun ScreenShareViewer( } Box(Modifier.weight(1f).fillMaxWidth().clipToBounds().onSizeChanged { size = it } .semantics { contentDescription = "Shared screen, ${(zoom * 100).toInt()} percent zoom. Pinch to zoom and drag to pan." } - .pointerInput(fittedWidth, fittedHeight) { - detectTransformGestures { _, movement, scale, _ -> setZoom(zoom * scale); pan = constrain(pan + movement, zoom) } + .pointerInput(fittedWidth, fittedHeight, drawing) { + if (!drawing) detectTransformGestures { _, movement, scale, _ -> setZoom(zoom * scale); pan = constrain(pan + movement, zoom) } }, contentAlignment = Alignment.Center) { if (track != null && eglBase != null) { val density = LocalDensity.current - ScreenTexture(track, eglBase, onAspect = { aspect = it }, modifier = Modifier - .size(with(density) { fittedWidth.toDp() }, with(density) { fittedHeight.toDp() }) - .graphicsLayer { scaleX = zoom; scaleY = zoom; translationX = pan.x; translationY = pan.y }) + Box(Modifier.size(with(density) { fittedWidth.toDp() }, with(density) { fittedHeight.toDp() }) + .graphicsLayer { scaleX = zoom; scaleY = zoom; translationX = pan.x; translationY = pan.y }) { + ScreenTexture(track, eglBase, onAspect = { aspect = it }, modifier = Modifier.fillMaxSize()) + ShareMarksOverlay(marks, Modifier.fillMaxSize()) + if (drawing && !inPictureInPicture && shareId != null) Box(Modifier.fillMaxSize() + .pointerInput(shareId) { + awaitEachGesture { + val down = awaitFirstDown() + fun point(offset: Offset) = AnnotationPoint( + (offset.x / size.width.coerceAtLeast(1)).toDouble().coerceIn(0.0, 1.0), + (offset.y / size.height.coerceAtLeast(1)).toDouble().coerceIn(0.0, 1.0)) + val stroke = StrokeSegments(shareId, onAnnotation) + stroke.start(point(down.position), down.uptimeMillis) + down.consume() + do { + val event = awaitPointerEvent() + val change = event.changes.firstOrNull { it.id == down.id } ?: break + stroke.move(point(change.position), change.uptimeMillis) + change.consume() + } while (change.pressed) + stroke.finish() + } + }) + } } else Text("Screen sharing has stopped or is reconnecting.", color = Color.White, modifier = Modifier.padding(24.dp)) } - if (!inPictureInPicture) Text("Pinch to zoom. Drag to move around. Fit to screen resets the view.", color = Color.White, style = MaterialTheme.typography.bodySmall, modifier = Modifier.padding(12.dp)) + if (!inPictureInPicture) Text(if (drawing) "Draw on the shared image. Everyone sees marks as you move." else "Pinch to zoom. Drag to move around. Fit to screen resets the view.", color = Color.White, style = MaterialTheme.typography.bodySmall, modifier = Modifier.padding(12.dp)) } } diff --git a/app/src/main/kotlin/dev/forgesworn/kithmoot/ui/room/StrokeSegments.kt b/app/src/main/kotlin/dev/forgesworn/kithmoot/ui/room/StrokeSegments.kt new file mode 100644 index 00000000..a18f3f60 --- /dev/null +++ b/app/src/main/kotlin/dev/forgesworn/kithmoot/ui/room/StrokeSegments.kt @@ -0,0 +1,22 @@ +package dev.forgesworn.kithmoot.ui.room + +import dev.forgesworn.kithmoot.protocol.AnnotationPoint +import dev.forgesworn.kithmoot.protocol.ScreenAnnotation +import java.util.UUID + +/** Each bounded segment has its own wire id; its last point joins the next. */ +class StrokeSegments(private val shareId: String, private val send: (ScreenAnnotation) -> Unit) { + private val points = mutableListOf() + private var sentAt = 0L + fun start(point: AnnotationPoint, at: Long) { points.clear(); points.add(point); sentAt = at } + fun move(point: AnnotationPoint, at: Long) { + if (points.lastOrNull() != point) points.add(point) + if (at - sentAt >= 50 || points.size >= 128) { flush(); sentAt = at } + } + fun finish() { flush(); points.clear() } + private fun flush() { + if (points.size < 2) return + send(ScreenAnnotation("stroke", shareId, UUID.randomUUID().toString(), points.toList())) + val last = points.last(); points.clear(); points.add(last) + } +} diff --git a/app/src/main/res/xml/image_paths.xml b/app/src/main/res/xml/image_paths.xml new file mode 100644 index 00000000..4df07c40 --- /dev/null +++ b/app/src/main/res/xml/image_paths.xml @@ -0,0 +1,3 @@ + + + diff --git a/app/src/test/kotlin/dev/forgesworn/kithmoot/media/PlaybackAudioTest.kt b/app/src/test/kotlin/dev/forgesworn/kithmoot/media/PlaybackAudioTest.kt new file mode 100644 index 00000000..6b293583 --- /dev/null +++ b/app/src/test/kotlin/dev/forgesworn/kithmoot/media/PlaybackAudioTest.kt @@ -0,0 +1,27 @@ +package dev.forgesworn.kithmoot.media + +import java.nio.ByteBuffer +import java.nio.ByteOrder +import kotlin.test.* + +class PlaybackAudioTest { + @Test fun `screen-only or muted capture never forwards microphone samples`() { + val audio = PlaybackAudio() + val buffer = ByteBuffer.allocateDirect(960).order(ByteOrder.LITTLE_ENDIAN) + for (i in 0 until 480) buffer.putShort(i * 2, 2345) + audio.fill(buffer, 2, 1, 48_000, 960) + assertTrue((0 until 960).all { buffer.get(it) == 0.toByte() }) + audio.microphone = true + buffer.putShort(0, 2345) + audio.fill(buffer, 2, 1, 48_000, 960) + assertEquals(2345.toShort(), buffer.getShort(0)) + audio.microphone = false + audio.fill(buffer, 99, 2, 44_100, 960) + assertTrue((0 until 960).all { buffer.get(it) == 0.toByte() }) + } + @Test fun `mixing clips instead of wrapping into inverted audio`() { + assertEquals(Short.MAX_VALUE, mixPcm(30_000, 10_000)) + assertEquals(Short.MIN_VALUE, mixPcm(-30_000, -10_000)) + assertEquals(1_234.toShort(), mixPcm(0, 1_234)) + } +} diff --git a/app/src/test/kotlin/dev/forgesworn/kithmoot/session/ChatAttachmentTest.kt b/app/src/test/kotlin/dev/forgesworn/kithmoot/session/ChatAttachmentTest.kt new file mode 100644 index 00000000..ded32fe0 --- /dev/null +++ b/app/src/test/kotlin/dev/forgesworn/kithmoot/session/ChatAttachmentTest.kt @@ -0,0 +1,40 @@ +package dev.forgesworn.kithmoot.session + +import dev.forgesworn.kithmoot.crypto.Digests +import dev.forgesworn.kithmoot.crypto.toHex +import kotlinx.serialization.json.* +import kotlin.test.* + +class ChatAttachmentTest { + private fun fixture(name: String): Pair { + val meta = Json.parseToJsonElement(javaClass.getResource("/attachments/$name.json")!!.readText()).jsonObject + return javaClass.getResourceAsStream("/attachments/$name.enc")!!.readBytes() to ChatAttachment( + "https://files.example/image", meta.getValue("sha256").jsonPrimitive.content, + meta.getValue("key").jsonPrimitive.content) + } + @Test fun `opens real web envelopes including more than one authenticated record`() { + for (name in listOf("web-image", "web-multirecord")) { + val (bytes, attachment) = fixture(name) + val opened = openAttachment(bytes, attachment) + assertEquals("picture.png", opened.name) + assertEquals("image/png", opened.type) + if (name == "web-image") assertEquals("89504e470d0a1a0a", opened.bytes.take(8).toByteArray().toHex()) + else { assertEquals(1_100_000, opened.bytes.size); assertTrue(opened.bytes.all { it == 73.toByte() }) } + } + } + @Test fun `refuses wrong hash wrong key tampered final record and truncation`() { + val (bytes, attachment) = fixture("web-multirecord") + assertFails { openAttachment(bytes, attachment.copy(sha256 = "0".repeat(64))) } + assertFails { openAttachment(bytes, attachment.copy(key = "0".repeat(64))) } + val changed = bytes.copyOf().also { it[it.lastIndex] = (it.last().toInt() xor 1).toByte() } + assertFails { openAttachment(changed, attachment.copy(sha256 = Digests.sha256(changed).toHex())) } + val truncated = bytes.copyOf(bytes.size - 16) + assertFails { openAttachment(truncated, attachment.copy(sha256 = Digests.sha256(truncated).toHex())) } + } + @Test fun `only accepts bounded HTTPS references with complete cryptographic material`() { + val valid = """{"url":"https://example.test/file","sha256":"${"ab".repeat(32)}","key":"${"cd".repeat(32)}"}""" + assertNotNull(parseAttachment(Json.parseToJsonElement(valid))) + assertNull(parseAttachment(Json.parseToJsonElement(valid.replace("https:", "http:")))) + assertNull(parseAttachment(Json.parseToJsonElement(valid.replace("cd".repeat(32), "invalid")))) + } +} diff --git a/app/src/test/kotlin/dev/forgesworn/kithmoot/session/ChatTest.kt b/app/src/test/kotlin/dev/forgesworn/kithmoot/session/ChatTest.kt index eebc2d5b..9ebe4e46 100644 --- a/app/src/test/kotlin/dev/forgesworn/kithmoot/session/ChatTest.kt +++ b/app/src/test/kotlin/dev/forgesworn/kithmoot/session/ChatTest.kt @@ -33,6 +33,19 @@ class ChatTest { sentAt = 100, ) + @Test + fun `authenticated attachment fields survive chat decoding and excess references are refused`() { + val original = message("A picture") + val fields = Json.parseToJsonElement(Nip44.decrypt(original.content, room.roomKey)) as kotlinx.serialization.json.JsonObject + val attachment = Json.parseToJsonElement("""{"url":"https://example.test/file","key":"${"ab".repeat(32)}","sha256":"${"cd".repeat(32)}"}""") + fun wrapped(count: Int) = Events.sign( + secretKey = owner.deviceSecretKey, kind = KIND_CHAT, createdAt = 100, tags = original.tags, + content = Nip44.encrypt(kotlinx.serialization.json.JsonObject(fields + ("attachments" to + kotlinx.serialization.json.JsonArray(List(count) { attachment }))).toString(), room.roomKey)) + assertEquals(1, assertNotNull(decodeChatEvent(wrapped(1), room.roomId, room.roomKey, 200)).attachments.size) + assertNull(decodeChatEvent(wrapped(5), room.roomId, room.roomKey, 200)) + } + @Test fun `a chat line round-trips`() { val decoded = decodeChatEvent(message("Start a room."), room.roomId, room.roomKey, now = 200) diff --git a/app/src/test/kotlin/dev/forgesworn/kithmoot/session/RoomSessionTest.kt b/app/src/test/kotlin/dev/forgesworn/kithmoot/session/RoomSessionTest.kt index 5b1433a3..ecde8046 100644 --- a/app/src/test/kotlin/dev/forgesworn/kithmoot/session/RoomSessionTest.kt +++ b/app/src/test/kotlin/dev/forgesworn/kithmoot/session/RoomSessionTest.kt @@ -29,6 +29,27 @@ import kotlin.test.assertTrue @OptIn(ExperimentalCoroutinesApi::class) class RoomSessionTest { + @Test + fun `rapid listening handover wins within one second and sharing leaves it alone`() = runTest { + val room = Fixtures.room() + val relay = FakeRelay() + val owner = Fixtures.primary(room, 1, 2) + val phone = Fixtures.secondary(room, owner, 30) + val timing = Fixtures.QUIET.copy(announceJitterMs = 0) + val desktop = session(room, owner, relay, timing = timing) + val mobile = session(room, phone, relay, timing = timing) + desktop.join(); mobile.join(); advanceTimeBy(2_000); runCurrent() + mobile.claim(Roles.MONITOR); runCurrent() + assertTrue(mobile.localRoles.value.holdsMonitor) + desktop.setTracks(listOf(TrackRef("screen", Roles.SCREEN))); runCurrent() + assertTrue(mobile.localRoles.value.holdsMonitor) + desktop.claim(Roles.MONITOR); runCurrent() + assertTrue(desktop.localRoles.value.holdsMonitor) + mobile.claim(Roles.MONITOR); runCurrent() + assertTrue(mobile.localRoles.value.holdsMonitor) + assertFalse(desktop.localRoles.value.holdsMonitor) + } + @Test fun `a gated member refuses an unproved roster publisher`() = runTest { val room = Fixtures.room() @@ -324,6 +345,15 @@ class RoomSessionTest { assertEquals(stroke, marks.single().annotation) // Drawing never falls through to ordinary negotiation traffic. assertTrue(signals.isEmpty()) + repeat(480) { index -> + theirs.sendSignal(owner.devicePubkey, SignalBody("annotation", room.roomId, + annotation = stroke.copy(strokeId = "segment-$index"))) + runCurrent() + } + assertEquals(480, marks.size) + theirs.sendSignal(owner.devicePubkey, SignalBody("offer", room.roomId, sdp = "v=0")) + runCurrent() + assertEquals(1, signals.size, "Drawing must not consume the negotiation allowance") } @Test diff --git a/app/src/test/kotlin/dev/forgesworn/kithmoot/ui/room/StrokeSegmentsTest.kt b/app/src/test/kotlin/dev/forgesworn/kithmoot/ui/room/StrokeSegmentsTest.kt new file mode 100644 index 00000000..2e494e94 --- /dev/null +++ b/app/src/test/kotlin/dev/forgesworn/kithmoot/ui/room/StrokeSegmentsTest.kt @@ -0,0 +1,22 @@ +package dev.forgesworn.kithmoot.ui.room + +import dev.forgesworn.kithmoot.protocol.AnnotationPoint +import dev.forgesworn.kithmoot.protocol.ScreenAnnotation +import kotlin.test.* + +class StrokeSegmentsTest { + @Test fun `remote viewer receives connected marks before finger lifts`() { + val sent = mutableListOf() + val stroke = StrokeSegments("screen", sent::add) + stroke.start(AnnotationPoint(0.1, 0.2), 0) + stroke.move(AnnotationPoint(0.2, 0.3), 20) + assertTrue(sent.isEmpty()) + stroke.move(AnnotationPoint(0.3, 0.4), 50) + assertEquals(1, sent.size) + stroke.move(AnnotationPoint(0.4, 0.5), 60) + stroke.finish() + assertEquals(sent[0].points!!.last(), sent[1].points!!.first()) + assertNotEquals(sent[0].strokeId, sent[1].strokeId) + assertEquals(AnnotationPoint(0.4, 0.5), sent[1].points!!.last()) + } +} diff --git a/app/src/test/resources/attachments/README.md b/app/src/test/resources/attachments/README.md new file mode 100644 index 00000000..671764ff --- /dev/null +++ b/app/src/test/resources/attachments/README.md @@ -0,0 +1,6 @@ +Synthetic FSWNENC2 fixtures generated by the current KithMoot TypeScript +`packages/context/src/blossom.ts` encryptEnvelope implementation. Keys in the +JSON files are fixture-only and protect no private data. web-image is a 640x360 +red/blue PNG; web-multirecord is 1,100,000 bytes of decimal 73. Both use random +salts, nonces and padding, as production writers do. The reader tests consume +the captured bytes rather than reimplementing the writer in Kotlin. diff --git a/app/src/test/resources/attachments/web-image.enc b/app/src/test/resources/attachments/web-image.enc new file mode 100644 index 00000000..5b5ad373 Binary files /dev/null and b/app/src/test/resources/attachments/web-image.enc differ diff --git a/app/src/test/resources/attachments/web-image.json b/app/src/test/resources/attachments/web-image.json new file mode 100644 index 00000000..d2eb94f3 --- /dev/null +++ b/app/src/test/resources/attachments/web-image.json @@ -0,0 +1,5 @@ +{ + "key": "597a8369feca4b7d11869e14e80d82aba2b924beee5b0ee5b09b3e4b3e31e428", + "sha256": "5872eded301a6c80f5f11b5cf60917e8e87e36f2e91a48a492806aff29ccc6ce", + "size": 2614 +} diff --git a/app/src/test/resources/attachments/web-multirecord.enc b/app/src/test/resources/attachments/web-multirecord.enc new file mode 100644 index 00000000..6ab82005 Binary files /dev/null and b/app/src/test/resources/attachments/web-multirecord.enc differ diff --git a/app/src/test/resources/attachments/web-multirecord.json b/app/src/test/resources/attachments/web-multirecord.json new file mode 100644 index 00000000..e6579c3d --- /dev/null +++ b/app/src/test/resources/attachments/web-multirecord.json @@ -0,0 +1,5 @@ +{ + "key": "6506c72e7873f2f141949c61fe92edcf2ad623b72c7bcedf5896197f6b5c3035", + "sha256": "c400baa9005d984fc9cbae77350b1b748335885f8533572f18dca10796dceba1", + "size": 1100000 +} diff --git a/docs/android-release.md b/docs/android-release.md index 138e6cb9..35b58d05 100644 --- a/docs/android-release.md +++ b/docs/android-release.md @@ -2,6 +2,39 @@ The public website currently offers production-signed 0.6.7 (30), Android 13 or later. Signing, publication and physical acceptance are recorded separately. +## 0.6.8 sharing and listening candidate + +Version code 31 adds app playback capture during screen sharing, preserves an +existing listening device on automatic joins, makes rapid explicit handovers +win within the same clock second, and sends drawing segments while the finger +is down. Expanded screen views display both local and remote marks. Drawing +has a separate 480-per-20-second sender allowance; the ordinary negotiation +allowance remains 120. Existing web/desktop releases with the old combined +allowance can still stop receiving sustained strokes until updated. + +Native WebRTC has one input stream here. While sharing sound, that stream is +advertised as screen-audio and mixes the microphone only when its independent +control is on and unmuted. Android playback capture excludes KithMoot's UID to +avoid sending room audio back to listeners. Apps can prohibit capture; protected +content and voice-communication playback are not bypassed. Android's own +MediaProjection consent and the foreground notification remain required. + +Incoming encrypted PNG, JPEG, WebP and GIF attachments can be opened, fitted, +viewed at actual size, zoomed and passed explicitly to another image app. +Downloads happen only on a tap, use HTTPS without redirects, verify the complete +ciphertext hash, and authenticate all envelope records before displaying data. +The viewer allows 32 MiB encrypted files and 16 megapixels. SVG, larger files, +attachment uploads and arbitrary floating overlays on other Android apps are +not part of this update. Open in temporarily writes only the selected decrypted +image into a narrowly scoped FileProvider cache; stale exports are removed on +the next export. Closing or retracting the message closes the viewer. + +Local unit, lint and build checks and emulator checks are recorded in +[the sharing evidence](sharing-and-listening-20260919.md). Production signing +still uses the owner's existing key and lineage on M4. This candidate is not +yet published or installed over the Pixel's 0.6.6; the owner is using that phone +for a call. Do not interrupt it for acceptance testing. + ## 0.6.7 rejoin, call membership, rotation and no telemetry Published on 18 September 2026 from `7d153e9`: owner-signed with the production key and lineage (v3 only), APK SHA-256 `a1f4c64b1c296d4eb66bbfdbbfc18206b0d5cc7815e1e92592d64597f4521a5a`, certificate `135bcabf…`. On the website and as GitHub pre-release `v0.6.7`. The signed build was installed on an arm64 emulator and started cleanly with no linkage errors; it has not yet run on a handset. diff --git a/docs/sharing-and-listening-20260919.md b/docs/sharing-and-listening-20260919.md new file mode 100644 index 00000000..1e8f5e1a --- /dev/null +++ b/docs/sharing-and-listening-20260919.md @@ -0,0 +1,37 @@ +# Android sharing and listening, 19 September 2026 + +Candidate: 0.6.8 (31), based on published 0.6.7 (`ddb9b84`). Work lives in an +isolated checkout; the existing rendezvous branch and its edits were preserved. + +## Evidence + +- Real TypeScript FSWNENC2 fixtures open in Kotlin, including a file spanning + multiple authenticated records. Bad hash, key, tag and truncation are refused. +- Microphone-off buffers are overwritten with silence, including unsupported + input configurations; audio sums saturate rather than wrap. +- Rapid paired listening handovers and screen-track publication are tested. +- Connected drawing segments leave before finger-up and retain their final point. +- A disposable Android 35 emulator passed the Compose chat/expanded-video test: + live synthetic frames, fit, zoom, pan, drawing before finger-up, picture in + picture and close. +- Android MediaProjection captured a 440 Hz tone from a separate synthetic app + UID into the WebRTC audio-input callback while the microphone control stayed + off. Stopping screen sharing stopped capture and removed local tracks. +- The image UI fetched a synthetic web-encrypted PNG over HTTPS, verified and + decrypted it, showed the expected two colour panels, fitted and zoomed it, + and closed the viewer when the message was retracted. + +The emulator used a suffixed debug package (`dev.forgesworn.kithmoot.sharingtest`) +so no installed account was touched. Pixel 10 Pro XL was initially visible on +M4 with production 0.6.6 (29), then disconnected; the owner confirmed it is in +use for a call. No phone installation or physical capture acceptance occurred. + +## Remaining release gates + +Use the existing production certificate and lineage, verify the exact APK, +then perform an in-place Pixel update only once the call is finished. Check +speech plus captured playback with a remote listener, microphone mute while +playback continues, repeated start/stop, system revocation, listening-device +handover with web/Mac, full-size images and sustained cross-client drawing. +CI, an unsigned build, emulator capture, signing and physical acceptance are +separate claims. The public download remains 0.6.7 until publication is verified. diff --git a/protocol/src/main/kotlin/dev/forgesworn/kithmoot/protocol/SignalGuard.kt b/protocol/src/main/kotlin/dev/forgesworn/kithmoot/protocol/SignalGuard.kt index 7b855357..2d04e031 100644 --- a/protocol/src/main/kotlin/dev/forgesworn/kithmoot/protocol/SignalGuard.kt +++ b/protocol/src/main/kotlin/dev/forgesworn/kithmoot/protocol/SignalGuard.kt @@ -50,7 +50,7 @@ const val MAX_REMEMBERED_SIGNALS: Int = 4096 */ const val MAX_UNWRAPS_PER_WINDOW: Int = 4096 -class SignalGuard { +class SignalGuard(private val senderLimit: Int = MAX_SIGNALS_PER_WINDOW) { private var unwrapWindow: Window? = null /** Bound anonymous crypto work before the sender can be authenticated. */ @@ -98,7 +98,7 @@ class SignalGuard { senders[sender] = Window(now, 1) return true } - if (window.count >= MAX_SIGNALS_PER_WINDOW) return false + if (window.count >= senderLimit) return false window.count++ return true }