From 503ef9545dbf8b21f0647c675c1511acd90e9eb1 Mon Sep 17 00:00:00 2001 From: "Elie G." Date: Fri, 25 Sep 2026 15:41:22 +0300 Subject: [PATCH 1/4] feat(screen-capture): native, AWT-free screen and window capture (#637) New screen-capture module: ScreenCapture.displays() / captureDisplay(display, region, includeCursor) / captureWindow(id) / permission, images in physical pixels with PNG and BGRA export. - Windows: GDI BitBlt in a per-monitor-v2 thread DPI context; windows through PrintWindow(PW_RENDERFULLCONTENT) on a worker thread with a timeout, since it waits forever on a thread that does not pump (a hung window, or the app's own UI thread blocked on the capture), falling back to the visible part on screen. - macOS: ScreenCaptureKit on 14+, CGDisplayCreateImageForRect via dlsym before. - Linux: X11 (XGetImage, RandR, XFixes cursor, XComposite), all dlopen'ed with X errors trapped per call; on Wayland the xdg-desktop-portal Screenshot interface. E2E: examples/screen-capture-demo self-test (pixel-exact pattern, occluded, minimized, blocked UI thread, hung foreign window, cursor, multi-thread torture) driven by scripts/screen-capture-windows-e2e.ps1 with GDI/handle leak sampling, and scripts/screen-capture-linux-e2e.sh (Xvfb at depths 24/16/8 against xwd, fake portal). --- .github/workflows/build-natives.yaml | 18 +- .github/workflows/pre-merge.yaml | 6 + .github/workflows/publish-maven.yaml | 6 + CLAUDE.md | 1 + README.md | 1 + examples/screen-capture-demo/build.gradle.kts | 42 + .../src/main/kotlin/screencapturedemo/Main.kt | 201 +++ .../main/kotlin/screencapturedemo/SelfTest.kt | 439 ++++++ .../kotlin/screencapturedemo/TargetPattern.kt | 141 ++ screen-capture/api/screen-capture.api | 102 ++ screen-capture/build.gradle.kts | 76 + .../screencapture/CaptureDisplay.kt | 74 + .../screencapture/ScreenCapture.kt | 263 ++++ .../screencapture/ScreenCaptureException.kt | 34 + .../screencapture/ScreenImage.kt | 85 + .../screencapture/internal/NativeCall.kt | 75 + .../internal/NativeScreenCapture.kt | 121 ++ .../screencapture/internal/PngEncoder.kt | 100 ++ screen-capture/src/main/native/linux/build.sh | 57 + .../linux/nucleus_screencapture_linux.c | 1387 +++++++++++++++++ .../main/native/macos/NucleusScreenCapture.m | 583 +++++++ screen-capture/src/main/native/macos/build.sh | 69 + .../src/main/native/windows/build.bat | 123 ++ .../windows/nucleus_screencapture_windows.c | 588 +++++++ .../reachability-metadata.json | 33 + .../screencapture/ScreenCaptureLiveTest.kt | 164 ++ .../screencapture/ScreenImageTest.kt | 124 ++ scripts/screen-capture-linux-e2e.sh | 108 ++ .../screen-capture-linux-e2e/fake_portal.py | 104 ++ .../core/runtime/JniExceptionReporter.java | 11 + .../internal/DisplayCollector.java | 18 + .../screencapture/internal/Harness.java | 507 ++++++ .../internal/NativeScreenCapture.java | 11 + scripts/screen-capture-linux-e2e/xtool.c | 120 ++ scripts/screen-capture-windows-e2e.ps1 | 140 ++ settings.gradle.kts | 2 + 36 files changed, 5933 insertions(+), 1 deletion(-) create mode 100644 examples/screen-capture-demo/build.gradle.kts create mode 100644 examples/screen-capture-demo/src/main/kotlin/screencapturedemo/Main.kt create mode 100644 examples/screen-capture-demo/src/main/kotlin/screencapturedemo/SelfTest.kt create mode 100644 examples/screen-capture-demo/src/main/kotlin/screencapturedemo/TargetPattern.kt create mode 100644 screen-capture/api/screen-capture.api create mode 100644 screen-capture/build.gradle.kts create mode 100644 screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/CaptureDisplay.kt create mode 100644 screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/ScreenCapture.kt create mode 100644 screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/ScreenCaptureException.kt create mode 100644 screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/ScreenImage.kt create mode 100644 screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/internal/NativeCall.kt create mode 100644 screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/internal/NativeScreenCapture.kt create mode 100644 screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/internal/PngEncoder.kt create mode 100644 screen-capture/src/main/native/linux/build.sh create mode 100644 screen-capture/src/main/native/linux/nucleus_screencapture_linux.c create mode 100644 screen-capture/src/main/native/macos/NucleusScreenCapture.m create mode 100755 screen-capture/src/main/native/macos/build.sh create mode 100644 screen-capture/src/main/native/windows/build.bat create mode 100644 screen-capture/src/main/native/windows/nucleus_screencapture_windows.c create mode 100644 screen-capture/src/main/resources/META-INF/native-image/dev.nucleusframework/nucleus.screen-capture/reachability-metadata.json create mode 100644 screen-capture/src/test/kotlin/dev/nucleusframework/screencapture/ScreenCaptureLiveTest.kt create mode 100644 screen-capture/src/test/kotlin/dev/nucleusframework/screencapture/ScreenImageTest.kt create mode 100644 scripts/screen-capture-linux-e2e.sh create mode 100644 scripts/screen-capture-linux-e2e/fake_portal.py create mode 100644 scripts/screen-capture-linux-e2e/harness/dev/nucleusframework/core/runtime/JniExceptionReporter.java create mode 100644 scripts/screen-capture-linux-e2e/harness/dev/nucleusframework/screencapture/internal/DisplayCollector.java create mode 100644 scripts/screen-capture-linux-e2e/harness/dev/nucleusframework/screencapture/internal/Harness.java create mode 100644 scripts/screen-capture-linux-e2e/harness/dev/nucleusframework/screencapture/internal/NativeScreenCapture.java create mode 100644 scripts/screen-capture-linux-e2e/xtool.c create mode 100644 scripts/screen-capture-windows-e2e.ps1 diff --git a/.github/workflows/build-natives.yaml b/.github/workflows/build-natives.yaml index 02f288f2d..203ccf4a0 100644 --- a/.github/workflows/build-natives.yaml +++ b/.github/workflows/build-natives.yaml @@ -53,6 +53,11 @@ jobs: shell: cmd run: call system-color\src\main\native\windows\build.bat + - name: Build screen-capture Windows DLLs + if: steps.natives-cache.outputs.cache-hit != 'true' + shell: cmd + run: call screen-capture\src\main\native\windows\build.bat + - name: Build energy-manager Windows DLLs if: steps.natives-cache.outputs.cache-hit != 'true' shell: cmd @@ -132,6 +137,7 @@ jobs: "darkmode-detector/nucleus_windows_theme.dll" "native-ssl/nucleus_ssl.dll" "system-color/nucleus_systemcolor.dll" + "screen-capture/nucleus_screencapture.dll" "energy-manager/nucleus_energy_manager.dll" "taskbar-progress/nucleus_taskbar_progress.dll" "notification-windows/nucleus_notification_windows.dll" @@ -204,6 +210,10 @@ jobs: if: steps.natives-cache.outputs.cache-hit != 'true' run: bash system-color/src/main/native/macos/build.sh + - name: Build screen-capture macOS dylibs + if: steps.natives-cache.outputs.cache-hit != 'true' + run: bash screen-capture/src/main/native/macos/build.sh + - name: Build energy-manager macOS dylibs if: steps.natives-cache.outputs.cache-hit != 'true' run: bash energy-manager/src/main/native/macos/build.sh @@ -277,6 +287,7 @@ jobs: "darkmode-detector/libnucleus_darkmode.dylib" "native-ssl/libnucleus_ssl.dylib" "system-color/libnucleus_systemcolor.dylib" + "screen-capture/libnucleus_screencapture.dylib" "energy-manager/libnucleus_energy_manager.dylib" "taskbar-progress/libnucleus_taskbar_progress.dylib" "notification-macos/libnucleus_notification.dylib" @@ -358,7 +369,7 @@ jobs: - name: Install build dependencies if: steps.natives-cache.outputs.cache-hit != 'true' - run: sudo apt-get update && sudo apt-get install -y libdbus-1-dev libx11-dev libglib2.0-dev libgtk-3-dev + run: sudo apt-get update && sudo apt-get install -y libdbus-1-dev libx11-dev libxrandr-dev libxfixes-dev libxcomposite-dev libglib2.0-dev libgtk-3-dev - name: Build darkmode-detector Linux native shared library if: steps.natives-cache.outputs.cache-hit != 'true' @@ -376,6 +387,10 @@ jobs: if: steps.natives-cache.outputs.cache-hit != 'true' run: bash system-color/src/main/native/linux/build.sh + - name: Build screen-capture Linux native shared library + if: steps.natives-cache.outputs.cache-hit != 'true' + run: bash screen-capture/src/main/native/linux/build.sh + - name: Build energy-manager Linux native shared library if: steps.natives-cache.outputs.cache-hit != 'true' run: bash energy-manager/src/main/native/linux/build.sh @@ -434,6 +449,7 @@ jobs: "linux-hidpi/libnucleus_linux_hidpi_jni.so" "spellcheck/libnucleus_spellcheck.so" "system-color/libnucleus_systemcolor.so" + "screen-capture/libnucleus_screencapture.so" "energy-manager/libnucleus_energy_manager.so" "notification-linux/libnucleus_notification_linux.so" "launcher-linux/libnucleus_launcher_linux.so" diff --git a/.github/workflows/pre-merge.yaml b/.github/workflows/pre-merge.yaml index 008507af7..d0d4177fa 100644 --- a/.github/workflows/pre-merge.yaml +++ b/.github/workflows/pre-merge.yaml @@ -68,6 +68,12 @@ jobs: "system-color/src/main/resources/nucleus/native/darwin-x64/libnucleus_systemcolor.dylib" "system-color/src/main/resources/nucleus/native/win32-x64/nucleus_systemcolor.dll" "system-color/src/main/resources/nucleus/native/win32-aarch64/nucleus_systemcolor.dll" + "screen-capture/src/main/resources/nucleus/native/linux-x64/libnucleus_screencapture.so" + "screen-capture/src/main/resources/nucleus/native/linux-aarch64/libnucleus_screencapture.so" + "screen-capture/src/main/resources/nucleus/native/darwin-aarch64/libnucleus_screencapture.dylib" + "screen-capture/src/main/resources/nucleus/native/darwin-x64/libnucleus_screencapture.dylib" + "screen-capture/src/main/resources/nucleus/native/win32-x64/nucleus_screencapture.dll" + "screen-capture/src/main/resources/nucleus/native/win32-aarch64/nucleus_screencapture.dll" "energy-manager/src/main/resources/nucleus/native/win32-x64/nucleus_energy_manager.dll" "energy-manager/src/main/resources/nucleus/native/win32-aarch64/nucleus_energy_manager.dll" "energy-manager/src/main/resources/nucleus/native/darwin-aarch64/libnucleus_energy_manager.dylib" diff --git a/.github/workflows/publish-maven.yaml b/.github/workflows/publish-maven.yaml index 85d3f6c31..d5979a774 100644 --- a/.github/workflows/publish-maven.yaml +++ b/.github/workflows/publish-maven.yaml @@ -70,6 +70,12 @@ jobs: "system-color/src/main/resources/nucleus/native/darwin-x64/libnucleus_systemcolor.dylib" "system-color/src/main/resources/nucleus/native/win32-x64/nucleus_systemcolor.dll" "system-color/src/main/resources/nucleus/native/win32-aarch64/nucleus_systemcolor.dll" + "screen-capture/src/main/resources/nucleus/native/linux-x64/libnucleus_screencapture.so" + "screen-capture/src/main/resources/nucleus/native/linux-aarch64/libnucleus_screencapture.so" + "screen-capture/src/main/resources/nucleus/native/darwin-aarch64/libnucleus_screencapture.dylib" + "screen-capture/src/main/resources/nucleus/native/darwin-x64/libnucleus_screencapture.dylib" + "screen-capture/src/main/resources/nucleus/native/win32-x64/nucleus_screencapture.dll" + "screen-capture/src/main/resources/nucleus/native/win32-aarch64/nucleus_screencapture.dll" "energy-manager/src/main/resources/nucleus/native/win32-x64/nucleus_energy_manager.dll" "energy-manager/src/main/resources/nucleus/native/win32-aarch64/nucleus_energy_manager.dll" "energy-manager/src/main/resources/nucleus/native/darwin-aarch64/libnucleus_energy_manager.dylib" diff --git a/CLAUDE.md b/CLAUDE.md index eb560a4a2..02cce898f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -27,6 +27,7 @@ Published releases are `2.5.x` (latest tag `v2.5.0`). Do not treat `IDEAL_API.md - `darkmode-detector` - Reactive OS dark mode detection via JNI - `system-color` - Reactive system accent color and high contrast detection via JNI - `system-info` - CPU, memory, GPU, temperature, network, processes +- `screen-capture` - AWT-free screen capture in physical pixels: `ScreenCapture.displays()` / `captureDisplay(display, region, includeCursor)` / `captureWindow(id)` / permission. Windows: GDI `BitBlt` in a per-monitor-v2 thread DPI context; windows through `PrintWindow(PW_RENDERFULLCONTENT)` on a **worker thread with a 2 s timeout** (it sends `WM_PRINT` and waits forever on a thread that does not pump — including an app's own UI thread blocked on the capturing thread), falling back to the visible part from the screen; the window's own thread prints inline. macOS: ScreenCaptureKit (14+) / `CGDisplayCreateImageForRect` via `dlsym` (obsoleted in the 15 SDK), never pumps the main run loop. Linux: X11 (`XGetImage`, RandR monitors, XFixes cursor, XComposite for covered windows; everything `dlopen`ed, X errors trapped per call) or, on Wayland, `org.freedesktop.portal.Screenshot` (whole desktop, cropped; the portal file is deleted). E2E: `scripts/screen-capture-windows-e2e.ps1` (`examples/screen-capture-demo` self-test: pixel-exact pattern, occluded / minimized / blocked-UI / hung-foreign windows, torture + GDI/handle leak sampling) and `scripts/screen-capture-linux-e2e.sh` (Xvfb depths 24/16/8 vs `xwd`, fake portal) - `energy-manager` - Energy efficiency & screen-awake APIs - `autolaunch` - Start at login (Win32/MSIX/SMAppService/systemd/Flatpak portal) - `scheduler` / `scheduler-testing` - OS-scheduled background tasks (Task Scheduler / launchd / systemd) + test doubles diff --git a/README.md b/README.md index 30aa7b36c..4e45696a8 100644 --- a/README.md +++ b/README.md @@ -220,6 +220,7 @@ Each module is published independently to Maven Central — use them together or | `nucleus.darkmode-detector` | Reactive OS dark mode detection | | `nucleus.system-color` | Reactive accent color & high contrast detection | | `nucleus.system-info` | CPU, memory, GPU (NVIDIA/AMD/Intel), temperature, network, processes | +| `nucleus.screen-capture` | Native screen & window capture in physical pixels, no AWT (GDI / ScreenCaptureKit / X11 / xdg-desktop-portal) | | `nucleus.decorated-window-tao` | Windowing backend (Rust `tao`, no AWT) | | `nucleus.decorated-window-core` | Shared window types, layout, chrome (design-system agnostic) | | `nucleus.decorated-window-jewel` | Jewel (IntelliJ theme) integration | diff --git a/examples/screen-capture-demo/build.gradle.kts b/examples/screen-capture-demo/build.gradle.kts new file mode 100644 index 000000000..a4a7f068c --- /dev/null +++ b/examples/screen-capture-demo/build.gradle.kts @@ -0,0 +1,42 @@ +import org.jetbrains.kotlin.gradle.dsl.JvmTarget + +// Demo and E2E fixture of screen-capture: lists the displays, captures a display, a region or +// the demo's own window, and previews the result. With SCREEN_CAPTURE_DEMO_SELFTEST=1 it draws +// a pixel-exact target pattern and checks every capture path against it, then exits with the +// number of failed checks (scripts/screen-capture-windows-e2e.ps1). + +plugins { + kotlin("jvm") + alias(libs.plugins.kotlinComposePlugin) + alias(libs.plugins.jetbrainsCompose) + id("dev.nucleusframework") +} + +dependencies { + implementation(compose.desktop.currentOs) + implementation(project(":core-runtime")) + implementation(project(":screen-capture")) + implementation(project(":decorated-window-tao")) + implementation(project(":nucleus-application")) + implementation(libs.coroutines.core) +} + +java { + sourceCompatibility = JavaVersion.VERSION_17 + targetCompatibility = JavaVersion.VERSION_17 +} + +kotlin { + compilerOptions { + jvmTarget.set(JvmTarget.JVM_17) + } +} + +nucleus.application { + mainClass = "screencapturedemo.MainKt" + + nativeDistributions { + packageName = "ScreenCaptureDemo" + packageVersion = "1.0.0" + } +} diff --git a/examples/screen-capture-demo/src/main/kotlin/screencapturedemo/Main.kt b/examples/screen-capture-demo/src/main/kotlin/screencapturedemo/Main.kt new file mode 100644 index 000000000..105ec439f --- /dev/null +++ b/examples/screen-capture-demo/src/main/kotlin/screencapturedemo/Main.kt @@ -0,0 +1,201 @@ +package screencapturedemo + +import androidx.compose.foundation.Image +import androidx.compose.foundation.background +import androidx.compose.foundation.border +import androidx.compose.foundation.clickable +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.FlowRow +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.text.BasicText +import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.rememberCoroutineScope +import androidx.compose.runtime.setValue +import androidx.compose.ui.Modifier +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.graphics.ImageBitmap +import androidx.compose.ui.graphics.toComposeImageBitmap +import androidx.compose.ui.layout.ContentScale +import androidx.compose.ui.text.TextStyle +import androidx.compose.ui.unit.DpSize +import androidx.compose.ui.unit.dp +import androidx.compose.ui.unit.sp +import androidx.compose.ui.window.WindowPosition +import androidx.compose.ui.window.rememberWindowState +import dev.nucleusframework.application.DecoratedWindow +import dev.nucleusframework.application.nucleusApplication +import dev.nucleusframework.core.runtime.Platform +import dev.nucleusframework.screencapture.CaptureRegion +import dev.nucleusframework.screencapture.ScreenCapture +import dev.nucleusframework.screencapture.ScreenCaptureException +import dev.nucleusframework.screencapture.ScreenImage +import dev.nucleusframework.window.NucleusDecoratedWindowTheme +import dev.nucleusframework.window.TitleBar +import dev.nucleusframework.window.tao.LocalTaoWindow +import dev.nucleusframework.window.tao.TaoWindow +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext +import org.jetbrains.skia.ColorAlphaType +import org.jetbrains.skia.ColorType +import org.jetbrains.skia.ImageInfo +import java.io.File +import kotlin.system.exitProcess + +private val selfTest = System.getenv("SCREEN_CAPTURE_DEMO_SELFTEST") == "1" + +fun main(args: Array) = + nucleusApplication(args, enableSingleInstance = false) { + var coverVisible by remember { mutableStateOf(false) } + NucleusDecoratedWindowTheme(isDark = true) { + DecoratedWindow( + onCloseRequest = ::exitApplication, + title = "Screen Capture Demo", + // Self-test: nothing else may cover the pattern (the cover window, shown later, goes above). + alwaysOnTop = selfTest, + state = rememberWindowState(size = DpSize(760.dp, 620.dp), position = WindowPosition(120.dp, 120.dp)), + ) { + TitleBar { BasicText("Screen Capture Demo", style = TextStyle(color = Color.White)) } + val window = LocalTaoWindow.current + if (window != null && selfTest) { + LaunchedEffect(window) { + val test = + SelfTest( + window = window, + windowId = windowIdOf(window), + showCover = { visible -> withContext(Dispatchers.Main) { coverVisible = visible } }, + setMinimized = { minimized -> + withContext(Dispatchers.Main) { window.setMinimized(minimized) } + }, + ) + val failures = withContext(Dispatchers.IO) { test.run() } + exitProcess(failures.coerceAtMost(100)) + } + } + DemoContent(window) + } + if (coverVisible) { + // Covers the demo window entirely, above it, for the occluded-window check. + DecoratedWindow( + onCloseRequest = { coverVisible = false }, + title = "Cover", + alwaysOnTop = true, + focusable = false, + state = rememberWindowState(size = DpSize(860.dp, 720.dp), position = WindowPosition(70.dp, 70.dp)), + ) { + Box(Modifier.fillMaxSize().background(Color(0xFFFF00FF))) + } + } + } + } + +/** The id [ScreenCapture.captureWindow] takes for a Tao window; `0` where the demo has none. */ +private fun windowIdOf(window: TaoWindow): Long = + when (Platform.Current) { + Platform.Windows -> window.nativeHandle + Platform.Linux -> window.x11WindowId ?: 0L + else -> 0L + } + +@Composable +private fun DemoContent(window: TaoWindow?) { + val scope = rememberCoroutineScope() + var preview by remember { mutableStateOf(null) } + var last by remember { mutableStateOf(null) } + var status by remember { + mutableStateOf("backend=${ScreenCapture.backend} permission=${ScreenCapture.permissionStatus()}") + } + val displays = remember { runCatching { ScreenCapture.displays() }.getOrDefault(emptyList()) } + + fun capture( + label: String, + block: () -> ScreenImage, + ) { + scope.launch { + status = "Capturing $label…" + val started = System.nanoTime() + val outcome = withContext(Dispatchers.IO) { runCatching(block) } + val ms = (System.nanoTime() - started) / 1_000_000 + outcome + .onSuccess { + last = it + preview = it.toImageBitmap() + status = "$label: ${it.width}x${it.height} in ${ms}ms" + }.onFailure { + val reason = if (it is ScreenCaptureException) "${it.failure} — ${it.message}" else "$it" + status = "$label failed: $reason" + } + } + } + + Column( + Modifier.fillMaxSize().background(Color(0xFF202124)).padding(12.dp), + verticalArrangement = Arrangement.spacedBy(8.dp), + ) { + TargetPattern.Content() + FlowRow(horizontalArrangement = Arrangement.spacedBy(8.dp), verticalArrangement = Arrangement.spacedBy(8.dp)) { + for (display in displays) { + DemoButton("${display.name} (${display.widthPx}x${display.heightPx})") { + capture(display.name) { ScreenCapture.captureDisplay(display, includeCursor = true) } + } + } + displays.firstOrNull()?.let { primary -> + DemoButton("Region 400x300") { + capture("region") { ScreenCapture.captureDisplay(primary, CaptureRegion(0, 0, 400, 300)) } + } + } + val id = window?.let(::windowIdOf) ?: 0L + if (id != 0L && ScreenCapture.isWindowCaptureSupported) { + DemoButton("This window") { capture("window") { ScreenCapture.captureWindow(id) } } + } + last?.let { image -> + DemoButton("Save PNG") { + scope.launch { + val file = + File( + System.getProperty("java.io.tmpdir"), + "screen-capture-${System.currentTimeMillis()}.png", + ) + withContext(Dispatchers.IO) { file.writeBytes(image.toPng()) } + status = "Saved ${file.absolutePath}" + } + } + } + } + BasicText(status, style = TextStyle(color = Color.White, fontSize = 13.sp)) + Box(Modifier.fillMaxWidth().weight(1f).border(1.dp, Color.Gray)) { + preview?.let { + Image(it, contentDescription = null, modifier = Modifier.fillMaxSize(), contentScale = ContentScale.Fit) + } + } + } +} + +@Composable +private fun DemoButton( + label: String, + onClick: () -> Unit, +) { + Box( + Modifier + .background( + Color(0xFF3C4043), + ).clickable(onClick = onClick) + .padding(horizontal = 12.dp, vertical = 6.dp), + ) { + BasicText(label, style = TextStyle(color = Color.White, fontSize = 13.sp)) + } +} + +private fun ScreenImage.toImageBitmap(): ImageBitmap = + org.jetbrains.skia.Image + .makeRaster(ImageInfo(width, height, ColorType.BGRA_8888, ColorAlphaType.OPAQUE), toBgraBytes(), width * 4) + .toComposeImageBitmap() diff --git a/examples/screen-capture-demo/src/main/kotlin/screencapturedemo/SelfTest.kt b/examples/screen-capture-demo/src/main/kotlin/screencapturedemo/SelfTest.kt new file mode 100644 index 000000000..34d71ae7d --- /dev/null +++ b/examples/screen-capture-demo/src/main/kotlin/screencapturedemo/SelfTest.kt @@ -0,0 +1,439 @@ +package screencapturedemo + +import dev.nucleusframework.core.runtime.Platform +import dev.nucleusframework.screencapture.CaptureDisplay +import dev.nucleusframework.screencapture.CaptureFailure +import dev.nucleusframework.screencapture.CaptureRegion +import dev.nucleusframework.screencapture.ScreenCapture +import dev.nucleusframework.screencapture.ScreenCaptureException +import dev.nucleusframework.screencapture.ScreenImage +import dev.nucleusframework.window.tao.TaoMonitors +import dev.nucleusframework.window.tao.TaoWindow +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.delay +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withContext +import java.io.File +import java.time.LocalTime +import java.util.concurrent.ConcurrentLinkedQueue +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicInteger +import kotlin.random.Random +import kotlin.system.measureTimeMillis + +/** + * The E2E checks. Each one logs `PASS` / `FAIL` lines to stdout and to [logFile]; the process + * exits with the number of failures. + */ +internal class SelfTest( + private val window: TaoWindow, + private val windowId: Long, + private val showCover: suspend (Boolean) -> Unit, + private val setMinimized: suspend (Boolean) -> Unit, +) { + private val logFile = + File( + System.getenv("SCREEN_CAPTURE_DEMO_LOG") + ?: File(System.getProperty("java.io.tmpdir"), "screen-capture-demo.log").path, + ) + private var failures = 0 + private val tortureIterations = System.getenv("SCREEN_CAPTURE_DEMO_TORTURE")?.toIntOrNull() ?: 800 + + fun log(line: String) { + val text = "${LocalTime.now()} $line" + println(text) + runCatching { logFile.appendText("$text\n") } + } + + private fun check( + name: String, + condition: Boolean, + detail: () -> String = { "" }, + ) { + if (condition) { + log("PASS $name ${detail()}") + } else { + failures++ + log("FAIL $name ${detail()}") + } + } + + private inline fun step( + name: String, + block: () -> Unit, + ) { + try { + block() + } catch (e: Throwable) { + failures++ + log("FAIL $name threw ${e::class.simpleName}: ${e.message}") + e.printStackTrace() + } + } + + suspend fun run(): Int { + logFile.delete() + log( + "START pid=${ProcessHandle.current().pid()} backend=${ScreenCapture.backend} os=${Platform.Current} windowId=$windowId", + ) + delay(1500) // first frames, compositor settle + val displays = ScreenCapture.displays() + displays.forEach { log("DISPLAY $it") } + + step("displays match TaoMonitors") { checkDisplays(displays) } + step("window capture") { checkWindow() } + step("display capture") { checkDisplayCapture(displays) } + step("own UI thread") { checkOnUiThread() } + step("blocked UI thread") { checkBlockedUiThread() } + step("occluded window") { checkOccluded(displays) } + step("minimized window") { checkMinimized() } + step("cursor") { checkCursor(displays) } + step("hung foreign window") { checkHungWindow() } + step("errors") { checkErrors(displays) } + log("PHASE torture-begin") + delay(2000) + step("torture") { torture(displays) } + System.gc() + log("PHASE torture-end heapMb=${usedHeapMb()}") + delay(3000) + log("DONE failures=$failures") + return failures + } + + private fun checkDisplays(displays: List) { + if (Platform.Current != Platform.Windows) return + val monitors = TaoMonitors.all(window) + check("display count", displays.size == monitors.size) { "${displays.size} vs ${monitors.size}" } + for (m in monitors) { + val d = displays.firstOrNull { it.id.equals(m.id, ignoreCase = true) } + check("display ${m.id} known", d != null) + if (d == null) continue + val b = d.bounds!! + check( + "display ${m.id} bounds", + b.x == m.boundsPx.left && + b.y == m.boundsPx.top && + b.width == m.boundsPx.width && + b.height == m.boundsPx.height, + ) { "$b vs ${m.boundsPx}" } + check("display ${m.id} scale", kotlin.math.abs(d.scaleFactor - m.scaleFactor) < 0.01f) { + "${d.scaleFactor} vs ${m.scaleFactor}" + } + check("display ${m.id} primary", d.isPrimary == m.isPrimary) + } + } + + private fun expectPattern( + name: String, + image: ScreenImage, + ): Pair? { + val found = TargetPattern.find(image) + if (found.size != 1) saveSample("fail-" + name.replace(' ', '-'), image) + check("$name finds one pattern", found.size == 1) { "found=$found in $image" } + val origin = found.singleOrNull() ?: return null + val (count, samples) = TargetPattern.mismatches(image, origin.first, origin.second) + check("$name pixel-exact", count == 0) { "mismatches=$count $samples at $origin" } + return origin + } + + private fun checkWindow() { + val image: ScreenImage + val ms = measureTimeMillis { image = ScreenCapture.captureWindow(windowId) } + log("INFO window capture ${image.width}x${image.height} in ${ms}ms") + expectPattern("window", image) + saveSample("window", image) + } + + private fun displayUnderWindow(displays: List): CaptureDisplay = + TaoMonitors.forWindow(window)?.let { m -> displays.firstOrNull { it.id.equals(m.id, ignoreCase = true) } } + ?: displays.first() + + private fun checkDisplayCapture(displays: List) { + val display = displayUnderWindow(displays) + val times = LongArray(10) + var full: ScreenImage? = null + for (i in times.indices) times[i] = measureTimeMillis { full = ScreenCapture.captureDisplay(display) } + val image = full!! + log( + "INFO display ${display.id} ${image.width}x${image.height} median=${times.sorted()[5]}ms max=${times.max()}ms", + ) + check( + "display size", + image.width == display.widthPx && image.height == display.heightPx, + ) { "$image vs $display" } + saveSample("display", image) + val origin = expectPattern("display", image) ?: return + + // A region exactly over the pattern must be the pattern, at the region's origin. + val region = CaptureRegion(origin.first, origin.second, TargetPattern.WIDTH, TargetPattern.HEIGHT) + val part = ScreenCapture.captureDisplay(display, region) + check("region size", part.width == region.width && part.height == region.height) { "$part" } + check("region at origin", TargetPattern.find(part) == listOf(0 to 0)) { "${TargetPattern.find(part)}" } + check("region pixel-exact", TargetPattern.mismatches(part, 0, 0).first == 0) { + "${TargetPattern.mismatches(part, 0, 0)}" + } + + // Regions shifted by every sub-offset still line up to the pixel. + var misaligned = 0 + for (dx in -3..3) { + for (dy in -3..3) { + val shifted = + CaptureRegion(origin.first + dx, origin.second + dy, TargetPattern.WIDTH, TargetPattern.HEIGHT) + val shot = ScreenCapture.captureDisplay(display, shifted) + val expected = (-dx) to (-dy) + if (dx > 0 || dy > 0) continue // the marker is cut; checked through the full image below + if (TargetPattern.find(shot) != listOf(expected)) misaligned++ + } + } + check("shifted regions aligned", misaligned == 0) { "misaligned=$misaligned" } + } + + private suspend fun checkOnUiThread() { + val image = withContext(Dispatchers.Main) { ScreenCapture.captureWindow(windowId) } + expectPattern("window from its own UI thread", image) + val shot = withContext(Dispatchers.Main) { ScreenCapture.captureDisplay(ScreenCapture.primaryDisplay()!!) } + check("display from UI thread", shot.width > 0) + } + + /** The UI thread waits on the capturing thread: PrintWindow's WM_PRINT can never be answered. */ + private suspend fun checkBlockedUiThread() { + var image: ScreenImage? = null + val ms = + withContext(Dispatchers.Main) { + measureTimeMillis { + image = runBlocking(Dispatchers.IO) { ScreenCapture.captureWindow(windowId) } + } + } + check("blocked UI thread returns", ms < 5000) { "${ms}ms" } + image?.let { expectPattern("window while its UI thread is blocked", it) } + // The abandoned PrintWindow completes once the UI thread pumps again; the next capture is normal. + delay(300) + val next: ScreenImage + val nextMs = measureTimeMillis { next = ScreenCapture.captureWindow(windowId) } + check("capture after the blocked one is fast", nextMs < 1000) { "${nextMs}ms" } + expectPattern("window after the blocked one", next) + } + + private suspend fun checkOccluded(displays: List) { + showCover(true) + delay(1200) + try { + val display = displayUnderWindow(displays) + val screen = ScreenCapture.captureDisplay(display) + check( + "cover hides the pattern on screen", + TargetPattern.find(screen).isEmpty(), + ) { "${TargetPattern.find(screen)}" } + val image = ScreenCapture.captureWindow(windowId) + if (Platform.Current == Platform.Windows) { + expectPattern("occluded window", image) + } else { + log("INFO occluded window found=${TargetPattern.find(image)}") + } + saveSample("occluded", image) + } finally { + showCover(false) + delay(800) + } + } + + private suspend fun checkMinimized() { + setMinimized(true) + delay(1000) + try { + val error = runCatching { ScreenCapture.captureWindow(windowId) }.exceptionOrNull() + check( + "minimized window is not capturable", + (error as? ScreenCaptureException)?.failure == CaptureFailure.WindowNotFound, + ) { + "$error" + } + } finally { + setMinimized(false) + delay(1200) + } + expectPattern("restored window", ScreenCapture.captureWindow(windowId)) + } + + /** + * Wherever the cursor is: of three captures (without, with, without) the cursor is what only + * the middle one has � pixels the two others agree on. Animated content can do that once, so + * only pixels doing it in every one of three trials count. They must be there and cursor-sized. + */ + private fun checkCursor(displays: List) { + var bbox: IntArray? = null + for (display in displays) { + var mask: BooleanArray? = null + repeat(3) { + val before = ScreenCapture.captureDisplay(display, includeCursor = false).toArgbArray() + val with = ScreenCapture.captureDisplay(display, includeCursor = true).toArgbArray() + val after = ScreenCapture.captureDisplay(display, includeCursor = false).toArgbArray() + val trial = BooleanArray(with.size) { i -> before[i] == after[i] && with[i] != before[i] } + mask = mask?.let { m -> BooleanArray(m.size) { i -> m[i] && trial[i] } } ?: trial + } + val box = intArrayOf(Int.MAX_VALUE, Int.MAX_VALUE, -1, -1) + mask!!.forEachIndexed { i, set -> + if (set) { + val x = i % display.widthPx + val y = i / display.widthPx + box[0] = minOf(box[0], x) + box[1] = minOf(box[1], y) + box[2] = maxOf(box[2], x) + box[3] = maxOf(box[3], y) + } + } + if (box[2] >= 0) { + check("cursor on one display only", bbox == null) + bbox = box + log("INFO cursor on ${display.id} bbox=(${box[0]},${box[1]})-(${box[2]},${box[3]})") + } + } + val box = bbox + if (System.getenv("SCREEN_CAPTURE_DEMO_CURSOR_EXPECTED") == "1") check("cursor drawn", box != null) + if (box != null) check("cursor sized", box[2] - box[0] < 128 && box[3] - box[1] < 128) { box.joinToString() } + } + + private fun checkHungWindow() { + val hung = + System.getenv("SCREEN_CAPTURE_DEMO_HUNG_HWND")?.toLongOrNull() + ?: return log("SKIP hung window: not provided") + repeat(3) { attempt -> + var outcome = "" + val ms = + measureTimeMillis { + outcome = + try { + ScreenCapture.captureWindow(hung).toString() + } catch (e: ScreenCaptureException) { + e.failure.name + } + } + check("hung window capture #$attempt returns in time", ms < 4000) { "${ms}ms $outcome" } + } + } + + private fun checkErrors(displays: List) { + val display = displays.first() + + fun failureOf(block: () -> Unit) = (runCatching(block).exceptionOrNull() as? ScreenCaptureException)?.failure + check( + "region outside", + failureOf { ScreenCapture.captureDisplay(display, CaptureRegion(-500, -500, 10, 10)) } == + CaptureFailure.InvalidRegion, + ) + check("window 0", failureOf { ScreenCapture.captureWindow(0) } == CaptureFailure.WindowNotFound) + } + + private fun torture(displays: List) { + val threads = 8 + val pool = Executors.newFixedThreadPool(threads + 3) + val errors = ConcurrentLinkedQueue() + val captures = AtomicInteger() + val invalid = AtomicInteger() + val pixels = + java.util.concurrent.atomic + .AtomicLong() + val start = System.nanoTime() + repeat(threads) { worker -> + pool.execute { + val random = Random(worker * 7919) + repeat(tortureIterations) { + val display = displays[random.nextInt(displays.size)] + val region = + when (random.nextInt(6)) { + 0 -> null + 1 -> + CaptureRegion( + random.nextInt(-5000, 5000), + random.nextInt(-5000, 5000), + random.nextInt(1, 10_000), + random.nextInt(1, 10_000), + ) + 2 -> + CaptureRegion( + random.nextInt(Int.MIN_VALUE, Int.MAX_VALUE), + random.nextInt(Int.MIN_VALUE, Int.MAX_VALUE), + random.nextInt(1, Int.MAX_VALUE), + random.nextInt(1, Int.MAX_VALUE), + ) + 3 -> CaptureRegion(display.widthPx - 1, display.heightPx - 1, 1, 1) + else -> + CaptureRegion( + random.nextInt(0, display.widthPx), + random.nextInt(0, display.heightPx), + random.nextInt(1, 300), + random.nextInt(1, 300), + ) + } + try { + val image = ScreenCapture.captureDisplay(display, region, includeCursor = random.nextBoolean()) + if (region != null && + (image.width > region.width || image.height > region.height) + ) { + errors += "$region -> $image" + } + pixels.addAndGet(image.width.toLong() * image.height) + captures.incrementAndGet() + } catch (e: ScreenCaptureException) { + if (e.failure == + CaptureFailure.InvalidRegion + ) { + invalid.incrementAndGet() + } else { + errors += + "${e.failure} ${e.message}" + } + } catch (e: Throwable) { + errors += "${e::class.simpleName} ${e.message}" + } + } + } + } + // Window captures of our own window, concurrently with the display ones. + repeat(2) { worker -> + pool.execute { + repeat(tortureIterations / 4) { + try { + val image = ScreenCapture.captureWindow(windowId, includeCursor = it % 2 == 0) + val found = TargetPattern.find(image) + if (found.size != 1) { + if (errors.isEmpty()) saveSample("fail-torture-window", image) + errors += "window capture found $found" + } + captures.incrementAndGet() + } catch (e: Throwable) { + errors += "window ${e::class.simpleName} ${e.message}" + } + } + } + } + // Garbage ids and display enumeration. + pool.execute { + val random = Random(42) + repeat(tortureIterations / 2) { + runCatching { ScreenCapture.captureWindow(random.nextLong() or 0x7000_0000_0000_0000L) } + if (ScreenCapture.displays().size != displays.size) errors += "display count changed" + } + } + pool.shutdown() + val finished = pool.awaitTermination(10, TimeUnit.MINUTES) + val seconds = (System.nanoTime() - start) / 1e9 + check("torture finished", finished) + check("torture errors", errors.isEmpty()) { "${errors.size} ${errors.take(5)}" } + log( + "INFO torture captures=${captures.get()} invalid=${invalid.get()} megapixels=${pixels.get() / 1_000_000} " + + "seconds=${"%.1f".format(seconds)}", + ) + } + + private fun saveSample( + name: String, + image: ScreenImage, + ) { + val dir = System.getenv("SCREEN_CAPTURE_DEMO_OUT") ?: return + File(dir, "$name.png").apply { parentFile.mkdirs() }.writeBytes(image.toPng()) + } + + private fun usedHeapMb(): Long = Runtime.getRuntime().let { (it.totalMemory() - it.freeMemory()) / (1024 * 1024) } +} diff --git a/examples/screen-capture-demo/src/main/kotlin/screencapturedemo/TargetPattern.kt b/examples/screen-capture-demo/src/main/kotlin/screencapturedemo/TargetPattern.kt new file mode 100644 index 000000000..3f69a6e9c --- /dev/null +++ b/examples/screen-capture-demo/src/main/kotlin/screencapturedemo/TargetPattern.kt @@ -0,0 +1,141 @@ +package screencapturedemo + +import androidx.compose.foundation.Canvas +import androidx.compose.foundation.layout.size +import androidx.compose.runtime.Composable +import androidx.compose.ui.Modifier +import androidx.compose.ui.geometry.Offset +import androidx.compose.ui.geometry.Size +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.platform.LocalDensity +import dev.nucleusframework.screencapture.ScreenImage + +/** + * A pattern laid out in physical pixels, so a capture can be checked pixel for pixel whatever + * the display scale: a marker block that locates it, a grid of distinct colours, a 1-pixel + * checkerboard (any resampling blurs it) and a ramp through every channel value. + */ +internal object TargetPattern { + const val WIDTH = 256 + const val HEIGHT = 160 + const val MARKER = 0xFF123456.toInt() + private const val MARKER_SIZE = 24 + private const val CELL = 24 + private const val GRID_TOP = 32 + private const val COLUMNS = 8 + private const val ROWS = 4 + private const val CHECKER_TOP = 136 + private const val RAMP_TOP = 148 + private const val BAND = 8 + + /** The expected pixel at ([x], [y]) of the pattern; `null` for the unpainted background. */ + fun expected( + x: Int, + y: Int, + ): Int? = + when { + x < MARKER_SIZE && y < MARKER_SIZE -> MARKER + y in GRID_TOP until GRID_TOP + ROWS * CELL && x < COLUMNS * CELL -> + cellColor((x / CELL), (y - GRID_TOP) / CELL) + y in CHECKER_TOP until CHECKER_TOP + BAND -> if ((x + y) % 2 == 0) BLACK else WHITE + y in RAMP_TOP until RAMP_TOP + BAND -> ramp(x) + else -> null + } + + private fun cellColor( + column: Int, + row: Int, + ): Int { + val r = (column * 33 + row * 7) and 0xFF + val g = (255 - row * 61 - column * 3) and 0xFF + val b = (column * 91 + row * 45 + 17) and 0xFF + return argb(r, g, b) + } + + private fun ramp(x: Int): Int = argb(x, 255 - x, (x * 7) and 0xFF) + + private fun argb( + r: Int, + g: Int, + b: Int, + ): Int = (0xFF shl 24) or (r shl 16) or (g shl 8) or b + + private const val BLACK = 0xFF000000.toInt() + private const val WHITE = 0xFFFFFFFF.toInt() + + /** Top-left corners of every pattern found in [image]. */ + fun find(image: ScreenImage): List> { + val pixels = image.toArgbArray() + val found = mutableListOf>() + val w = image.width + for (y in 0..image.height - MARKER_SIZE) { + for (x in 0..w - MARKER_SIZE) { + if (pixels[y * w + x] != MARKER) continue + if (x > 0 && pixels[y * w + x - 1] == MARKER) continue + if (y > 0 && pixels[(y - 1) * w + x] == MARKER) continue + if (pixels[y * w + x + MARKER_SIZE - 1] == MARKER && pixels[(y + MARKER_SIZE - 1) * w + x] == MARKER) { + found += x to y + } + } + } + return found + } + + /** Pixels of the pattern at ([ox], [oy]) of [image] that differ from [expected]; the first ones described. */ + fun mismatches( + image: ScreenImage, + ox: Int, + oy: Int, + ): Pair> { + if (ox + WIDTH > image.width || oy + HEIGHT > image.height) return WIDTH * HEIGHT to listOf("pattern clipped") + var count = 0 + val samples = mutableListOf() + for (y in 0 until HEIGHT) { + for (x in 0 until WIDTH) { + val want = expected(x, y) ?: continue + val got = image.pixelAt(ox + x, oy + y) + if (got != want) { + count++ + if (samples.size < 5) samples += "($x,$y) want %08X got %08X".format(want, got) + } + } + } + return count to samples + } + + @Composable + fun Content(modifier: Modifier = Modifier) { + val density = LocalDensity.current + val size = + with(density) { + androidx.compose.ui.unit + .DpSize(WIDTH.toDp(), HEIGHT.toDp()) + } + Canvas(modifier.size(size)) { + fun px( + x: Int, + y: Int, + w: Int, + h: Int, + color: Int, + ) = drawRect(Color(color), Offset(x.toFloat(), y.toFloat()), Size(w.toFloat(), h.toFloat())) + px(0, 0, MARKER_SIZE, MARKER_SIZE, MARKER) + for (row in 0 until ROWS) { + for (column in 0 until COLUMNS) { + px( + column * CELL, + GRID_TOP + row * CELL, + CELL, + CELL, + cellColor(column, row), + ) + } + } + px(0, CHECKER_TOP, WIDTH, BAND, WHITE) + for (y in CHECKER_TOP until CHECKER_TOP + BAND) { + for (x in 0 until WIDTH) if ((x + y) % 2 == 0) px(x, y, 1, 1, BLACK) + } + for (x in 0 until WIDTH) px(x, RAMP_TOP, 1, BAND, ramp(x)) + } + } +} diff --git a/screen-capture/api/screen-capture.api b/screen-capture/api/screen-capture.api new file mode 100644 index 000000000..5a7feef50 --- /dev/null +++ b/screen-capture/api/screen-capture.api @@ -0,0 +1,102 @@ +public final class dev/nucleusframework/screencapture/CaptureBackend : java/lang/Enum { + public static final field CoreGraphics Ldev/nucleusframework/screencapture/CaptureBackend; + public static final field Gdi Ldev/nucleusframework/screencapture/CaptureBackend; + public static final field ScreenCaptureKit Ldev/nucleusframework/screencapture/CaptureBackend; + public static final field Unavailable Ldev/nucleusframework/screencapture/CaptureBackend; + public static final field X11 Ldev/nucleusframework/screencapture/CaptureBackend; + public static final field XdgDesktopPortal Ldev/nucleusframework/screencapture/CaptureBackend; + public static fun getEntries ()Lkotlin/enums/EnumEntries; + public static fun valueOf (Ljava/lang/String;)Ldev/nucleusframework/screencapture/CaptureBackend; + public static fun values ()[Ldev/nucleusframework/screencapture/CaptureBackend; +} + +public final class dev/nucleusframework/screencapture/CaptureDisplay { + public fun equals (Ljava/lang/Object;)Z + public final fun getBounds ()Ldev/nucleusframework/screencapture/CaptureRegion; + public final fun getHeightPx ()I + public final fun getId ()Ljava/lang/String; + public final fun getName ()Ljava/lang/String; + public final fun getScaleFactor ()F + public final fun getWidthPx ()I + public fun hashCode ()I + public final fun isPrimary ()Z + public fun toString ()Ljava/lang/String; +} + +public final class dev/nucleusframework/screencapture/CaptureFailure : java/lang/Enum { + public static final field Cancelled Ldev/nucleusframework/screencapture/CaptureFailure; + public static final field DisplayNotFound Ldev/nucleusframework/screencapture/CaptureFailure; + public static final field Failed Ldev/nucleusframework/screencapture/CaptureFailure; + public static final field InvalidRegion Ldev/nucleusframework/screencapture/CaptureFailure; + public static final field PermissionDenied Ldev/nucleusframework/screencapture/CaptureFailure; + public static final field Timeout Ldev/nucleusframework/screencapture/CaptureFailure; + public static final field Unsupported Ldev/nucleusframework/screencapture/CaptureFailure; + public static final field WindowNotFound Ldev/nucleusframework/screencapture/CaptureFailure; + public static fun getEntries ()Lkotlin/enums/EnumEntries; + public static fun valueOf (Ljava/lang/String;)Ldev/nucleusframework/screencapture/CaptureFailure; + public static fun values ()[Ldev/nucleusframework/screencapture/CaptureFailure; +} + +public final class dev/nucleusframework/screencapture/CapturePermission : java/lang/Enum { + public static final field Denied Ldev/nucleusframework/screencapture/CapturePermission; + public static final field Granted Ldev/nucleusframework/screencapture/CapturePermission; + public static final field NotDetermined Ldev/nucleusframework/screencapture/CapturePermission; + public static final field NotRequired Ldev/nucleusframework/screencapture/CapturePermission; + public static fun getEntries ()Lkotlin/enums/EnumEntries; + public static fun valueOf (Ljava/lang/String;)Ldev/nucleusframework/screencapture/CapturePermission; + public static fun values ()[Ldev/nucleusframework/screencapture/CapturePermission; +} + +public final class dev/nucleusframework/screencapture/CaptureRegion { + public fun (IIII)V + public final fun component1 ()I + public final fun component2 ()I + public final fun component3 ()I + public final fun component4 ()I + public final fun copy (IIII)Ldev/nucleusframework/screencapture/CaptureRegion; + public static synthetic fun copy$default (Ldev/nucleusframework/screencapture/CaptureRegion;IIIIILjava/lang/Object;)Ldev/nucleusframework/screencapture/CaptureRegion; + public fun equals (Ljava/lang/Object;)Z + public final fun getBottom ()I + public final fun getHeight ()I + public final fun getRight ()I + public final fun getWidth ()I + public final fun getX ()I + public final fun getY ()I + public fun hashCode ()I + public fun toString ()Ljava/lang/String; +} + +public final class dev/nucleusframework/screencapture/ScreenCapture { + public static final field INSTANCE Ldev/nucleusframework/screencapture/ScreenCapture; + public final fun captureDisplay (Ldev/nucleusframework/screencapture/CaptureDisplay;Ldev/nucleusframework/screencapture/CaptureRegion;Z)Ldev/nucleusframework/screencapture/ScreenImage; + public static synthetic fun captureDisplay$default (Ldev/nucleusframework/screencapture/ScreenCapture;Ldev/nucleusframework/screencapture/CaptureDisplay;Ldev/nucleusframework/screencapture/CaptureRegion;ZILjava/lang/Object;)Ldev/nucleusframework/screencapture/ScreenImage; + public final fun captureWindow (JZ)Ldev/nucleusframework/screencapture/ScreenImage; + public static synthetic fun captureWindow$default (Ldev/nucleusframework/screencapture/ScreenCapture;JZILjava/lang/Object;)Ldev/nucleusframework/screencapture/ScreenImage; + public final fun displays ()Ljava/util/List; + public final fun getBackend ()Ldev/nucleusframework/screencapture/CaptureBackend; + public final fun isSupported ()Z + public final fun isWindowCaptureSupported ()Z + public final fun permissionStatus ()Ldev/nucleusframework/screencapture/CapturePermission; + public final fun primaryDisplay ()Ldev/nucleusframework/screencapture/CaptureDisplay; + public final fun requestPermission ()Ldev/nucleusframework/screencapture/CapturePermission; +} + +public final class dev/nucleusframework/screencapture/ScreenCaptureException : java/lang/RuntimeException { + public fun (Ldev/nucleusframework/screencapture/CaptureFailure;Ljava/lang/String;)V + public final fun getFailure ()Ldev/nucleusframework/screencapture/CaptureFailure; +} + +public final class dev/nucleusframework/screencapture/ScreenImage { + public static final field BYTES_PER_PIXEL I + public final fun crop (Ldev/nucleusframework/screencapture/CaptureRegion;)Ldev/nucleusframework/screencapture/ScreenImage; + public final fun getHeight ()I + public final fun getScaleFactor ()F + public final fun getWidth ()I + public final fun pixelAt (II)I + public final fun toArgbArray ()[I + public final fun toBgraBytes ()[B + public final fun toPng ()[B + public fun toString ()Ljava/lang/String; + public final fun writePng (Ljava/io/OutputStream;)V +} + diff --git a/screen-capture/build.gradle.kts b/screen-capture/build.gradle.kts new file mode 100644 index 000000000..9cc06d880 --- /dev/null +++ b/screen-capture/build.gradle.kts @@ -0,0 +1,76 @@ +import org.jetbrains.kotlin.gradle.dsl.JvmTarget + +plugins { + kotlin("jvm") + id("nucleus.native-module") + alias(libs.plugins.vanniktechMavenPublish) +} + +val publishVersion = + providers + .environmentVariable("GITHUB_REF") + .orNull + ?.removePrefix("refs/tags/v") + ?: "1.0.0" + +dependencies { + api(project(":core-runtime")) + testImplementation(kotlin("test")) +} + +java { + sourceCompatibility = JavaVersion.VERSION_11 + targetCompatibility = JavaVersion.VERSION_11 +} + +kotlin { + compilerOptions { + jvmTarget.set(JvmTarget.JVM_11) + } +} + +tasks.test { + useJUnitPlatform() +} + +nucleusNative { + macos("nucleus_screencapture") + linux("nucleus_screencapture") + windows("nucleus_screencapture") +} + +mavenPublishing { + coordinates("dev.nucleusframework", "nucleus.screen-capture", publishVersion) + + pom { + name.set("Nucleus Screen Capture") + description.set("Native, AWT-free screen and window capture (GDI, ScreenCaptureKit, X11, xdg-desktop-portal)") + url.set("https://github.com/NucleusFramework/Nucleus") + + licenses { + license { + name.set("MIT License") + url.set("https://opensource.org/licenses/MIT") + } + } + + developers { + developer { + id.set("nucleusframework") + name.set("NucleusFramework") + url.set("https://github.com/NucleusFramework") + } + } + + scm { + url.set("https://github.com/NucleusFramework/Nucleus") + connection.set("scm:git:git://github.com/NucleusFramework/Nucleus.git") + developerConnection.set("scm:git:ssh://git@github.com/NucleusFramework/Nucleus.git") + } + } + + publishToMavenCentral() + if (project.hasProperty("signingInMemoryKey")) { + signAllPublications() + } +} diff --git a/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/CaptureDisplay.kt b/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/CaptureDisplay.kt new file mode 100644 index 000000000..5d8c1f0ef --- /dev/null +++ b/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/CaptureDisplay.kt @@ -0,0 +1,74 @@ +package dev.nucleusframework.screencapture + +/** + * A display that [ScreenCapture] can capture. + * + * @property id platform identifier, stable for the session: the GDI device name on Windows + * (`\\.\DISPLAY1`), the `CGDirectDisplayID` on macOS, the RandR output name on X11 + * (`screen` without RandR), `portal` on Wayland. + * @property name human-readable name (the monitor's name when the platform reports one). + * @property bounds the display's rectangle in the platform's desktop coordinate space: + * physical pixels of the virtual screen on Windows and X11, points of the global display + * space on macOS (origin at the primary display's top-left). `null` on Wayland, where the + * compositor does not expose it. + * @property widthPx width of a full capture of this display, in physical pixels; `0` while + * unknown (Wayland, before the first capture). + * @property heightPx height of a full capture of this display, in physical pixels; `0` while + * unknown. + * @property scaleFactor physical pixels per [bounds] unit on macOS, the display's DPI scale + * (`dpi / 96`) on Windows, `1` on X11 and Wayland. + * @property isPrimary whether this is the primary display. + */ +public class CaptureDisplay internal constructor( + public val id: String, + public val name: String, + public val bounds: CaptureRegion?, + public val widthPx: Int, + public val heightPx: Int, + public val scaleFactor: Float, + public val isPrimary: Boolean, +) { + override fun equals(other: Any?): Boolean = + other is CaptureDisplay && + id == other.id && + name == other.name && + bounds == other.bounds && + widthPx == other.widthPx && + heightPx == other.heightPx && + scaleFactor == other.scaleFactor && + isPrimary == other.isPrimary + + override fun hashCode(): Int { + var result = id.hashCode() + result = 31 * result + bounds.hashCode() + result = 31 * result + widthPx + result = 31 * result + heightPx + return result + } + + override fun toString(): String = + "CaptureDisplay(id=$id, name=$name, bounds=$bounds, px=${widthPx}x$heightPx, " + + "scale=$scaleFactor, primary=$isPrimary)" +} + +/** + * An axis-aligned rectangle. In [ScreenCapture.captureDisplay] it is expressed in the + * captured display's physical pixels, relative to its top-left corner — the coordinates of + * the image a full capture returns. + */ +public data class CaptureRegion( + val x: Int, + val y: Int, + val width: Int, + val height: Int, +) { + init { + require(width > 0 && height > 0) { "Region must not be empty: ${width}x$height" } + } + + /** Right edge, exclusive. */ + val right: Int get() = x + width + + /** Bottom edge, exclusive. */ + val bottom: Int get() = y + height +} diff --git a/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/ScreenCapture.kt b/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/ScreenCapture.kt new file mode 100644 index 000000000..1409a912c --- /dev/null +++ b/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/ScreenCapture.kt @@ -0,0 +1,263 @@ +package dev.nucleusframework.screencapture + +import dev.nucleusframework.core.runtime.Platform +import dev.nucleusframework.screencapture.internal.DisplayCollector +import dev.nucleusframework.screencapture.internal.NativeCall +import dev.nucleusframework.screencapture.internal.NativeScreenCapture +import dev.nucleusframework.screencapture.internal.RawDisplay +import dev.nucleusframework.screencapture.internal.clip +import dev.nucleusframework.screencapture.internal.permissionOf +import java.util.logging.Level +import java.util.logging.Logger + +/** The mechanism [ScreenCapture] captures with on this machine. */ +public enum class CaptureBackend { + /** Windows GDI, rendered in a per-monitor DPI context (physical pixels). */ + Gdi, + + /** macOS 14+ ScreenCaptureKit. */ + ScreenCaptureKit, + + /** macOS before 14: Core Graphics display images. */ + CoreGraphics, + + /** Linux X11 (`XGetImage`, RandR outputs, XFixes cursor). */ + X11, + + /** + * Linux Wayland: `org.freedesktop.portal.Screenshot`. The compositor captures the whole + * desktop and may ask the user for permission; display regions are cropped from it. + */ + XdgDesktopPortal, + + /** No backend: the native library did not load or the platform is not supported. */ + Unavailable, +} + +/** Whether the app may capture the screen. */ +public enum class CapturePermission { + /** Capture is allowed. */ + Granted, + + /** The user denied it; on macOS it can only be granted again in System Settings. */ + Denied, + + /** Not asked yet: the first capture (or [ScreenCapture.requestPermission]) prompts. */ + NotDetermined, + + /** The platform has no screen capture permission (Windows, X11). */ + NotRequired, +} + +/** + * Native screen capture, without AWT. + * + * Every function blocks until the platform answers — call them off the UI thread. They are + * safe to call from several threads at once. + * + * Images are in physical pixels: a 4K display at 200 % captures as 3840×2160, not + * the 1920×1080 `java.awt.Robot` returns. + * + * ```kotlin + * val display = ScreenCapture.displays().first { it.isPrimary } + * val image = ScreenCapture.captureDisplay(display, region = CaptureRegion(0, 0, 800, 600)) + * File("shot.png").writeBytes(image.toPng()) + * ``` + */ +public object ScreenCapture { + private val logger = Logger.getLogger(ScreenCapture::class.java.name) + private const val PORTAL_DISPLAY_ID = "portal" + private const val PORTAL_TIMEOUT_MS = 60_000 + private const val LINUX_BACKEND_PROPERTY = "nucleus.screencapture.linuxBackend" + + @Volatile + private var portalImageSize: Pair? = null + + /** The backend captures go through; [CaptureBackend.Unavailable] when there is none. */ + public val backend: CaptureBackend by lazy { resolveBackend() } + + /** `true` when [backend] is not [CaptureBackend.Unavailable]. */ + public val isSupported: Boolean get() = backend != CaptureBackend.Unavailable + + /** Whether window capture ([captureWindow]) is available: not on Wayland. */ + public val isWindowCaptureSupported: Boolean + get() = backend != CaptureBackend.Unavailable && backend != CaptureBackend.XdgDesktopPortal + + /** + * The displays currently connected, primary first. + * + * On Wayland this is a single `portal` display standing for the whole desktop. + * + * @throws ScreenCaptureException when the backend cannot enumerate them. + */ + public fun displays(): List { + when (backend) { + CaptureBackend.Unavailable -> return emptyList() + CaptureBackend.XdgDesktopPortal -> return listOf(portalDisplay()) + else -> Unit + } + val collector = DisplayCollector() + val call = NativeCall("list displays") + call.check(NativeScreenCapture.nativeListDisplays(collector, call.message)) + return collector.displays + .map(RawDisplay::toDisplay) + .sortedByDescending { it.isPrimary } + } + + /** The primary display, or `null` when none is reported. */ + public fun primaryDisplay(): CaptureDisplay? = displays().firstOrNull() + + /** + * Captures [display], or the [region] of it (display pixels, relative to its top-left; + * clipped to the display). + * + * @param includeCursor draw the mouse cursor into the image. Ignored on Wayland, where the + * compositor decides. + * @throws ScreenCaptureException see [ScreenCaptureException.failure]. + */ + public fun captureDisplay( + display: CaptureDisplay, + region: CaptureRegion? = null, + includeCursor: Boolean = false, + ): ScreenImage { + requireSupported() + if (backend == CaptureBackend.XdgDesktopPortal) { + val image = portalScreenshot() + return if (region == null) image else image.crop(clip(region, image.width, image.height)) + } + val call = NativeCall("capture display ${display.id}") + val pixels = + NativeScreenCapture.nativeCaptureDisplay( + display.id, + region?.x ?: 0, + region?.y ?: 0, + region?.width ?: 0, + region?.height ?: 0, + includeCursor, + call.result, + call.message, + ) + return call.image(pixels, display.scaleFactor) + } + + /** + * Captures a top-level window, including the parts other windows cover, when the platform + * can: an `HWND` on Windows, a `CGWindowID` (`NSWindow.windowNumber`) on macOS, an XID on + * X11. Not available on Wayland ([isWindowCaptureSupported]). + * + * @throws ScreenCaptureException see [ScreenCaptureException.failure]. + */ + public fun captureWindow( + windowId: Long, + includeCursor: Boolean = false, + ): ScreenImage { + requireSupported() + if (!isWindowCaptureSupported) { + throw ScreenCaptureException(CaptureFailure.Unsupported, "Window capture is not available on $backend") + } + val call = NativeCall("capture window $windowId") + val pixels = NativeScreenCapture.nativeCaptureWindow(windowId, includeCursor, call.result, call.message) + return call.image(pixels, 1f) + } + + /** Whether the app may capture the screen, without prompting. */ + public fun permissionStatus(): CapturePermission = + when (backend) { + CaptureBackend.Unavailable -> CapturePermission.Denied + CaptureBackend.ScreenCaptureKit, CaptureBackend.CoreGraphics -> + permissionOf(NativeScreenCapture.nativePermissionStatus()) + else -> CapturePermission.NotRequired + } + + /** + * Asks for screen recording permission where the platform has one (macOS: the system + * prompt, shown once per app; the answer takes effect after the app restarts). Returns the + * resulting status without waiting for the user. + */ + public fun requestPermission(): CapturePermission = + when (backend) { + CaptureBackend.ScreenCaptureKit, CaptureBackend.CoreGraphics -> + permissionOf(NativeScreenCapture.nativeRequestPermission()) + else -> permissionStatus() + } + + private fun resolveBackend(): CaptureBackend { + if (Platform.Current == Platform.Unknown) return CaptureBackend.Unavailable + val loaded = + try { + NativeScreenCapture.isLoaded + } catch (e: LinkageError) { + logger.log(Level.WARNING, "Screen capture native library failed to load", e) + false + } + if (!loaded) { + logger.warning("Screen capture native library is not available on ${Platform.Current}") + return CaptureBackend.Unavailable + } + if (Platform.Current == Platform.Linux && useLinuxPortal()) return CaptureBackend.XdgDesktopPortal + return when (NativeScreenCapture.nativeBackend()) { + NativeScreenCapture.BACKEND_GDI -> CaptureBackend.Gdi + NativeScreenCapture.BACKEND_SCREEN_CAPTURE_KIT -> CaptureBackend.ScreenCaptureKit + NativeScreenCapture.BACKEND_CORE_GRAPHICS -> CaptureBackend.CoreGraphics + NativeScreenCapture.BACKEND_X11 -> CaptureBackend.X11 + // Linux without an X server may still have a portal. + else -> + if (Platform.Current == + Platform.Linux + ) { + CaptureBackend.XdgDesktopPortal + } else { + CaptureBackend.Unavailable + } + } + } + + /** + * On a Wayland session the X server is XWayland, whose root window does not hold the + * native Wayland windows — only the portal sees the real desktop. + */ + private fun useLinuxPortal(): Boolean = + when (System.getProperty(LINUX_BACKEND_PROPERTY)?.lowercase()) { + "x11" -> false + "portal" -> true + else -> Platform.isWayland + } + + private fun portalDisplay(): CaptureDisplay { + val size = portalImageSize + return CaptureDisplay( + id = PORTAL_DISPLAY_ID, + name = "Desktop", + bounds = null, + widthPx = size?.first ?: 0, + heightPx = size?.second ?: 0, + scaleFactor = 1f, + isPrimary = true, + ) + } + + private fun portalScreenshot(): ScreenImage { + val call = NativeCall("take a portal screenshot") + val pixels = NativeScreenCapture.nativePortalScreenshot(false, PORTAL_TIMEOUT_MS, call.result, call.message) + val image = call.image(pixels, 1f) + portalImageSize = image.width to image.height + return image + } + + private fun requireSupported() { + if (backend == CaptureBackend.Unavailable) { + throw ScreenCaptureException(CaptureFailure.Unsupported, "No screen capture backend on ${Platform.Current}") + } + } +} + +private fun RawDisplay.toDisplay(): CaptureDisplay = + CaptureDisplay( + id = id, + name = name, + bounds = CaptureRegion(x, y, width.coerceAtLeast(1), height.coerceAtLeast(1)), + widthPx = widthPx, + heightPx = heightPx, + scaleFactor = scaleFactor, + isPrimary = isPrimary, + ) diff --git a/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/ScreenCaptureException.kt b/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/ScreenCaptureException.kt new file mode 100644 index 000000000..c4840f8c5 --- /dev/null +++ b/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/ScreenCaptureException.kt @@ -0,0 +1,34 @@ +package dev.nucleusframework.screencapture + +/** Why a capture failed. */ +public enum class CaptureFailure { + /** The platform or session has no capture backend (e.g. Wayland without a portal). */ + Unsupported, + + /** The user or the system denied screen recording (macOS TCC, portal permission store). */ + PermissionDenied, + + /** The user dismissed the platform's capture dialog (interactive portal screenshot). */ + Cancelled, + + /** No display matches the requested one — it was unplugged or the id is stale. */ + DisplayNotFound, + + /** No window matches the requested id, or it is not capturable (minimized, unmapped). */ + WindowNotFound, + + /** The requested region does not intersect the display. */ + InvalidRegion, + + /** The platform did not answer in time. */ + Timeout, + + /** Any other platform error. */ + Failed, +} + +/** Thrown by [ScreenCapture] when a capture cannot be produced; see [failure]. */ +public class ScreenCaptureException( + public val failure: CaptureFailure, + message: String, +) : RuntimeException(message) diff --git a/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/ScreenImage.kt b/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/ScreenImage.kt new file mode 100644 index 000000000..626539741 --- /dev/null +++ b/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/ScreenImage.kt @@ -0,0 +1,85 @@ +package dev.nucleusframework.screencapture + +import dev.nucleusframework.screencapture.internal.PngEncoder +import java.io.OutputStream + +/** + * A captured image, in physical pixels. + * + * Pixels are opaque, row-major, packed as `0xAARRGGBB` — the layout of Skia's + * `BGRA_8888` on little-endian machines, and of `java.awt.image.BufferedImage.TYPE_INT_ARGB`. + * + * @property width width in pixels. + * @property height height in pixels. + * @property scaleFactor physical pixels per logical unit of the captured source (see + * [CaptureDisplay.scaleFactor]); `1` when the platform does not report one. + */ +public class ScreenImage internal constructor( + public val width: Int, + public val height: Int, + private val argb: IntArray, + public val scaleFactor: Float, +) { + init { + require(width > 0 && height > 0 && argb.size == width * height) { + "Pixel buffer does not match ${width}x$height: ${argb.size}" + } + } + + /** The pixel at ([x], [y]), as `0xAARRGGBB`. */ + public fun pixelAt( + x: Int, + y: Int, + ): Int { + require(x in 0 until width && y in 0 until height) { "($x, $y) outside ${width}x$height" } + return argb[y * width + x] + } + + /** A copy of every pixel, row-major, `0xAARRGGBB`. */ + public fun toArgbArray(): IntArray = argb.copyOf() + + /** + * The pixels as `B, G, R, A` bytes, row-major, `width * 4` bytes per row — ready for + * `org.jetbrains.skia.Image.makeRaster(ImageInfo(width, height, ColorType.BGRA_8888, + * ColorAlphaType.OPAQUE), bytes, width * 4)`. + */ + @Suppress("MagicNumber") // byte lanes of 0xAARRGGBB + public fun toBgraBytes(): ByteArray { + val out = ByteArray(argb.size * BYTES_PER_PIXEL) + var o = 0 + for (p in argb) { + out[o] = p.toByte() + out[o + 1] = (p ushr 8).toByte() + out[o + 2] = (p ushr 16).toByte() + out[o + 3] = (p ushr 24).toByte() + o += BYTES_PER_PIXEL + } + return out + } + + /** Encodes the image as an RGB PNG. */ + public fun toPng(): ByteArray = PngEncoder.encode(width, height, argb) + + /** Writes the image to [out] as an RGB PNG; [out] is left open. */ + public fun writePng(out: OutputStream) { + out.write(toPng()) + } + + /** A new image holding the [region] of this one. */ + public fun crop(region: CaptureRegion): ScreenImage { + require(region.x >= 0 && region.y >= 0 && region.right <= width && region.bottom <= height) { + "$region outside ${width}x$height" + } + val out = IntArray(region.width * region.height) + for (row in 0 until region.height) { + System.arraycopy(argb, (region.y + row) * width + region.x, out, row * region.width, region.width) + } + return ScreenImage(region.width, region.height, out, scaleFactor) + } + + override fun toString(): String = "ScreenImage(${width}x$height, scale=$scaleFactor)" + + private companion object { + const val BYTES_PER_PIXEL = 4 + } +} diff --git a/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/internal/NativeCall.kt b/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/internal/NativeCall.kt new file mode 100644 index 000000000..5621ba4f0 --- /dev/null +++ b/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/internal/NativeCall.kt @@ -0,0 +1,75 @@ +package dev.nucleusframework.screencapture.internal + +import dev.nucleusframework.screencapture.CaptureFailure +import dev.nucleusframework.screencapture.CapturePermission +import dev.nucleusframework.screencapture.CaptureRegion +import dev.nucleusframework.screencapture.ScreenCaptureException +import dev.nucleusframework.screencapture.ScreenImage + +/** The out-parameters of one [NativeScreenCapture] call, turned into a result or a [ScreenCaptureException]. */ +internal class NativeCall( + private val what: String, +) { + /** `[status, width, height]`. */ + val result = IntArray(RESULT_SIZE) + + /** A diagnostic from the native side, on failure. */ + val message = arrayOfNulls(1) + + fun check(status: Int) { + if (status == NativeScreenCapture.STATUS_OK) return + val failure = failureOf(status) + val detail = message[0]?.let { ": $it" }.orEmpty() + throw ScreenCaptureException(failure, "Cannot $what ($failure)$detail") + } + + fun image( + pixels: IntArray?, + scaleFactor: Float, + ): ScreenImage { + // A null array without an error status means the JVM could not allocate it. + val lost = pixels == null && result[0] == NativeScreenCapture.STATUS_OK + check(if (lost) NativeScreenCapture.STATUS_FAILED else result[0]) + return ScreenImage(result[1], result[2], checkNotNull(pixels), scaleFactor) + } + + private fun failureOf(status: Int): CaptureFailure = + when (status) { + NativeScreenCapture.STATUS_UNSUPPORTED -> CaptureFailure.Unsupported + NativeScreenCapture.STATUS_PERMISSION_DENIED -> CaptureFailure.PermissionDenied + NativeScreenCapture.STATUS_DISPLAY_NOT_FOUND -> CaptureFailure.DisplayNotFound + NativeScreenCapture.STATUS_WINDOW_NOT_FOUND -> CaptureFailure.WindowNotFound + NativeScreenCapture.STATUS_CANCELLED -> CaptureFailure.Cancelled + NativeScreenCapture.STATUS_TIMEOUT -> CaptureFailure.Timeout + NativeScreenCapture.STATUS_INVALID_REGION -> CaptureFailure.InvalidRegion + else -> CaptureFailure.Failed + } + + private companion object { + const val RESULT_SIZE = 3 + } +} + +internal fun permissionOf(value: Int): CapturePermission = + when (value) { + NativeScreenCapture.PERMISSION_GRANTED -> CapturePermission.Granted + NativeScreenCapture.PERMISSION_DENIED -> CapturePermission.Denied + NativeScreenCapture.PERMISSION_NOT_DETERMINED -> CapturePermission.NotDetermined + else -> CapturePermission.NotRequired + } + +/** [region] clipped to a `width` × `height` image; [CaptureFailure.InvalidRegion] when nothing is left. */ +internal fun clip( + region: CaptureRegion, + width: Int, + height: Int, +): CaptureRegion { + val left = region.x.coerceAtLeast(0) + val top = region.y.coerceAtLeast(0) + val right = region.right.coerceAtMost(width) + val bottom = region.bottom.coerceAtMost(height) + if (right <= left || bottom <= top) { + throw ScreenCaptureException(CaptureFailure.InvalidRegion, "$region is outside the ${width}x$height desktop") + } + return CaptureRegion(left, top, right - left, bottom - top) +} diff --git a/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/internal/NativeScreenCapture.kt b/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/internal/NativeScreenCapture.kt new file mode 100644 index 000000000..52366fbc5 --- /dev/null +++ b/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/internal/NativeScreenCapture.kt @@ -0,0 +1,121 @@ +package dev.nucleusframework.screencapture.internal + +import dev.nucleusframework.core.runtime.NativeLibraryLoader + +/** + * JNI bridge, one library per platform with the same symbols. + * + * Capture calls return the pixels (`0xAARRGGBB`, row-major, alpha forced opaque) or `null`, + * and fill `result` with `[status, width, height]`; `message[0]` receives a diagnostic on + * failure. Status codes are the `STATUS_*` constants. + */ +internal object NativeScreenCapture { + const val STATUS_OK = 0 + const val STATUS_UNSUPPORTED = 1 + const val STATUS_PERMISSION_DENIED = 2 + const val STATUS_DISPLAY_NOT_FOUND = 3 + const val STATUS_WINDOW_NOT_FOUND = 4 + const val STATUS_CANCELLED = 5 + const val STATUS_FAILED = 6 + const val STATUS_TIMEOUT = 7 + const val STATUS_INVALID_REGION = 8 + + const val PERMISSION_GRANTED = 0 + const val PERMISSION_DENIED = 1 + const val PERMISSION_NOT_DETERMINED = 2 + const val PERMISSION_NOT_REQUIRED = 3 + + // nativeBackend() values. + const val BACKEND_NONE = 0 + const val BACKEND_GDI = 1 + const val BACKEND_SCREEN_CAPTURE_KIT = 2 + const val BACKEND_CORE_GRAPHICS = 3 + const val BACKEND_X11 = 4 + + val isLoaded: Boolean by lazy { NativeLibraryLoader.load("nucleus_screencapture", NativeScreenCapture::class.java) } + + /** The backend the native side uses for display capture; [BACKEND_NONE] when it has none. */ + @JvmStatic + external fun nativeBackend(): Int + + /** Reports every display to [sink]; returns a status code. */ + @JvmStatic + external fun nativeListDisplays( + sink: DisplayCollector, + message: Array, + ): Int + + /** + * Captures [displayId]. A region with `width <= 0` means the whole display; otherwise it is + * in display pixels and the native side clips it to the display. + */ + @JvmStatic + external fun nativeCaptureDisplay( + displayId: String, + x: Int, + y: Int, + width: Int, + height: Int, + includeCursor: Boolean, + result: IntArray, + message: Array, + ): IntArray? + + /** Captures a top-level window: an `HWND` on Windows, a `CGWindowID` on macOS, an XID on X11. */ + @JvmStatic + external fun nativeCaptureWindow( + windowId: Long, + includeCursor: Boolean, + result: IntArray, + message: Array, + ): IntArray? + + @JvmStatic + external fun nativePermissionStatus(): Int + + @JvmStatic + external fun nativeRequestPermission(): Int + + /** Linux only: `org.freedesktop.portal.Screenshot`, decoded to pixels. */ + @JvmStatic + external fun nativePortalScreenshot( + interactive: Boolean, + timeoutMs: Int, + result: IntArray, + message: Array, + ): IntArray? +} + +/** Receives [NativeScreenCapture.nativeListDisplays] records; called from native code. */ +internal class DisplayCollector { + val displays = mutableListOf() + + @Suppress("LongParameterList") + fun add( + id: String, + name: String, + x: Int, + y: Int, + width: Int, + height: Int, + widthPx: Int, + heightPx: Int, + scaleFactor: Float, + isPrimary: Boolean, + ) { + displays += RawDisplay(id, name, x, y, width, height, widthPx, heightPx, scaleFactor, isPrimary) + } +} + +internal data class RawDisplay( + val id: String, + val name: String, + val x: Int, + val y: Int, + val width: Int, + val height: Int, + val widthPx: Int, + val heightPx: Int, + val scaleFactor: Float, + val isPrimary: Boolean, +) diff --git a/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/internal/PngEncoder.kt b/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/internal/PngEncoder.kt new file mode 100644 index 000000000..3bf33171e --- /dev/null +++ b/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/internal/PngEncoder.kt @@ -0,0 +1,100 @@ +package dev.nucleusframework.screencapture.internal + +import java.io.ByteArrayOutputStream +import java.io.DataOutputStream +import java.util.zip.CRC32 +import java.util.zip.Deflater +import java.util.zip.DeflaterOutputStream + +/** Minimal PNG writer (8-bit RGB, filter `Sub`) — no ImageIO, so no AWT and no native-image metadata. */ +@Suppress("MagicNumber") // PNG signature bytes and RGB byte lanes +internal object PngEncoder { + private val SIGNATURE = + byteArrayOf(0x89.toByte(), 'P'.code.toByte(), 'N'.code.toByte(), 'G'.code.toByte(), 13, 10, 26, 10) + private const val COLOR_TYPE_RGB = 2 + private const val BIT_DEPTH = 8 + private const val FILTER_SUB = 1 + private const val CHANNELS = 3 + + fun encode( + width: Int, + height: Int, + argb: IntArray, + ): ByteArray { + val out = ByteArrayOutputStream(argb.size + 1024) + out.write(SIGNATURE) + val header = ByteArrayOutputStream() + DataOutputStream(header).apply { + writeInt(width) + writeInt(height) + writeByte(BIT_DEPTH) + writeByte(COLOR_TYPE_RGB) + writeByte(0) // compression + writeByte(0) // filter method + writeByte(0) // no interlace + } + writeChunk(out, "IHDR", header.toByteArray()) + writeChunk(out, "IDAT", compress(width, height, argb)) + writeChunk(out, "IEND", ByteArray(0)) + return out.toByteArray() + } + + private fun compress( + width: Int, + height: Int, + argb: IntArray, + ): ByteArray { + val stride = width * CHANNELS + val row = ByteArray(1 + stride) + val compressed = ByteArrayOutputStream(argb.size) + val deflater = Deflater(Deflater.BEST_SPEED) + try { + DeflaterOutputStream(compressed, deflater, 1 shl 16).use { z -> + for (y in 0 until height) { + row[0] = FILTER_SUB.toByte() + var prevR = 0 + var prevG = 0 + var prevB = 0 + var o = 1 + val base = y * width + for (x in 0 until width) { + val p = argb[base + x] + val r = (p ushr 16) and 0xFF + val g = (p ushr 8) and 0xFF + val b = p and 0xFF + row[o] = (r - prevR).toByte() + row[o + 1] = (g - prevG).toByte() + row[o + 2] = (b - prevB).toByte() + prevR = r + prevG = g + prevB = b + o += CHANNELS + } + z.write(row) + } + } + } finally { + deflater.end() + } + return compressed.toByteArray() + } + + private fun writeChunk( + out: ByteArrayOutputStream, + type: String, + data: ByteArray, + ) { + val typeBytes = type.toByteArray(Charsets.US_ASCII) + val crc = + CRC32().apply { + update(typeBytes) + update(data) + } + DataOutputStream(out).apply { + writeInt(data.size) + write(typeBytes) + write(data) + writeInt(crc.value.toInt()) + } + } +} diff --git a/screen-capture/src/main/native/linux/build.sh b/screen-capture/src/main/native/linux/build.sh new file mode 100644 index 000000000..f307cb9ac --- /dev/null +++ b/screen-capture/src/main/native/linux/build.sh @@ -0,0 +1,57 @@ +#!/bin/bash +# Compiles nucleus_screencapture_linux.c into libnucleus_screencapture.so for the host architecture. +# The output is placed in the JAR resources so it ships with the library. +# +# Every runtime library (libX11, libXrandr, libXfixes, libXcomposite, libdbus-1, gdk-pixbuf) is +# dlopen'd, so the .so links against none of them; only the headers are needed. +# +# Prerequisites: gcc, JDK with JNI headers, libx11-dev, libxrandr-dev, libxfixes-dev, +# libxcomposite-dev, libdbus-1-dev. +# Usage: ./build.sh + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +SRC="$SCRIPT_DIR/nucleus_screencapture_linux.c" +RESOURCE_DIR="$SCRIPT_DIR/../../resources/nucleus/native" + +# Detect JAVA_HOME for JNI headers +if [ -z "${JAVA_HOME:-}" ]; then + JAVA_HOME=$(dirname "$(dirname "$(readlink -f "$(command -v java)")")") +fi +if [ -z "${JAVA_HOME:-}" ]; then + echo "ERROR: JAVA_HOME not set and auto-detection failed." >&2 + exit 1 +fi + +JNI_INCLUDE="$JAVA_HOME/include" +JNI_INCLUDE_LINUX="$JAVA_HOME/include/linux" + +if [ ! -f "$JNI_INCLUDE/jni.h" ]; then + echo "ERROR: JNI headers not found at $JNI_INCLUDE" >&2 + exit 1 +fi + +DBUS_CFLAGS=$(pkg-config --cflags dbus-1) + +case "$(uname -m)" in + x86_64) OUT_DIR="$RESOURCE_DIR/linux-x64" ;; + aarch64) OUT_DIR="$RESOURCE_DIR/linux-aarch64" ;; + *) + echo "WARNING: Unsupported architecture $(uname -m), building for current arch anyway." + OUT_DIR="$RESOURCE_DIR/linux-$(uname -m)" + ;; +esac +mkdir -p "$OUT_DIR" + +gcc -shared -fPIC \ + -I"$JNI_INCLUDE" -I"$JNI_INCLUDE_LINUX" \ + $DBUS_CFLAGS \ + -O2 -Wall -Wextra -Wno-unused-parameter \ + -fvisibility=hidden \ + -Wl,--strip-all \ + -o "$OUT_DIR/libnucleus_screencapture.so" "$SRC" \ + -ldl -lpthread + +echo "Built:" +ls -lh "$OUT_DIR/libnucleus_screencapture.so" diff --git a/screen-capture/src/main/native/linux/nucleus_screencapture_linux.c b/screen-capture/src/main/native/linux/nucleus_screencapture_linux.c new file mode 100644 index 000000000..63297f543 --- /dev/null +++ b/screen-capture/src/main/native/linux/nucleus_screencapture_linux.c @@ -0,0 +1,1387 @@ +/** + * JNI bridge for Linux screen capture. + * + * - X11: XGetImage on the root window (or a window / its XComposite pixmap), + * RandR monitors for the display list, XFixes for the cursor. + * - Wayland: org.freedesktop.portal.Screenshot over D-Bus, decoded with + * gdk-pixbuf. + * + * Every library is dlopen'd on first use - nothing is linked at build time, so + * the .so loads on a machine without X11, D-Bus or gdk-pixbuf and the + * missing backend simply reports itself as unavailable. + * + * X11 calls are serialized by one mutex: the error handlers are process-wide, + * and a capture installs its own for the duration of the call (chaining every + * error that is not on its connection to the previous handler, so a toolkit + * that shares the process keeps its own handling). Each call opens and closes + * its own connection, so display changes are always seen and nothing is held + * between captures. + * + * Portal screenshots are written by the portal to a file (usually in the + * user's Pictures folder). The caller asked for pixels, not a saved picture, so + * the file is deleted once decoded - only when it is a regular file (never a + * symlink or anything else). + */ + +#include +#include "../../../../../native-common/nucleus_jni.h" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define EXPORT __attribute__((visibility("default"))) + +/* Mirrors NativeScreenCapture.STATUS_* / PERMISSION_* / BACKEND_*. */ +enum { + ST_OK = 0, + ST_UNSUPPORTED = 1, + ST_PERMISSION_DENIED = 2, + ST_DISPLAY_NOT_FOUND = 3, + ST_WINDOW_NOT_FOUND = 4, + ST_CANCELLED = 5, + ST_FAILED = 6, + ST_TIMEOUT = 7, + ST_INVALID_REGION = 8, +}; +#define PERMISSION_NOT_REQUIRED 3 +#define BACKEND_NONE 0 +#define BACKEND_X11 4 + +/* Java arrays are indexed by int. */ +#define MAX_PIXELS ((int64_t)0x7FFFFFF0) + +/* ------------------------------------------------------------------------ */ +/* JNI helpers */ +/* ------------------------------------------------------------------------ */ + +static void set_message(JNIEnv *env, jobjectArray message, const char *fmt, ...) { + if (message == NULL || (*env)->GetArrayLength(env, message) < 1) return; + char buf[512]; + va_list ap; + va_start(ap, fmt); + vsnprintf(buf, sizeof(buf), fmt, ap); + va_end(ap); + jstring s = (*env)->NewStringUTF(env, buf); + if (s == NULL) { + nucleus_jni_clear_exception(env); + return; + } + (*env)->SetObjectArrayElement(env, message, 0, s); + (*env)->DeleteLocalRef(env, s); +} + +static void set_result(JNIEnv *env, jintArray result, jint status, jint w, jint h) { + if (result == NULL || (*env)->GetArrayLength(env, result) < 3) return; + jint v[3] = {status, w, h}; + (*env)->SetIntArrayRegion(env, result, 0, 3, v); +} + +/* ------------------------------------------------------------------------ */ +/* Dynamic loading */ +/* ------------------------------------------------------------------------ */ + +typedef int (*XIOErrorExitHandler_t)(Display *, void *); + +static struct { + int loaded; + Display *(*OpenDisplay)(const char *); + int (*CloseDisplay)(Display *); + XErrorHandler (*SetErrorHandler)(XErrorHandler); + XIOErrorHandler (*SetIOErrorHandler)(XIOErrorHandler); + void (*SetIOErrorExitHandler)(Display *, XIOErrorExitHandler_t, void *); + int (*Sync)(Display *, Bool); + XImage *(*GetImage)(Display *, Drawable, int, int, unsigned int, unsigned int, unsigned long, int); + Status (*GetWindowAttributes)(Display *, Window, XWindowAttributes *); + Bool (*TranslateCoordinates)(Display *, Window, Window, int, int, int *, int *, Window *); + int (*QueryColors)(Display *, Colormap, XColor *, int); + char *(*GetAtomName)(Display *, Atom); + int (*Free)(void *); + int (*FreePixmap)(Display *, Pixmap); + Bool (*QueryExtension)(Display *, const char *, int *, int *, int *); +} X; + +static struct { + int loaded; + Bool (*QueryExtension)(Display *, int *, int *); + Status (*QueryVersion)(Display *, int *, int *); + XRRMonitorInfo *(*GetMonitors)(Display *, Window, Bool, int *); + void (*FreeMonitors)(XRRMonitorInfo *); + XRRScreenResources *(*GetScreenResourcesCurrent)(Display *, Window); + void (*FreeScreenResources)(XRRScreenResources *); + XRROutputInfo *(*GetOutputInfo)(Display *, XRRScreenResources *, RROutput); + void (*FreeOutputInfo)(XRROutputInfo *); + XRRCrtcInfo *(*GetCrtcInfo)(Display *, XRRScreenResources *, RRCrtc); + void (*FreeCrtcInfo)(XRRCrtcInfo *); + RROutput (*GetOutputPrimary)(Display *, Window); +} XRR; + +static struct { + int loaded; + Bool (*QueryExtension)(Display *, int *, int *); + XFixesCursorImage *(*GetCursorImage)(Display *); +} XFX; + +static struct { + int loaded; + Bool (*QueryExtension)(Display *, int *, int *); + Status (*QueryVersion)(Display *, int *, int *); + Pixmap (*NameWindowPixmap)(Display *, Window); +} XCOMP; + +static pthread_once_t g_x11_once = PTHREAD_ONCE_INIT; + +#define LOAD(lib, table, field, name) (table).field = (void *)dlsym(lib, name) + +static void load_x11(void) { + void *lib = dlopen("libX11.so.6", RTLD_LAZY | RTLD_LOCAL); + if (lib != NULL) { + LOAD(lib, X, OpenDisplay, "XOpenDisplay"); + LOAD(lib, X, CloseDisplay, "XCloseDisplay"); + LOAD(lib, X, SetErrorHandler, "XSetErrorHandler"); + LOAD(lib, X, SetIOErrorHandler, "XSetIOErrorHandler"); + LOAD(lib, X, SetIOErrorExitHandler, "XSetIOErrorExitHandler"); + LOAD(lib, X, Sync, "XSync"); + LOAD(lib, X, GetImage, "XGetImage"); + LOAD(lib, X, GetWindowAttributes, "XGetWindowAttributes"); + LOAD(lib, X, TranslateCoordinates, "XTranslateCoordinates"); + LOAD(lib, X, QueryColors, "XQueryColors"); + LOAD(lib, X, GetAtomName, "XGetAtomName"); + LOAD(lib, X, Free, "XFree"); + LOAD(lib, X, FreePixmap, "XFreePixmap"); + LOAD(lib, X, QueryExtension, "XQueryExtension"); + X.loaded = X.OpenDisplay && X.CloseDisplay && X.SetErrorHandler && X.Sync && X.GetImage && + X.GetWindowAttributes && X.TranslateCoordinates && X.QueryColors && X.GetAtomName && + X.Free && X.FreePixmap; + } + if (!X.loaded) return; + + lib = dlopen("libXrandr.so.2", RTLD_LAZY | RTLD_LOCAL); + if (lib != NULL) { + LOAD(lib, XRR, QueryExtension, "XRRQueryExtension"); + LOAD(lib, XRR, QueryVersion, "XRRQueryVersion"); + LOAD(lib, XRR, GetMonitors, "XRRGetMonitors"); + LOAD(lib, XRR, FreeMonitors, "XRRFreeMonitors"); + LOAD(lib, XRR, GetScreenResourcesCurrent, "XRRGetScreenResourcesCurrent"); + LOAD(lib, XRR, FreeScreenResources, "XRRFreeScreenResources"); + LOAD(lib, XRR, GetOutputInfo, "XRRGetOutputInfo"); + LOAD(lib, XRR, FreeOutputInfo, "XRRFreeOutputInfo"); + LOAD(lib, XRR, GetCrtcInfo, "XRRGetCrtcInfo"); + LOAD(lib, XRR, FreeCrtcInfo, "XRRFreeCrtcInfo"); + LOAD(lib, XRR, GetOutputPrimary, "XRRGetOutputPrimary"); + XRR.loaded = XRR.QueryExtension && XRR.QueryVersion && XRR.GetScreenResourcesCurrent && + XRR.FreeScreenResources && XRR.GetOutputInfo && XRR.FreeOutputInfo && XRR.GetCrtcInfo && + XRR.FreeCrtcInfo && XRR.GetOutputPrimary; + } + + lib = dlopen("libXfixes.so.3", RTLD_LAZY | RTLD_LOCAL); + if (lib != NULL) { + LOAD(lib, XFX, QueryExtension, "XFixesQueryExtension"); + LOAD(lib, XFX, GetCursorImage, "XFixesGetCursorImage"); + XFX.loaded = XFX.QueryExtension && XFX.GetCursorImage; + } + + lib = dlopen("libXcomposite.so.1", RTLD_LAZY | RTLD_LOCAL); + if (lib != NULL) { + LOAD(lib, XCOMP, QueryExtension, "XCompositeQueryExtension"); + LOAD(lib, XCOMP, QueryVersion, "XCompositeQueryVersion"); + LOAD(lib, XCOMP, NameWindowPixmap, "XCompositeNameWindowPixmap"); + XCOMP.loaded = XCOMP.QueryExtension && XCOMP.QueryVersion && XCOMP.NameWindowPixmap; + } +} + +/* ------------------------------------------------------------------------ */ +/* X11 session: one connection, error handlers scoped to it */ +/* ------------------------------------------------------------------------ */ + +static pthread_mutex_t g_x11_lock = PTHREAD_MUTEX_INITIALIZER; +static Display *g_dpy = NULL; /* connection of the call in progress */ +static volatile int g_x_error = 0; /* last X error code on g_dpy */ +static volatile int g_x_io_dead = 0; /* g_dpy lost its server */ +static XErrorHandler g_prev_error = NULL; +static XIOErrorHandler g_prev_io = NULL; + +static int on_x_error(Display *dpy, XErrorEvent *event) { + if (dpy == g_dpy) { + g_x_error = event->error_code; + return 0; + } + return g_prev_error != NULL ? g_prev_error(dpy, event) : 0; +} + +static int on_x_io_error(Display *dpy) { + if (dpy == g_dpy) { + g_x_io_dead = 1; + return 0; /* Xlib then calls the per-display exit handler instead of exit() */ + } + return g_prev_io != NULL ? g_prev_io(dpy) : 0; +} + +static int on_x_io_exit(Display *dpy, void *data) { + (void)dpy; + (void)data; + g_x_io_dead = 1; + return 0; +} + +/* + * Opens a connection with the lock held. The IO error handler is only + * replaced when libX11 has XSetIOErrorExitHandler (1.7+): without it, a + * returning IO handler still ends in exit(), so there is nothing to gain. + */ +static Display *x_session_open(void) { + pthread_mutex_lock(&g_x11_lock); + Display *dpy = X.OpenDisplay(NULL); + if (dpy == NULL) { + pthread_mutex_unlock(&g_x11_lock); + return NULL; + } + g_dpy = dpy; + g_x_error = 0; + g_x_io_dead = 0; + g_prev_error = X.SetErrorHandler(on_x_error); + if (g_prev_error == on_x_error) g_prev_error = NULL; + if (X.SetIOErrorExitHandler != NULL && X.SetIOErrorHandler != NULL) { + g_prev_io = X.SetIOErrorHandler(on_x_io_error); + if (g_prev_io == on_x_io_error) g_prev_io = NULL; + X.SetIOErrorExitHandler(dpy, on_x_io_exit, NULL); + } + return dpy; +} + +static void x_session_close(Display *dpy) { + if (!g_x_io_dead) X.Sync(dpy, False); + /* Put the previous handlers back, unless someone replaced ours meanwhile. */ + XErrorHandler current = X.SetErrorHandler(g_prev_error); + if (current != on_x_error) X.SetErrorHandler(current); + if (X.SetIOErrorExitHandler != NULL && X.SetIOErrorHandler != NULL) { + XIOErrorHandler io = X.SetIOErrorHandler(g_prev_io); + if (io != on_x_io_error) X.SetIOErrorHandler(io); + } + X.CloseDisplay(dpy); + g_dpy = NULL; + g_prev_error = NULL; + g_prev_io = NULL; + pthread_mutex_unlock(&g_x11_lock); +} + +/* Flushes the request queue; returns the X error it raised (0 = none). */ +static int x_take_error(Display *dpy) { + if (!g_x_io_dead) X.Sync(dpy, False); + int error = g_x_error; + g_x_error = 0; + return error; +} + +/* ------------------------------------------------------------------------ */ +/* Displays */ +/* ------------------------------------------------------------------------ */ + +typedef struct { + char name[128]; + int x, y, w, h; + int primary; +} Monitor; + +typedef struct { + Monitor *items; + int count; + int capacity; +} MonitorList; + +static void monitors_add(MonitorList *list, const char *name, int x, int y, int w, int h, int primary) { + if (w <= 0 || h <= 0) return; + if (list->count == list->capacity) { + int capacity = list->capacity == 0 ? 8 : list->capacity * 2; + Monitor *items = realloc(list->items, (size_t)capacity * sizeof(Monitor)); + if (items == NULL) return; + list->items = items; + list->capacity = capacity; + } + Monitor *m = &list->items[list->count++]; + snprintf(m->name, sizeof(m->name), "%s", name != NULL && name[0] != '\0' ? name : "screen"); + m->x = x; + m->y = y; + m->w = w; + m->h = h; + m->primary = primary; +} + +static int has_randr(Display *dpy, int major_min, int minor_min) { + if (!XRR.loaded) return 0; + int event_base, error_base, major = 0, minor = 0; + if (!XRR.QueryExtension(dpy, &event_base, &error_base)) return 0; + if (!XRR.QueryVersion(dpy, &major, &minor)) return 0; + return major > major_min || (major == major_min && minor >= minor_min); +} + +static void list_monitors(Display *dpy, MonitorList *list) { + Window root = DefaultRootWindow(dpy); + + /* RandR 1.5 monitors: what desktops call a monitor (tiled outputs merged, setmonitor honoured). */ + if (XRR.GetMonitors != NULL && XRR.FreeMonitors != NULL && has_randr(dpy, 1, 5)) { + int n = 0; + XRRMonitorInfo *monitors = XRR.GetMonitors(dpy, root, True, &n); + if (x_take_error(dpy) == 0 && monitors != NULL) { + for (int i = 0; i < n; i++) { + char *name = monitors[i].name != None ? X.GetAtomName(dpy, monitors[i].name) : NULL; + monitors_add(list, name, monitors[i].x, monitors[i].y, monitors[i].width, monitors[i].height, + monitors[i].primary); + if (name != NULL) X.Free(name); + } + } + if (monitors != NULL) XRR.FreeMonitors(monitors); + if (list->count > 0) goto primary; + } + + /* RandR 1.3: one entry per connected output with a CRTC. */ + if (has_randr(dpy, 1, 3)) { + XRRScreenResources *res = XRR.GetScreenResourcesCurrent(dpy, root); + if (x_take_error(dpy) == 0 && res != NULL) { + RROutput primary = XRR.GetOutputPrimary(dpy, root); + for (int i = 0; i < res->noutput; i++) { + XRROutputInfo *output = XRR.GetOutputInfo(dpy, res, res->outputs[i]); + if (output == NULL) continue; + if (output->connection == RR_Connected && output->crtc != None) { + XRRCrtcInfo *crtc = XRR.GetCrtcInfo(dpy, res, output->crtc); + if (crtc != NULL) { + monitors_add(list, output->name, crtc->x, crtc->y, (int)crtc->width, (int)crtc->height, + res->outputs[i] == primary); + XRR.FreeCrtcInfo(crtc); + } + } + XRR.FreeOutputInfo(output); + } + } + if (res != NULL) XRR.FreeScreenResources(res); + x_take_error(dpy); + if (list->count > 0) goto primary; + } + + /* No RandR: the whole root window. */ + monitors_add(list, "screen", 0, 0, DisplayWidth(dpy, DefaultScreen(dpy)), DisplayHeight(dpy, DefaultScreen(dpy)), + 1); + +primary: + for (int i = 0; i < list->count; i++) { + if (list->items[i].primary) return; + } + if (list->count > 0) list->items[0].primary = 1; +} + +/* ------------------------------------------------------------------------ */ +/* Pixel conversion */ +/* ------------------------------------------------------------------------ */ + +typedef struct { + int shift; + int bits; +} Channel; + +static Channel channel_of(unsigned long mask) { + Channel c = {0, 0}; + if (mask == 0) return c; + while (((mask >> c.shift) & 1UL) == 0) c.shift++; + while (((mask >> (c.shift + c.bits)) & 1UL) != 0) c.bits++; + return c; +} + +/* + * Widens a channel to 8 bits by bit replication (5-bit 0b10110 -> 0b10110101), + * the expansion xwd, ImageMagick and pixman use, so captures of a 16-bit + * screen match what the rest of the toolchain reports. + */ +static inline uint32_t channel_value(unsigned long pixel, Channel c) { + if (c.bits == 0) return 0; + uint32_t max = (c.bits >= 32) ? 0xFFFFFFFFu : ((1u << c.bits) - 1u); + uint32_t v = (uint32_t)((pixel >> c.shift) & max); + if (c.bits >= 8) return v >> (c.bits - 8); + uint32_t out = 0; + int filled = 0; + while (filled < 8) { + out = (out << c.bits) | v; + filled += c.bits; + } + return out >> (filled - 8); +} + +typedef struct { + unsigned long red_mask, green_mask, blue_mask; + Colormap colormap; + int indexed; /* PseudoColor / StaticColor / GrayScale / StaticGray */ + int depth; +} PixelFormat; + +static PixelFormat format_of_visual(Visual *visual, int depth, Colormap colormap) { + PixelFormat f; + memset(&f, 0, sizeof(f)); + f.depth = depth; + f.colormap = colormap; + if (visual != NULL) { + f.red_mask = visual->red_mask; + f.green_mask = visual->green_mask; + f.blue_mask = visual->blue_mask; +#if defined(__cplusplus) || defined(c_plusplus) + int cls = visual->c_class; +#else + int cls = visual->class; +#endif + f.indexed = cls != TrueColor && cls != DirectColor; + } + return f; +} + +/* + * Colormap of an indexed visual as 0xAARRGGBB, one entry per pixel value; + * NULL for a TrueColor / DirectColor format. *failed is set when the format + * is indexed but cannot be resolved. + */ +static uint32_t *indexed_lut(Display *dpy, PixelFormat f, int *failed) { + *failed = 0; + if (!f.indexed) return NULL; + if (f.depth <= 0 || f.depth > 12 || f.colormap == None) { + *failed = 1; + return NULL; + } + int n = 1 << f.depth; + XColor *colors = calloc((size_t)n, sizeof(XColor)); + uint32_t *lut = calloc((size_t)n, sizeof(uint32_t)); + if (colors == NULL || lut == NULL) { + free(colors); + free(lut); + *failed = 1; + return NULL; + } + for (int i = 0; i < n; i++) colors[i].pixel = (unsigned long)i; + X.QueryColors(dpy, f.colormap, colors, n); + if (x_take_error(dpy) != 0) { + free(colors); + free(lut); + *failed = 1; + return NULL; + } + for (int i = 0; i < n; i++) { + lut[i] = 0xFF000000u | ((uint32_t)(colors[i].red >> 8) << 16) | ((uint32_t)(colors[i].green >> 8) << 8) | + (uint32_t)(colors[i].blue >> 8); + } + free(colors); + return lut; +} + +/* + * Writes the w x h image into out (0xAARRGGBB, opaque). Makes no X or JNI + * call, so it can run inside a primitive-array critical section. + */ +static int convert_image(XImage *image, PixelFormat f, const uint32_t *lut, int w, int h, jint *out) { + unsigned long rm = image->red_mask != 0 ? image->red_mask : f.red_mask; + unsigned long gm = image->green_mask != 0 ? image->green_mask : f.green_mask; + unsigned long bm = image->blue_mask != 0 ? image->blue_mask : f.blue_mask; + + if (f.indexed) { + if (lut == NULL) return 0; + unsigned long index_mask = (1UL << f.depth) - 1UL; + for (int y = 0; y < h; y++) { + for (int x = 0; x < w; x++) out[(size_t)y * w + x] = (jint)lut[XGetPixel(image, x, y) & index_mask]; + } + return 1; + } + + /* Fast path: 32 bpp, native little-endian x8r8g8b8. */ + if (image->bits_per_pixel == 32 && image->byte_order == LSBFirst && rm == 0xFF0000UL && gm == 0xFF00UL && + bm == 0xFFUL) { + for (int y = 0; y < h; y++) { + const uint32_t *row = (const uint32_t *)(image->data + (size_t)y * image->bytes_per_line); + jint *dst = out + (size_t)y * w; + for (int x = 0; x < w; x++) dst[x] = (jint)(row[x] | 0xFF000000u); + } + return 1; + } + + if (rm == 0 || gm == 0 || bm == 0) return 0; + Channel r = channel_of(rm), g = channel_of(gm), b = channel_of(bm); + for (int y = 0; y < h; y++) { + for (int x = 0; x < w; x++) { + unsigned long p = XGetPixel(image, x, y); + out[(size_t)y * w + x] = (jint)(0xFF000000u | (channel_value(p, r) << 16) | (channel_value(p, g) << 8) | + channel_value(p, b)); + } + } + return 1; +} + +static XFixesCursorImage *cursor_image(Display *dpy) { + if (!XFX.loaded) return NULL; + int event_base, error_base; + if (!XFX.QueryExtension(dpy, &event_base, &error_base)) return NULL; + XFixesCursorImage *cursor = XFX.GetCursorImage(dpy); + if (x_take_error(dpy) != 0) { + if (cursor != NULL) X.Free(cursor); + return NULL; + } + return cursor; +} + +/* Blends the cursor over out, whose top-left pixel is at (origin_x, origin_y) in root coordinates. */ +static void draw_cursor(const XFixesCursorImage *cursor, jint *out, int w, int h, int origin_x, int origin_y) { + int left = cursor->x - cursor->xhot - origin_x; + int top = cursor->y - cursor->yhot - origin_y; + for (int cy = 0; cy < cursor->height; cy++) { + int y = top + cy; + if (y < 0 || y >= h) continue; + for (int cx = 0; cx < cursor->width; cx++) { + int x = left + cx; + if (x < 0 || x >= w) continue; + /* XFixes pixels: premultiplied ARGB, one per unsigned long. */ + uint32_t src = (uint32_t)cursor->pixels[(size_t)cy * cursor->width + cx]; + uint32_t a = src >> 24; + if (a == 0) continue; + uint32_t dst = (uint32_t)out[(size_t)y * w + x]; + uint32_t inv = 255u - a; + uint32_t rr = ((src >> 16) & 0xFF) + (((dst >> 16) & 0xFF) * inv + 127) / 255; + uint32_t gg = ((src >> 8) & 0xFF) + (((dst >> 8) & 0xFF) * inv + 127) / 255; + uint32_t bb = (src & 0xFF) + ((dst & 0xFF) * inv + 127) / 255; + if (rr > 255) rr = 255; + if (gg > 255) gg = 255; + if (bb > 255) bb = 255; + out[(size_t)y * w + x] = (jint)(0xFF000000u | (rr << 16) | (gg << 8) | bb); + } + } +} + +/* + * Grabs (x, y, w, h) of drawable and converts it straight into a new Java + * array - no intermediate native buffer: on a 4K screen that is 33 MB per + * capture, which per-thread malloc arenas would otherwise keep resident. + * Returns ST_OK with *out set, or an error with a message. + */ +static int grab(JNIEnv *env, Display *dpy, Drawable drawable, PixelFormat f, int x, int y, int w, int h, + int include_cursor, int cursor_origin_x, int cursor_origin_y, jintArray *out, jobjectArray message) { + *out = NULL; + if ((int64_t)w * (int64_t)h > MAX_PIXELS) { + set_message(env, message, "%dx%d is too large for one image", w, h); + return ST_FAILED; + } + XImage *image = X.GetImage(dpy, drawable, x, y, (unsigned int)w, (unsigned int)h, AllPlanes, ZPixmap); + int error = x_take_error(dpy); + if (g_x_io_dead) { + if (image != NULL) XDestroyImage(image); + set_message(env, message, "X server connection lost"); + return ST_FAILED; + } + if (image == NULL || error != 0) { + if (image != NULL) XDestroyImage(image); + set_message(env, message, "XGetImage failed (X error %d)", error); + return ST_FAILED; + } + + /* Every X round trip happens before the critical section. */ + int lut_failed = 0; + uint32_t *lut = indexed_lut(dpy, f, &lut_failed); + XFixesCursorImage *cursor = include_cursor && !lut_failed ? cursor_image(dpy) : NULL; + int status = ST_FAILED; + jintArray array = NULL; + if (lut_failed) { + set_message(env, message, "Cannot resolve the colormap of a depth %d visual", f.depth); + goto done; + } + array = (*env)->NewIntArray(env, (jsize)((int64_t)w * h)); + if (array == NULL) goto done; /* OutOfMemoryError pending: the caller sees it */ + jint *pixels = (*env)->GetPrimitiveArrayCritical(env, array, NULL); + if (pixels == NULL) goto done; + int converted = convert_image(image, f, lut, w, h, pixels); + if (converted && cursor != NULL) draw_cursor(cursor, pixels, w, h, cursor_origin_x, cursor_origin_y); + (*env)->ReleasePrimitiveArrayCritical(env, array, pixels, 0); + if (!converted) { + set_message(env, message, "Unsupported pixel format (depth %d, %d bpp)", image->depth, image->bits_per_pixel); + goto done; + } + *out = array; + array = NULL; + status = ST_OK; + +done: + if (array != NULL) (*env)->DeleteLocalRef(env, array); + if (cursor != NULL) X.Free(cursor); + free(lut); + XDestroyImage(image); + return status; +} + +/* ------------------------------------------------------------------------ */ +/* JNI: X11 */ +/* ------------------------------------------------------------------------ */ + +EXPORT JNIEXPORT jint JNICALL +Java_dev_nucleusframework_screencapture_internal_NativeScreenCapture_nativeBackend(JNIEnv *env, jclass cls) { + (void)env; + (void)cls; + pthread_once(&g_x11_once, load_x11); + if (!X.loaded) return BACKEND_NONE; + Display *dpy = x_session_open(); + if (dpy == NULL) return BACKEND_NONE; + x_session_close(dpy); + return BACKEND_X11; +} + +EXPORT JNIEXPORT jint JNICALL Java_dev_nucleusframework_screencapture_internal_NativeScreenCapture_nativeListDisplays( + JNIEnv *env, jclass cls, jobject sink, jobjectArray message) { + (void)cls; + pthread_once(&g_x11_once, load_x11); + if (!X.loaded) { + set_message(env, message, "libX11 is not available"); + return ST_UNSUPPORTED; + } + jclass sink_class = (*env)->GetObjectClass(env, sink); + jmethodID add = (*env)->GetMethodID(env, sink_class, "add", + "(Ljava/lang/String;Ljava/lang/String;IIIIIIFZ)V"); + if (add == NULL) { + nucleus_jni_clear_exception(env); + set_message(env, message, "DisplayCollector.add not found"); + return ST_FAILED; + } + + Display *dpy = x_session_open(); + if (dpy == NULL) { + set_message(env, message, "Cannot open X display '%s'", getenv("DISPLAY") != NULL ? getenv("DISPLAY") : ""); + return ST_UNSUPPORTED; + } + MonitorList list = {0}; + list_monitors(dpy, &list); + x_session_close(dpy); + + int status = ST_OK; + for (int i = 0; i < list.count; i++) { + Monitor *m = &list.items[i]; + jstring id = (*env)->NewStringUTF(env, m->name); + if (id == NULL) { + nucleus_jni_clear_exception(env); + status = ST_FAILED; + break; + } + (*env)->CallVoidMethod(env, sink, add, id, id, m->x, m->y, m->w, m->h, m->w, m->h, (jfloat)1.0f, + (jboolean)(m->primary ? JNI_TRUE : JNI_FALSE)); + (*env)->DeleteLocalRef(env, id); + if (nucleus_jni_clear_exception(env)) { + set_message(env, message, "DisplayCollector.add threw"); + status = ST_FAILED; + break; + } + } + free(list.items); + return status; +} + +EXPORT JNIEXPORT jintArray JNICALL +Java_dev_nucleusframework_screencapture_internal_NativeScreenCapture_nativeCaptureDisplay( + JNIEnv *env, jclass cls, jstring display_id, jint rx, jint ry, jint rw, jint rh, jboolean include_cursor, + jintArray result, jobjectArray message) { + (void)cls; + set_result(env, result, ST_FAILED, 0, 0); + pthread_once(&g_x11_once, load_x11); + if (!X.loaded) { + set_result(env, result, ST_UNSUPPORTED, 0, 0); + set_message(env, message, "libX11 is not available"); + return NULL; + } + if (display_id == NULL) { + set_result(env, result, ST_DISPLAY_NOT_FOUND, 0, 0); + return NULL; + } + const char *id = (*env)->GetStringUTFChars(env, display_id, NULL); + if (id == NULL) return NULL; + + Display *dpy = x_session_open(); + if (dpy == NULL) { + (*env)->ReleaseStringUTFChars(env, display_id, id); + set_result(env, result, ST_UNSUPPORTED, 0, 0); + set_message(env, message, "Cannot open X display"); + return NULL; + } + + MonitorList list = {0}; + list_monitors(dpy, &list); + Monitor *monitor = NULL; + for (int i = 0; i < list.count; i++) { + if (strcmp(list.items[i].name, id) == 0) { + monitor = &list.items[i]; + break; + } + } + + jintArray pixels = NULL; + int status; + int out_w = 0, out_h = 0; + if (monitor == NULL) { + status = ST_DISPLAY_NOT_FOUND; + set_message(env, message, "No display '%s'", id); + } else { + /* Clip the monitor to the root window (a RandR monitor may extend past it), then the region to that. */ + int64_t root_w = DisplayWidth(dpy, DefaultScreen(dpy)), root_h = DisplayHeight(dpy, DefaultScreen(dpy)); + int64_t ml = monitor->x < 0 ? 0 : monitor->x, mt = monitor->y < 0 ? 0 : monitor->y; + int64_t mr = (int64_t)monitor->x + monitor->w, mb = (int64_t)monitor->y + monitor->h; + if (mr > root_w) mr = root_w; + if (mb > root_h) mb = root_h; + int64_t l = ml, t = mt, r = mr, b = mb; + if (rw > 0 && rh > 0) { + int64_t ql = (int64_t)monitor->x + rx, qt = (int64_t)monitor->y + ry; + int64_t qr = ql + rw, qb = qt + rh; + if (ql > l) l = ql; + if (qt > t) t = qt; + if (qr < r) r = qr; + if (qb < b) b = qb; + } + if (r <= l || b <= t) { + status = ST_INVALID_REGION; + set_message(env, message, "Region (%d, %d, %d, %d) does not intersect display '%s'", rx, ry, rw, rh, id); + } else { + Window root = DefaultRootWindow(dpy); + int screen = DefaultScreen(dpy); + PixelFormat f = format_of_visual(DefaultVisual(dpy, screen), DefaultDepth(dpy, screen), + DefaultColormap(dpy, screen)); + out_w = (int)(r - l); + out_h = (int)(b - t); + status = grab(env, dpy, root, f, (int)l, (int)t, out_w, out_h, include_cursor, (int)l, (int)t, &pixels, + message); + } + } + free(list.items); + x_session_close(dpy); + (*env)->ReleaseStringUTFChars(env, display_id, id); + if (status != ST_OK) { + set_result(env, result, status, 0, 0); + return NULL; + } + set_result(env, result, ST_OK, out_w, out_h); + return pixels; +} + +/* The window's content through its XComposite pixmap (valid even where it is covered). */ +static int grab_composite(JNIEnv *env, Display *dpy, Window window, XWindowAttributes *attrs, int include_cursor, + int origin_x, int origin_y, jintArray *out) { + *out = NULL; + if (!XCOMP.loaded) return 0; + int event_base, error_base, major = 0, minor = 0; + if (!XCOMP.QueryExtension(dpy, &event_base, &error_base)) return 0; + if (!XCOMP.QueryVersion(dpy, &major, &minor) || (major == 0 && minor < 2)) return 0; + Pixmap pixmap = XCOMP.NameWindowPixmap(dpy, window); + /* BadMatch when the window is not redirected (no compositing manager). */ + if (x_take_error(dpy) != 0 || pixmap == None) { + if (pixmap != None) { + X.FreePixmap(dpy, pixmap); + x_take_error(dpy); + } + return 0; + } + PixelFormat f = format_of_visual(attrs->visual, attrs->depth, attrs->colormap); + int bw = attrs->border_width; + int status = grab(env, dpy, pixmap, f, bw, bw, attrs->width, attrs->height, include_cursor, origin_x, origin_y, + out, NULL); + X.FreePixmap(dpy, pixmap); + x_take_error(dpy); + if (status != ST_OK && (*env)->ExceptionCheck(env)) return -1; + return status == ST_OK; +} + +EXPORT JNIEXPORT jintArray JNICALL +Java_dev_nucleusframework_screencapture_internal_NativeScreenCapture_nativeCaptureWindow( + JNIEnv *env, jclass cls, jlong window_id, jboolean include_cursor, jintArray result, jobjectArray message) { + (void)cls; + set_result(env, result, ST_FAILED, 0, 0); + pthread_once(&g_x11_once, load_x11); + if (!X.loaded) { + set_result(env, result, ST_UNSUPPORTED, 0, 0); + set_message(env, message, "libX11 is not available"); + return NULL; + } + if (window_id <= 0 || window_id > 0xFFFFFFFFLL) { + set_result(env, result, ST_WINDOW_NOT_FOUND, 0, 0); + set_message(env, message, "Not an X11 window id: %lld", (long long)window_id); + return NULL; + } + Display *dpy = x_session_open(); + if (dpy == NULL) { + set_result(env, result, ST_UNSUPPORTED, 0, 0); + set_message(env, message, "Cannot open X display"); + return NULL; + } + Window window = (Window)window_id; + Window root = DefaultRootWindow(dpy); + XWindowAttributes attrs; + memset(&attrs, 0, sizeof(attrs)); + Status ok = X.GetWindowAttributes(dpy, window, &attrs); + int error = x_take_error(dpy); + + jintArray pixels = NULL; + int status = ST_OK; + int out_w = 0, out_h = 0; + int root_x = 0, root_y = 0; + Window child; + if (!ok || error != 0) { + status = ST_WINDOW_NOT_FOUND; + set_message(env, message, "No window 0x%lx (X error %d)", (unsigned long)window, error); + } else if (attrs.map_state != IsViewable) { + status = ST_WINDOW_NOT_FOUND; + set_message(env, message, "Window 0x%lx is not viewable", (unsigned long)window); + } else if (!X.TranslateCoordinates(dpy, window, root, 0, 0, &root_x, &root_y, &child) || x_take_error(dpy) != 0) { + status = ST_WINDOW_NOT_FOUND; + set_message(env, message, "Window 0x%lx vanished", (unsigned long)window); + } else { + int composite = grab_composite(env, dpy, window, &attrs, include_cursor, root_x, root_y, &pixels); + if (composite < 0) { + status = ST_FAILED; /* pending Java exception */ + } else if (composite > 0) { + out_w = attrs.width; + out_h = attrs.height; + } else { + /* + * No composite pixmap: read the window itself, clipped to the part + * that is on the screen (XGetImage raises BadMatch past its edge). + */ + int64_t sw = DisplayWidth(dpy, DefaultScreen(dpy)), sh = DisplayHeight(dpy, DefaultScreen(dpy)); + int64_t l = root_x < 0 ? -(int64_t)root_x : 0, t = root_y < 0 ? -(int64_t)root_y : 0; + int64_t r = attrs.width, b = attrs.height; + if ((int64_t)root_x + r > sw) r = sw - root_x; + if ((int64_t)root_y + b > sh) b = sh - root_y; + if (r <= l || b <= t) { + status = ST_WINDOW_NOT_FOUND; + set_message(env, message, "Window 0x%lx is off screen", (unsigned long)window); + } else { + PixelFormat f = format_of_visual(attrs.visual, attrs.depth, attrs.colormap); + out_w = (int)(r - l); + out_h = (int)(b - t); + status = grab(env, dpy, window, f, (int)l, (int)t, out_w, out_h, include_cursor, + root_x + (int)l, root_y + (int)t, &pixels, message); + if (status != ST_OK && !(*env)->ExceptionCheck(env)) { + /* The window disappeared between the checks and the grab. */ + XWindowAttributes again; + if (!X.GetWindowAttributes(dpy, window, &again) || x_take_error(dpy) != 0) { + status = ST_WINDOW_NOT_FOUND; + } + } + } + } + } + x_session_close(dpy); + if (status != ST_OK) { + set_result(env, result, status, 0, 0); + return NULL; + } + set_result(env, result, ST_OK, out_w, out_h); + return pixels; +} + +EXPORT JNIEXPORT jint JNICALL +Java_dev_nucleusframework_screencapture_internal_NativeScreenCapture_nativePermissionStatus(JNIEnv *env, jclass cls) { + (void)env; + (void)cls; + return PERMISSION_NOT_REQUIRED; +} + +EXPORT JNIEXPORT jint JNICALL +Java_dev_nucleusframework_screencapture_internal_NativeScreenCapture_nativeRequestPermission(JNIEnv *env, jclass cls) { + (void)env; + (void)cls; + return PERMISSION_NOT_REQUIRED; +} + +/* ------------------------------------------------------------------------ */ +/* Portal (Wayland) */ +/* ------------------------------------------------------------------------ */ + +static struct { + int loaded; + void (*error_init)(DBusError *); + void (*error_free)(DBusError *); + dbus_bool_t (*error_is_set)(const DBusError *); + dbus_bool_t (*threads_init_default)(void); + DBusConnection *(*bus_get_private)(DBusBusType, DBusError *); + void (*connection_close)(DBusConnection *); + void (*connection_unref)(DBusConnection *); + void (*connection_set_exit_on_disconnect)(DBusConnection *, dbus_bool_t); + const char *(*bus_get_unique_name)(DBusConnection *); + void (*bus_add_match)(DBusConnection *, const char *, DBusError *); + DBusMessage *(*message_new_method_call)(const char *, const char *, const char *, const char *); + void (*message_unref)(DBusMessage *); + void (*message_set_no_reply)(DBusMessage *, dbus_bool_t); + void (*message_iter_init_append)(DBusMessage *, DBusMessageIter *); + dbus_bool_t (*message_iter_append_basic)(DBusMessageIter *, int, const void *); + dbus_bool_t (*message_iter_open_container)(DBusMessageIter *, int, const char *, DBusMessageIter *); + dbus_bool_t (*message_iter_close_container)(DBusMessageIter *, DBusMessageIter *); + dbus_bool_t (*message_iter_init)(DBusMessage *, DBusMessageIter *); + int (*message_iter_get_arg_type)(DBusMessageIter *); + void (*message_iter_get_basic)(DBusMessageIter *, void *); + dbus_bool_t (*message_iter_next)(DBusMessageIter *); + void (*message_iter_recurse)(DBusMessageIter *, DBusMessageIter *); + DBusMessage *(*connection_send_with_reply_and_block)(DBusConnection *, DBusMessage *, int, DBusError *); + dbus_bool_t (*connection_send)(DBusConnection *, DBusMessage *, dbus_uint32_t *); + void (*connection_flush)(DBusConnection *); + dbus_bool_t (*connection_read_write)(DBusConnection *, int); + DBusMessage *(*connection_pop_message)(DBusConnection *); + dbus_bool_t (*message_is_signal)(DBusMessage *, const char *, const char *); + const char *(*message_get_path)(DBusMessage *); +} D; + +/* GError, declared here so the build needs no GLib headers. */ +typedef struct { + uint32_t domain; + int code; + char *message; +} GError_; + +static struct { + int loaded; + void *(*new_from_file)(const char *, GError_ **); + int (*get_width)(const void *); + int (*get_height)(const void *); + int (*get_rowstride)(const void *); + int (*get_n_channels)(const void *); + int (*get_bits_per_sample)(const void *); + unsigned char *(*get_pixels)(const void *); + void (*object_unref)(void *); + void (*error_free)(GError_ *); +} PB; + +static pthread_once_t g_portal_once = PTHREAD_ONCE_INIT; +static pthread_mutex_t g_portal_lock = PTHREAD_MUTEX_INITIALIZER; +static unsigned long g_portal_counter = 0; + +static void load_portal(void) { + void *lib = dlopen("libdbus-1.so.3", RTLD_LAZY | RTLD_LOCAL); + if (lib != NULL) { + LOAD(lib, D, error_init, "dbus_error_init"); + LOAD(lib, D, error_free, "dbus_error_free"); + LOAD(lib, D, error_is_set, "dbus_error_is_set"); + LOAD(lib, D, threads_init_default, "dbus_threads_init_default"); + LOAD(lib, D, bus_get_private, "dbus_bus_get_private"); + LOAD(lib, D, connection_close, "dbus_connection_close"); + LOAD(lib, D, connection_unref, "dbus_connection_unref"); + LOAD(lib, D, connection_set_exit_on_disconnect, "dbus_connection_set_exit_on_disconnect"); + LOAD(lib, D, bus_get_unique_name, "dbus_bus_get_unique_name"); + LOAD(lib, D, bus_add_match, "dbus_bus_add_match"); + LOAD(lib, D, message_new_method_call, "dbus_message_new_method_call"); + LOAD(lib, D, message_unref, "dbus_message_unref"); + LOAD(lib, D, message_set_no_reply, "dbus_message_set_no_reply"); + LOAD(lib, D, message_iter_init_append, "dbus_message_iter_init_append"); + LOAD(lib, D, message_iter_append_basic, "dbus_message_iter_append_basic"); + LOAD(lib, D, message_iter_open_container, "dbus_message_iter_open_container"); + LOAD(lib, D, message_iter_close_container, "dbus_message_iter_close_container"); + LOAD(lib, D, message_iter_init, "dbus_message_iter_init"); + LOAD(lib, D, message_iter_get_arg_type, "dbus_message_iter_get_arg_type"); + LOAD(lib, D, message_iter_get_basic, "dbus_message_iter_get_basic"); + LOAD(lib, D, message_iter_next, "dbus_message_iter_next"); + LOAD(lib, D, message_iter_recurse, "dbus_message_iter_recurse"); + LOAD(lib, D, connection_send_with_reply_and_block, "dbus_connection_send_with_reply_and_block"); + LOAD(lib, D, connection_send, "dbus_connection_send"); + LOAD(lib, D, connection_flush, "dbus_connection_flush"); + LOAD(lib, D, connection_read_write, "dbus_connection_read_write"); + LOAD(lib, D, connection_pop_message, "dbus_connection_pop_message"); + LOAD(lib, D, message_is_signal, "dbus_message_is_signal"); + LOAD(lib, D, message_get_path, "dbus_message_get_path"); + D.loaded = D.error_init && D.error_free && D.error_is_set && D.threads_init_default && D.bus_get_private && + D.connection_close && D.connection_unref && D.connection_set_exit_on_disconnect && + D.bus_get_unique_name && D.bus_add_match && D.message_new_method_call && D.message_unref && + D.message_set_no_reply && D.message_iter_init_append && D.message_iter_append_basic && + D.message_iter_open_container && D.message_iter_close_container && D.message_iter_init && + D.message_iter_get_arg_type && D.message_iter_get_basic && D.message_iter_next && + D.message_iter_recurse && D.connection_send_with_reply_and_block && D.connection_send && + D.connection_flush && D.connection_read_write && D.connection_pop_message && + D.message_is_signal && D.message_get_path; + if (D.loaded) D.threads_init_default(); + } + + void *pixbuf = dlopen("libgdk_pixbuf-2.0.so.0", RTLD_LAZY | RTLD_LOCAL); + void *gobject = dlopen("libgobject-2.0.so.0", RTLD_LAZY | RTLD_LOCAL); + void *glib = dlopen("libglib-2.0.so.0", RTLD_LAZY | RTLD_LOCAL); + if (pixbuf != NULL && gobject != NULL && glib != NULL) { + LOAD(pixbuf, PB, new_from_file, "gdk_pixbuf_new_from_file"); + LOAD(pixbuf, PB, get_width, "gdk_pixbuf_get_width"); + LOAD(pixbuf, PB, get_height, "gdk_pixbuf_get_height"); + LOAD(pixbuf, PB, get_rowstride, "gdk_pixbuf_get_rowstride"); + LOAD(pixbuf, PB, get_n_channels, "gdk_pixbuf_get_n_channels"); + LOAD(pixbuf, PB, get_bits_per_sample, "gdk_pixbuf_get_bits_per_sample"); + LOAD(pixbuf, PB, get_pixels, "gdk_pixbuf_get_pixels"); + LOAD(gobject, PB, object_unref, "g_object_unref"); + LOAD(glib, PB, error_free, "g_error_free"); + PB.loaded = PB.new_from_file && PB.get_width && PB.get_height && PB.get_rowstride && PB.get_n_channels && + PB.get_bits_per_sample && PB.get_pixels && PB.object_unref && PB.error_free; + } +} + +static int64_t now_ms(void) { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (int64_t)ts.tv_sec * 1000 + ts.tv_nsec / 1000000; +} + +static int hex_value(char c) { + if (c >= '0' && c <= '9') return c - '0'; + if (c >= 'a' && c <= 'f') return c - 'a' + 10; + if (c >= 'A' && c <= 'F') return c - 'A' + 10; + return -1; +} + +/* file:///a%20b -> /a b; NULL for anything but a local file URI. Caller frees. */ +static char *path_of_uri(const char *uri) { + const char *p; + if (strncmp(uri, "file://", 7) != 0) return NULL; + p = uri + 7; + if (strncmp(p, "localhost/", 10) == 0) p += 9; + if (*p != '/') return NULL; + size_t n = strlen(p); + char *out = malloc(n + 1); + if (out == NULL) return NULL; + size_t o = 0; + for (size_t i = 0; i < n; i++) { + if (p[i] == '%' && i + 2 < n && hex_value(p[i + 1]) >= 0 && hex_value(p[i + 2]) >= 0) { + char c = (char)(hex_value(p[i + 1]) * 16 + hex_value(p[i + 2])); + if (c == '\0') { + free(out); + return NULL; + } + out[o++] = c; + i += 2; + } else if (p[i] == '?' || p[i] == '#') { + break; + } else { + out[o++] = p[i]; + } + } + out[o] = '\0'; + return out; +} + +/* Appends a{sv} entry key -> variant of basic type. */ +static int append_option(DBusMessageIter *dict, const char *key, int type, const char *signature, const void *value) { + DBusMessageIter entry, variant; + if (!D.message_iter_open_container(dict, DBUS_TYPE_DICT_ENTRY, NULL, &entry)) return 0; + if (!D.message_iter_append_basic(&entry, DBUS_TYPE_STRING, &key)) return 0; + if (!D.message_iter_open_container(&entry, DBUS_TYPE_VARIANT, signature, &variant)) return 0; + if (!D.message_iter_append_basic(&variant, type, value)) return 0; + if (!D.message_iter_close_container(&entry, &variant)) return 0; + return D.message_iter_close_container(dict, &entry); +} + +/* Parses Response(u response, a{sv} results); *uri is a malloc'd copy of results["uri"]. */ +static int parse_response(DBusMessage *msg, dbus_uint32_t *code, char **uri) { + DBusMessageIter iter, dict, entry, variant; + *uri = NULL; + if (!D.message_iter_init(msg, &iter) || D.message_iter_get_arg_type(&iter) != DBUS_TYPE_UINT32) return 0; + D.message_iter_get_basic(&iter, code); + if (!D.message_iter_next(&iter) || D.message_iter_get_arg_type(&iter) != DBUS_TYPE_ARRAY) return 1; + D.message_iter_recurse(&iter, &dict); + while (D.message_iter_get_arg_type(&dict) == DBUS_TYPE_DICT_ENTRY) { + const char *key = NULL; + D.message_iter_recurse(&dict, &entry); + if (D.message_iter_get_arg_type(&entry) == DBUS_TYPE_STRING) { + D.message_iter_get_basic(&entry, &key); + if (key != NULL && strcmp(key, "uri") == 0 && D.message_iter_next(&entry) && + D.message_iter_get_arg_type(&entry) == DBUS_TYPE_VARIANT) { + D.message_iter_recurse(&entry, &variant); + if (D.message_iter_get_arg_type(&variant) == DBUS_TYPE_STRING) { + const char *value = NULL; + D.message_iter_get_basic(&variant, &value); + if (value != NULL) { + free(*uri); + *uri = strdup(value); + } + } + } + } + D.message_iter_next(&dict); + } + return 1; +} + +static int add_match(DBusConnection *conn, const char *rule) { + DBusError err; + D.error_init(&err); + D.bus_add_match(conn, rule, &err); + int ok = !D.error_is_set(&err); + D.error_free(&err); + return ok; +} + +#define RESPONSE_RULE \ + "type='signal',sender='org.freedesktop.portal.Desktop',interface='org.freedesktop.portal.Request',member='Response'," + +/* + * Subscribes to Response signals before the call so none can be missed: on + * this connection's whole request namespace (a portal that ignores + * handle_token still builds its path there, and may answer before its reply + * has been read), else - buses before dbus 1.5 reject path_namespace - on + * the predicted path only. + */ +static void add_response_match(DBusConnection *conn, const char *sender, const char *expected) { + char rule[1024]; + snprintf(rule, sizeof(rule), RESPONSE_RULE "path_namespace='/org/freedesktop/portal/desktop/request/%s'", sender); + if (add_match(conn, rule)) return; + snprintf(rule, sizeof(rule), RESPONSE_RULE "path='%s'", expected); + add_match(conn, rule); +} + +static void add_path_match(DBusConnection *conn, const char *path) { + char rule[1024]; + snprintf(rule, sizeof(rule), RESPONSE_RULE "path='%s'", path); + add_match(conn, rule); +} + +/* Best effort: tell the portal we gave up on the request. */ +static void close_request(DBusConnection *conn, const char *path) { + DBusMessage *msg = + D.message_new_method_call("org.freedesktop.portal.Desktop", path, "org.freedesktop.portal.Request", "Close"); + if (msg == NULL) return; + D.message_set_no_reply(msg, TRUE); + D.connection_send(conn, msg, NULL); + D.connection_flush(conn); + D.message_unref(msg); +} + +/* Decodes path into a new Java array; ST_OK or an error with a message. */ +static int decode_file(JNIEnv *env, const char *path, jintArray *out, int *out_w, int *out_h, jobjectArray message) { + *out = NULL; + GError_ *error = NULL; + void *pixbuf = PB.new_from_file(path, &error); + if (pixbuf == NULL) { + set_message(env, message, "Cannot decode %s: %s", path, error != NULL && error->message ? error->message : "?"); + if (error != NULL) PB.error_free(error); + return ST_FAILED; + } + int w = PB.get_width(pixbuf), h = PB.get_height(pixbuf); + int stride = PB.get_rowstride(pixbuf), channels = PB.get_n_channels(pixbuf); + int bits = PB.get_bits_per_sample(pixbuf); + const unsigned char *data = PB.get_pixels(pixbuf); + if (w <= 0 || h <= 0 || bits != 8 || (channels != 3 && channels != 4) || data == NULL || + (int64_t)w * h > MAX_PIXELS) { + set_message(env, message, "Unsupported screenshot %dx%d, %d channels of %d bits", w, h, channels, bits); + PB.object_unref(pixbuf); + return ST_FAILED; + } + jint *pixels = malloc((size_t)w * (size_t)h * sizeof(jint)); + if (pixels == NULL) { + PB.object_unref(pixbuf); + set_message(env, message, "Out of native memory for %dx%d", w, h); + return ST_FAILED; + } + for (int y = 0; y < h; y++) { + const unsigned char *row = data + (size_t)y * stride; + for (int x = 0; x < w; x++) { + const unsigned char *p = row + (size_t)x * channels; + pixels[(size_t)y * w + x] = (jint)(0xFF000000u | ((uint32_t)p[0] << 16) | ((uint32_t)p[1] << 8) | p[2]); + } + } + PB.object_unref(pixbuf); + jintArray array = (*env)->NewIntArray(env, (jsize)((int64_t)w * h)); + if (array == NULL) { + free(pixels); + return ST_FAILED; + } + (*env)->SetIntArrayRegion(env, array, 0, (jsize)((int64_t)w * h), pixels); + free(pixels); + *out = array; + *out_w = w; + *out_h = h; + return ST_OK; +} + +static int is_unsupported_error(const char *name) { + return name != NULL && + (strcmp(name, DBUS_ERROR_SERVICE_UNKNOWN) == 0 || strcmp(name, DBUS_ERROR_UNKNOWN_METHOD) == 0 || + strcmp(name, "org.freedesktop.DBus.Error.UnknownObject") == 0 || + strcmp(name, "org.freedesktop.DBus.Error.UnknownInterface") == 0 || + strcmp(name, DBUS_ERROR_NAME_HAS_NO_OWNER) == 0 || strcmp(name, DBUS_ERROR_SPAWN_SERVICE_NOT_FOUND) == 0 || + strcmp(name, "org.freedesktop.DBus.Error.ServiceNotFound") == 0); +} + +static int portal_screenshot(JNIEnv *env, int interactive, int timeout_ms, jintArray *out, int *out_w, int *out_h, + jobjectArray message) { + DBusError err; + D.error_init(&err); + DBusConnection *conn = D.bus_get_private(DBUS_BUS_SESSION, &err); + if (conn == NULL) { + set_message(env, message, "No D-Bus session bus: %s", D.error_is_set(&err) ? err.message : "?"); + D.error_free(&err); + return ST_UNSUPPORTED; + } + D.connection_set_exit_on_disconnect(conn, FALSE); + + /* Predicted request path: /org/freedesktop/portal/desktop/request//. */ + const char *unique = D.bus_get_unique_name(conn); + char sender[256]; + size_t s = 0; + for (const char *c = unique != NULL ? unique : ""; *c != '\0' && s + 1 < sizeof(sender); c++) { + if (*c == ':') continue; + sender[s++] = *c == '.' ? '_' : *c; + } + sender[s] = '\0'; + char token[64]; + pthread_mutex_lock(&g_portal_lock); + unsigned long counter = ++g_portal_counter; + pthread_mutex_unlock(&g_portal_lock); + snprintf(token, sizeof(token), "nucleus_sc_%ld_%lu", (long)getpid(), counter); + char expected[512]; + snprintf(expected, sizeof(expected), "/org/freedesktop/portal/desktop/request/%s/%s", sender, token); + add_response_match(conn, sender, expected); + + int status = ST_FAILED; + char handle[512]; + snprintf(handle, sizeof(handle), "%s", expected); + int64_t deadline = now_ms() + (timeout_ms > 0 ? timeout_ms : 60000); + + DBusMessage *call = D.message_new_method_call("org.freedesktop.portal.Desktop", "/org/freedesktop/portal/desktop", + "org.freedesktop.portal.Screenshot", "Screenshot"); + if (call == NULL) { + set_message(env, message, "Out of memory building the portal call"); + goto done; + } + { + DBusMessageIter args, dict; + const char *parent = ""; + const char *token_ptr = token; + dbus_bool_t interactive_value = interactive ? TRUE : FALSE; + D.message_iter_init_append(call, &args); + int built = D.message_iter_append_basic(&args, DBUS_TYPE_STRING, &parent) && + D.message_iter_open_container(&args, DBUS_TYPE_ARRAY, "{sv}", &dict) && + append_option(&dict, "handle_token", DBUS_TYPE_STRING, "s", &token_ptr) && + append_option(&dict, "interactive", DBUS_TYPE_BOOLEAN, "b", &interactive_value) && + D.message_iter_close_container(&args, &dict); + if (!built) { + D.message_unref(call); + set_message(env, message, "Out of memory building the portal call"); + goto done; + } + } + int64_t call_timeout = deadline - now_ms(); + if (call_timeout < 1) call_timeout = 1; + if (call_timeout > 25000) call_timeout = 25000; + DBusMessage *reply = D.connection_send_with_reply_and_block(conn, call, (int)call_timeout, &err); + D.message_unref(call); + if (reply == NULL) { + if (is_unsupported_error(err.name)) { + status = ST_UNSUPPORTED; + set_message(env, message, "No screenshot portal: %s", err.message != NULL ? err.message : err.name); + } else if (err.name != NULL && (strcmp(err.name, DBUS_ERROR_NO_REPLY) == 0 || + strcmp(err.name, DBUS_ERROR_TIMEOUT) == 0)) { + status = ST_TIMEOUT; + set_message(env, message, "Screenshot portal did not answer"); + } else if (err.name != NULL && (strcmp(err.name, DBUS_ERROR_ACCESS_DENIED) == 0 || + strcmp(err.name, "org.freedesktop.portal.Error.NotAllowed") == 0)) { + status = ST_PERMISSION_DENIED; + set_message(env, message, "%s", err.message != NULL ? err.message : err.name); + } else { + status = ST_FAILED; + set_message(env, message, "Screenshot portal call failed: %s: %s", err.name != NULL ? err.name : "?", + err.message != NULL ? err.message : ""); + } + D.error_free(&err); + goto done; + } + { + DBusMessageIter iter; + if (D.message_iter_init(reply, &iter) && D.message_iter_get_arg_type(&iter) == DBUS_TYPE_OBJECT_PATH) { + const char *path = NULL; + D.message_iter_get_basic(&iter, &path); + /* Older portals ignore handle_token: listen on the path they actually use too. */ + if (path != NULL && strcmp(path, expected) != 0) { + snprintf(handle, sizeof(handle), "%s", path); + add_path_match(conn, handle); + } + } + D.message_unref(reply); + } + + for (;;) { + DBusMessage *msg; + while ((msg = D.connection_pop_message(conn)) != NULL) { + const char *path = D.message_get_path(msg); + if (D.message_is_signal(msg, "org.freedesktop.portal.Request", "Response") && path != NULL && + (strcmp(path, handle) == 0 || strcmp(path, expected) == 0)) { + dbus_uint32_t code = 2; + char *uri = NULL; + if (!parse_response(msg, &code, &uri)) { + status = ST_FAILED; + set_message(env, message, "Malformed portal response"); + } else if (code == 0) { + char *file = uri != NULL ? path_of_uri(uri) : NULL; + if (file == NULL) { + status = ST_FAILED; + set_message(env, message, "Portal returned no usable uri: %s", uri != NULL ? uri : "(none)"); + } else { + status = decode_file(env, file, out, out_w, out_h, message); + struct stat st; + if (status == ST_OK && lstat(file, &st) == 0 && S_ISREG(st.st_mode)) unlink(file); + free(file); + } + } else if (code == 1) { + status = ST_CANCELLED; + set_message(env, message, "The screenshot was cancelled"); + } else { + status = ST_FAILED; + set_message(env, message, "The screenshot portal failed (response %u)", (unsigned)code); + } + free(uri); + D.message_unref(msg); + goto done; + } + D.message_unref(msg); + } + int64_t remaining = deadline - now_ms(); + if (remaining <= 0) { + status = ST_TIMEOUT; + set_message(env, message, "The screenshot portal did not respond in %d ms", timeout_ms); + close_request(conn, handle); + goto done; + } + if (!D.connection_read_write(conn, (int)(remaining < 200 ? remaining : 200))) { + status = ST_FAILED; + set_message(env, message, "D-Bus connection lost"); + goto done; + } + } + +done: + D.connection_close(conn); + D.connection_unref(conn); + return status; +} + +EXPORT JNIEXPORT jintArray JNICALL +Java_dev_nucleusframework_screencapture_internal_NativeScreenCapture_nativePortalScreenshot( + JNIEnv *env, jclass cls, jboolean interactive, jint timeout_ms, jintArray result, jobjectArray message) { + (void)cls; + set_result(env, result, ST_FAILED, 0, 0); + pthread_once(&g_portal_once, load_portal); + if (!D.loaded) { + set_result(env, result, ST_UNSUPPORTED, 0, 0); + set_message(env, message, "libdbus-1 is not available"); + return NULL; + } + if (!PB.loaded) { + set_result(env, result, ST_UNSUPPORTED, 0, 0); + set_message(env, message, "gdk-pixbuf is not available to decode the screenshot"); + return NULL; + } + jintArray pixels = NULL; + int w = 0, h = 0; + int status = portal_screenshot(env, interactive, timeout_ms, &pixels, &w, &h, message); + if (status != ST_OK) { + set_result(env, result, status, 0, 0); + return NULL; + } + set_result(env, result, ST_OK, w, h); + return pixels; +} diff --git a/screen-capture/src/main/native/macos/NucleusScreenCapture.m b/screen-capture/src/main/native/macos/NucleusScreenCapture.m new file mode 100644 index 000000000..17dabefc1 --- /dev/null +++ b/screen-capture/src/main/native/macos/NucleusScreenCapture.m @@ -0,0 +1,583 @@ +// macOS backend of the screen-capture module. +// +// macOS 14+: ScreenCaptureKit (SCScreenshotManager). Earlier: Core Graphics display and window +// images, resolved with dlsym because the macOS 15 SDK marks them obsoleted. +// +// Pixels leave as jint 0xAARRGGBB: a CGBitmapContext with kCGImageAlphaPremultipliedFirst | +// kCGBitmapByteOrder32Little stores B, G, R, A bytes, which is that int on little-endian. + +#import +#import +#import +#import +#include +#include +#include +#include +#include +#include +#include "../../../../../native-common/nucleus_jni.h" + +#define STATUS_OK 0 +#define STATUS_UNSUPPORTED 1 +#define STATUS_PERMISSION_DENIED 2 +#define STATUS_DISPLAY_NOT_FOUND 3 +#define STATUS_WINDOW_NOT_FOUND 4 +#define STATUS_FAILED 6 +#define STATUS_TIMEOUT 7 +#define STATUS_INVALID_REGION 8 + +#define PERMISSION_GRANTED 0 +#define PERMISSION_DENIED 1 +#define PERMISSION_NOT_DETERMINED 2 + +#define BACKEND_NONE 0 +#define BACKEND_SCREEN_CAPTURE_KIT 2 +#define BACKEND_CORE_GRAPHICS 3 + +#define MAX_DISPLAYS 32 +#define CAPTURE_TIMEOUT_SECONDS 10.0 + +// SCStreamErrorUserDeclined, spelled out so the file also builds where the enum is missing. +#define SC_ERROR_USER_DECLINED (-3801) + +typedef CGImageRef (*CreateDisplayImageForRectFn)(CGDirectDisplayID, CGRect); +typedef CGImageRef (*CreateWindowListImageFn)(CGRect, CGWindowListOption, CGWindowID, CGWindowImageOption); + +static CreateDisplayImageForRectFn cgCreateDisplayImageForRect(void) { + static CreateDisplayImageForRectFn fn; + static dispatch_once_t once; + dispatch_once(&once, ^{ + fn = (CreateDisplayImageForRectFn)dlsym(RTLD_DEFAULT, "CGDisplayCreateImageForRect"); + }); + return fn; +} + +static CreateWindowListImageFn cgCreateWindowListImage(void) { + static CreateWindowListImageFn fn; + static dispatch_once_t once; + dispatch_once(&once, ^{ + fn = (CreateWindowListImageFn)dlsym(RTLD_DEFAULT, "CGWindowListCreateImage"); + }); + return fn; +} + +static BOOL hasScreenCaptureKit(void) { + if (@available(macOS 14.0, *)) { + return [SCScreenshotManager class] != nil; + } + return NO; +} + +// --------------------------------------------------------------------------------------------- +// JNI result helpers +// --------------------------------------------------------------------------------------------- + +static void setResult(JNIEnv *env, jintArray result, jint status, jint width, jint height) { + if (result == NULL || (*env)->GetArrayLength(env, result) < 3) return; + jint values[3] = {status, width, height}; + (*env)->SetIntArrayRegion(env, result, 0, 3, values); +} + +static void setMessage(JNIEnv *env, jobjectArray message, NSString *text) { + if (message == NULL || text == nil || (*env)->GetArrayLength(env, message) < 1) return; + jstring str = (*env)->NewStringUTF(env, text.UTF8String ?: ""); + if (str == NULL) { + nucleus_jni_clear_exception(env); + return; + } + (*env)->SetObjectArrayElement(env, message, 0, str); + (*env)->DeleteLocalRef(env, str); +} + +static jintArray fail(JNIEnv *env, jintArray result, jobjectArray message, jint status, NSString *text) { + setResult(env, result, status, 0, 0); + setMessage(env, message, text); + return NULL; +} + +// Draws [image] into an opaque BGRA buffer and hands it to Java as 0xAARRGGBB ints. +static jintArray imageToPixels(JNIEnv *env, CGImageRef image, jintArray result, jobjectArray message) { + size_t width = CGImageGetWidth(image); + size_t height = CGImageGetHeight(image); + if (width == 0 || height == 0 || (uint64_t)width * height > (uint64_t)INT32_MAX - 8) { + return fail(env, result, message, STATUS_FAILED, [NSString stringWithFormat:@"Unusable image size %zux%zu", width, height]); + } + size_t count = width * height; + uint32_t *buffer = calloc(count, sizeof(uint32_t)); + if (buffer == NULL) { + return fail(env, result, message, STATUS_FAILED, @"Out of memory"); + } + CGColorSpaceRef space = CGColorSpaceCreateWithName(kCGColorSpaceSRGB); + CGContextRef context = CGBitmapContextCreate( + buffer, width, height, 8, width * 4, space, + (CGBitmapInfo)kCGImageAlphaPremultipliedFirst | kCGBitmapByteOrder32Little); + CGColorSpaceRelease(space); + if (context == NULL) { + free(buffer); + return fail(env, result, message, STATUS_FAILED, @"CGBitmapContextCreate failed"); + } + CGContextSetBlendMode(context, kCGBlendModeCopy); + CGContextDrawImage(context, CGRectMake(0, 0, (CGFloat)width, (CGFloat)height), image); + CGContextRelease(context); + for (size_t i = 0; i < count; i++) buffer[i] |= 0xFF000000u; + + jintArray pixels = (*env)->NewIntArray(env, (jsize)count); + if (pixels == NULL) { + free(buffer); + nucleus_jni_clear_exception(env); + return fail(env, result, message, STATUS_FAILED, @"Cannot allocate the pixel array"); + } + (*env)->SetIntArrayRegion(env, pixels, 0, (jsize)count, (const jint *)buffer); + free(buffer); + setResult(env, result, STATUS_OK, (jint)width, (jint)height); + return pixels; +} + +// --------------------------------------------------------------------------------------------- +// ScreenCaptureKit plumbing +// --------------------------------------------------------------------------------------------- + +// Shared between a waiting caller and a completion handler that may outlive it (timeout). +@interface NucleusCaptureBox : NSObject +@property(nonatomic) BOOL done; +@property(nonatomic) BOOL abandoned; +@property(nonatomic, strong) NSError *error; +@property(nonatomic, strong) id value; +@property(nonatomic) CGImageRef image; +@end + +@implementation NucleusCaptureBox +- (void)dealloc { + if (_image != NULL) CGImageRelease(_image); +} +@end + +// Waits for [sem]. ScreenCaptureKit completes on its own queue, so a plain wait is safe on any +// thread — including the main thread, whose run loop is deliberately not pumped here: turning +// it would dispatch the app's own events re-entrantly from inside a capture call. +static BOOL waitFor(dispatch_semaphore_t sem) { + return dispatch_semaphore_wait(sem, dispatch_time(DISPATCH_TIME_NOW, (int64_t)(CAPTURE_TIMEOUT_SECONDS * NSEC_PER_SEC))) == 0; +} + +static BOOL isPermissionError(NSError *error) { + return error != nil && error.code == SC_ERROR_USER_DECLINED; +} + +// Fetches the shareable content; returns a status and fills [out] on success. +API_AVAILABLE(macos(14.0)) +static jint shareableContent(BOOL onScreenOnly, SCShareableContent **out, NSString **why) { + NucleusCaptureBox *box = [NucleusCaptureBox new]; + dispatch_semaphore_t sem = dispatch_semaphore_create(0); + [SCShareableContent getShareableContentExcludingDesktopWindows:NO + onScreenWindowsOnly:onScreenOnly + completionHandler:^(SCShareableContent *content, NSError *error) { + @synchronized(box) { + box.value = content; + box.error = error; + box.done = YES; + } + dispatch_semaphore_signal(sem); + }]; + if (!waitFor(sem)) { + @synchronized(box) { box.abandoned = YES; } + *why = @"Timed out waiting for SCShareableContent"; + return STATUS_TIMEOUT; + } + if (box.value == nil) { + *why = box.error.localizedDescription ?: @"SCShareableContent returned nothing"; + if (isPermissionError(box.error) || !CGPreflightScreenCaptureAccess()) return STATUS_PERMISSION_DENIED; + return STATUS_FAILED; + } + *out = box.value; + return STATUS_OK; +} + +// Captures with SCScreenshotManager; returns a status and a +1 image in [out]. +API_AVAILABLE(macos(14.0)) +static jint screenshot(SCContentFilter *filter, SCStreamConfiguration *config, CGImageRef *out, NSString **why) { + NucleusCaptureBox *box = [NucleusCaptureBox new]; + dispatch_semaphore_t sem = dispatch_semaphore_create(0); + [SCScreenshotManager captureImageWithFilter:filter + configuration:config + completionHandler:^(CGImageRef image, NSError *error) { + @synchronized(box) { + // The image is only valid for the handler's duration: retain it for the caller, + // unless the caller already gave up (the box then releases nothing it never got). + if (image != NULL && !box.abandoned) box.image = CGImageRetain(image); + box.error = error; + box.done = YES; + } + dispatch_semaphore_signal(sem); + }]; + if (!waitFor(sem)) { + @synchronized(box) { box.abandoned = YES; } + *why = @"Timed out waiting for SCScreenshotManager"; + return STATUS_TIMEOUT; + } + CGImageRef image = NULL; + NSError *error = nil; + @synchronized(box) { + image = box.image; + box.image = NULL; // ownership moves to the caller + error = box.error; + } + if (image == NULL) { + *why = error.localizedDescription ?: @"SCScreenshotManager returned no image"; + if (isPermissionError(error) || !CGPreflightScreenCaptureAccess()) return STATUS_PERMISSION_DENIED; + return STATUS_FAILED; + } + *out = image; + return STATUS_OK; +} + +API_AVAILABLE(macos(14.0)) +static void configureCommon(SCStreamConfiguration *config, BOOL includeCursor) { + config.showsCursor = includeCursor; + config.pixelFormat = kCVPixelFormatType_32BGRA; + config.colorSpaceName = kCGColorSpaceSRGB; + config.scalesToFit = NO; + config.captureResolution = SCCaptureResolutionBest; +} + +// --------------------------------------------------------------------------------------------- +// Displays +// --------------------------------------------------------------------------------------------- + +typedef struct { + CGRect bounds; // points, global coordinates + size_t widthPx; + size_t heightPx; +} DisplayGeometry; + +static BOOL displayGeometry(CGDirectDisplayID display, DisplayGeometry *out) { + CGRect bounds = CGDisplayBounds(display); + if (CGRectIsEmpty(bounds)) return NO; + size_t widthPx = 0; + size_t heightPx = 0; + CGDisplayModeRef mode = CGDisplayCopyDisplayMode(display); + if (mode != NULL) { + widthPx = CGDisplayModeGetPixelWidth(mode); + heightPx = CGDisplayModeGetPixelHeight(mode); + CGDisplayModeRelease(mode); + } + if (widthPx == 0 || heightPx == 0) { + widthPx = CGDisplayPixelsWide(display); + heightPx = CGDisplayPixelsHigh(display); + } + out->bounds = bounds; + out->widthPx = widthPx; + out->heightPx = heightPx; + return widthPx > 0 && heightPx > 0; +} + +static BOOL isActiveDisplay(CGDirectDisplayID display) { + CGDirectDisplayID displays[MAX_DISPLAYS]; + uint32_t count = 0; + if (CGGetActiveDisplayList(MAX_DISPLAYS, displays, &count) != kCGErrorSuccess) return NO; + for (uint32_t i = 0; i < count; i++) { + if (displays[i] == display) return YES; + } + return NO; +} + +static BOOL parseDisplayId(JNIEnv *env, jstring id, CGDirectDisplayID *out) { + if (id == NULL) return NO; + const char *chars = (*env)->GetStringUTFChars(env, id, NULL); + if (chars == NULL) { + nucleus_jni_clear_exception(env); + return NO; + } + char *end = NULL; + unsigned long long value = strtoull(chars, &end, 10); + BOOL ok = end != chars && *end == '\0' && value <= UINT32_MAX; + (*env)->ReleaseStringUTFChars(env, id, chars); + if (ok) *out = (CGDirectDisplayID)value; + return ok; +} + +JNIEXPORT jint JNICALL +Java_dev_nucleusframework_screencapture_internal_NativeScreenCapture_nativeBackend(JNIEnv *env, jclass clazz) { + (void)env; + (void)clazz; + if (hasScreenCaptureKit()) return BACKEND_SCREEN_CAPTURE_KIT; + if (cgCreateDisplayImageForRect() != NULL) return BACKEND_CORE_GRAPHICS; + return BACKEND_NONE; +} + +JNIEXPORT jint JNICALL +Java_dev_nucleusframework_screencapture_internal_NativeScreenCapture_nativeListDisplays( + JNIEnv *env, jclass clazz, jobject sink, jobjectArray message) { + (void)clazz; + @autoreleasepool { + CGDirectDisplayID displays[MAX_DISPLAYS]; + uint32_t count = 0; + CGError err = CGGetActiveDisplayList(MAX_DISPLAYS, displays, &count); + if (err != kCGErrorSuccess) { + setMessage(env, message, [NSString stringWithFormat:@"CGGetActiveDisplayList failed: %d", err]); + return STATUS_FAILED; + } + jclass sinkClass = (*env)->GetObjectClass(env, sink); + jmethodID add = (*env)->GetMethodID(env, sinkClass, "add", "(Ljava/lang/String;Ljava/lang/String;IIIIIIFZ)V"); + if (add == NULL) { + nucleus_jni_clear_exception(env); + (*env)->DeleteLocalRef(env, sinkClass); + setMessage(env, message, @"DisplayCollector.add not found"); + return STATUS_FAILED; + } + for (uint32_t i = 0; i < count; i++) { + CGDirectDisplayID display = displays[i]; + DisplayGeometry geometry; + if (!displayGeometry(display, &geometry)) continue; + // Names come from Core Graphics only: NSScreen is AppKit state owned by the main + // thread, and this runs on any thread. + NSString *name = CGDisplayIsBuiltin(display) + ? @"Built-in Display" + : [NSString stringWithFormat:@"Display %u", display]; + NSString *idText = [NSString stringWithFormat:@"%u", display]; + jstring jid = (*env)->NewStringUTF(env, idText.UTF8String); + jstring jname = (*env)->NewStringUTF(env, name.UTF8String); + if (jid == NULL || jname == NULL) { + nucleus_jni_clear_exception(env); + if (jid != NULL) (*env)->DeleteLocalRef(env, jid); + if (jname != NULL) (*env)->DeleteLocalRef(env, jname); + continue; + } + CGRect b = geometry.bounds; + jfloat scale = (jfloat)((CGFloat)geometry.widthPx / b.size.width); + (*env)->CallVoidMethod( + env, sink, add, jid, jname, + (jint)lround(b.origin.x), (jint)lround(b.origin.y), + (jint)lround(b.size.width), (jint)lround(b.size.height), + (jint)geometry.widthPx, (jint)geometry.heightPx, + scale, (jboolean)(CGDisplayIsMain(display) ? JNI_TRUE : JNI_FALSE)); + nucleus_jni_clear_exception(env); + (*env)->DeleteLocalRef(env, jid); + (*env)->DeleteLocalRef(env, jname); + } + (*env)->DeleteLocalRef(env, sinkClass); + return STATUS_OK; + } +} + +API_AVAILABLE(macos(14.0)) +static jintArray captureDisplayWithKit( + JNIEnv *env, CGDirectDisplayID displayId, CGRect regionPx, CGFloat scale, jboolean includeCursor, + jintArray result, jobjectArray message) { + NSString *why = nil; + SCShareableContent *content = nil; + jint status = shareableContent(YES, &content, &why); + if (status != STATUS_OK) return fail(env, result, message, status, why); + + SCDisplay *target = nil; + for (SCDisplay *display in content.displays) { + if (display.displayID == displayId) { + target = display; + break; + } + } + if (target == nil) { + return fail(env, result, message, STATUS_DISPLAY_NOT_FOUND, + [NSString stringWithFormat:@"ScreenCaptureKit does not list display %u", displayId]); + } + + SCContentFilter *filter = [[SCContentFilter alloc] initWithDisplay:target excludingWindows:@[]]; + SCStreamConfiguration *config = [SCStreamConfiguration new]; + configureCommon(config, includeCursor); + config.width = (size_t)regionPx.size.width; + config.height = (size_t)regionPx.size.height; + // sourceRect is in points, relative to the display. + config.sourceRect = CGRectMake(regionPx.origin.x / scale, regionPx.origin.y / scale, + regionPx.size.width / scale, regionPx.size.height / scale); + + CGImageRef image = NULL; + status = screenshot(filter, config, &image, &why); + if (status != STATUS_OK) return fail(env, result, message, status, why); + jintArray pixels = imageToPixels(env, image, result, message); + CGImageRelease(image); + return pixels; +} + +static jintArray captureDisplayWithCoreGraphics( + JNIEnv *env, CGDirectDisplayID displayId, CGRect regionPx, CGFloat scale, + jintArray result, jobjectArray message) { + CreateDisplayImageForRectFn create = cgCreateDisplayImageForRect(); + if (create == NULL) { + return fail(env, result, message, STATUS_UNSUPPORTED, @"CGDisplayCreateImageForRect is unavailable"); + } + // Without the permission Core Graphics silently returns the wallpaper alone. + if (!CGPreflightScreenCaptureAccess()) { + return fail(env, result, message, STATUS_PERMISSION_DENIED, @"Screen recording permission is not granted"); + } + CGRect points = CGRectMake(regionPx.origin.x / scale, regionPx.origin.y / scale, + regionPx.size.width / scale, regionPx.size.height / scale); + CGImageRef image = create(displayId, points); + if (image == NULL) { + return fail(env, result, message, STATUS_FAILED, @"CGDisplayCreateImageForRect returned nothing"); + } + jintArray pixels = imageToPixels(env, image, result, message); + CGImageRelease(image); + return pixels; +} + +JNIEXPORT jintArray JNICALL +Java_dev_nucleusframework_screencapture_internal_NativeScreenCapture_nativeCaptureDisplay( + JNIEnv *env, jclass clazz, jstring displayId, jint x, jint y, jint width, jint height, + jboolean includeCursor, jintArray result, jobjectArray message) { + (void)clazz; + @autoreleasepool { + CGDirectDisplayID display = 0; + if (!parseDisplayId(env, displayId, &display) || !isActiveDisplay(display)) { + return fail(env, result, message, STATUS_DISPLAY_NOT_FOUND, @"No active display with this id"); + } + DisplayGeometry geometry; + if (!displayGeometry(display, &geometry)) { + return fail(env, result, message, STATUS_DISPLAY_NOT_FOUND, @"The display reports no geometry"); + } + CGFloat scale = (CGFloat)geometry.widthPx / geometry.bounds.size.width; + if (!(scale > 0)) scale = 1; + + // Region in display pixels, clipped to the display; width <= 0 is the whole display. + long long left = 0; + long long top = 0; + long long right = (long long)geometry.widthPx; + long long bottom = (long long)geometry.heightPx; + if (width > 0 && height > 0) { + long long rx = x; + long long ry = y; + long long rr = rx + width; + long long rb = ry + height; + if (rx > left) left = rx; + if (ry > top) top = ry; + if (rr < right) right = rr; + if (rb < bottom) bottom = rb; + } + if (right <= left || bottom <= top) { + return fail(env, result, message, STATUS_INVALID_REGION, + [NSString stringWithFormat:@"Region does not intersect the %zux%zu display", geometry.widthPx, geometry.heightPx]); + } + CGRect regionPx = CGRectMake((CGFloat)left, (CGFloat)top, (CGFloat)(right - left), (CGFloat)(bottom - top)); + + if (hasScreenCaptureKit()) { + if (@available(macOS 14.0, *)) { + return captureDisplayWithKit(env, display, regionPx, scale, includeCursor, result, message); + } + } + return captureDisplayWithCoreGraphics(env, display, regionPx, scale, result, message); + } +} + +// --------------------------------------------------------------------------------------------- +// Windows +// --------------------------------------------------------------------------------------------- + +API_AVAILABLE(macos(14.0)) +static jintArray captureWindowWithKit( + JNIEnv *env, CGWindowID windowId, jboolean includeCursor, jintArray result, jobjectArray message) { + NSString *why = nil; + SCShareableContent *content = nil; + jint status = shareableContent(NO, &content, &why); + if (status != STATUS_OK) return fail(env, result, message, status, why); + + SCWindow *target = nil; + for (SCWindow *window in content.windows) { + if (window.windowID == windowId) { + target = window; + break; + } + } + if (target == nil) { + return fail(env, result, message, STATUS_WINDOW_NOT_FOUND, + [NSString stringWithFormat:@"ScreenCaptureKit does not list window %u", windowId]); + } + + SCContentFilter *filter = [[SCContentFilter alloc] initWithDesktopIndependentWindow:target]; + CGRect rect = filter.contentRect; + CGFloat scale = filter.pointPixelScale; + if (!(scale > 0)) scale = 1; + size_t w = (size_t)llround(rect.size.width * scale); + size_t h = (size_t)llround(rect.size.height * scale); + if (w == 0 || h == 0) { + return fail(env, result, message, STATUS_WINDOW_NOT_FOUND, @"The window has no visible area"); + } + SCStreamConfiguration *config = [SCStreamConfiguration new]; + configureCommon(config, includeCursor); + config.width = w; + config.height = h; + config.ignoreShadowsSingleWindow = YES; + + CGImageRef image = NULL; + status = screenshot(filter, config, &image, &why); + if (status != STATUS_OK) return fail(env, result, message, status, why); + jintArray pixels = imageToPixels(env, image, result, message); + CGImageRelease(image); + return pixels; +} + +static jintArray captureWindowWithCoreGraphics( + JNIEnv *env, CGWindowID windowId, jintArray result, jobjectArray message) { + CreateWindowListImageFn create = cgCreateWindowListImage(); + if (create == NULL) { + return fail(env, result, message, STATUS_UNSUPPORTED, @"CGWindowListCreateImage is unavailable"); + } + if (!CGPreflightScreenCaptureAccess()) { + return fail(env, result, message, STATUS_PERMISSION_DENIED, @"Screen recording permission is not granted"); + } + CGImageRef image = create(CGRectNull, kCGWindowListOptionIncludingWindow, windowId, + kCGWindowImageBoundsIgnoreFraming | kCGWindowImageBestResolution); + if (image == NULL || CGImageGetWidth(image) == 0 || CGImageGetHeight(image) == 0) { + if (image != NULL) CGImageRelease(image); + return fail(env, result, message, STATUS_WINDOW_NOT_FOUND, + [NSString stringWithFormat:@"No capturable window %u", windowId]); + } + jintArray pixels = imageToPixels(env, image, result, message); + CGImageRelease(image); + return pixels; +} + +JNIEXPORT jintArray JNICALL +Java_dev_nucleusframework_screencapture_internal_NativeScreenCapture_nativeCaptureWindow( + JNIEnv *env, jclass clazz, jlong windowId, jboolean includeCursor, jintArray result, jobjectArray message) { + (void)clazz; + @autoreleasepool { + if (windowId <= 0 || windowId > (jlong)UINT32_MAX) { + return fail(env, result, message, STATUS_WINDOW_NOT_FOUND, @"Not a CGWindowID"); + } + CGWindowID window = (CGWindowID)windowId; + if (hasScreenCaptureKit()) { + if (@available(macOS 14.0, *)) { + return captureWindowWithKit(env, window, includeCursor, result, message); + } + } + return captureWindowWithCoreGraphics(env, window, result, message); + } +} + +// --------------------------------------------------------------------------------------------- +// Permission +// --------------------------------------------------------------------------------------------- + +JNIEXPORT jint JNICALL +Java_dev_nucleusframework_screencapture_internal_NativeScreenCapture_nativePermissionStatus(JNIEnv *env, jclass clazz) { + (void)env; + (void)clazz; + // The public API cannot tell "denied" from "never asked" (that is private TCC state), so a + // missing grant reads as not determined: a request may still prompt. + return CGPreflightScreenCaptureAccess() ? PERMISSION_GRANTED : PERMISSION_NOT_DETERMINED; +} + +JNIEXPORT jint JNICALL +Java_dev_nucleusframework_screencapture_internal_NativeScreenCapture_nativeRequestPermission(JNIEnv *env, jclass clazz) { + (void)env; + (void)clazz; + return CGRequestScreenCaptureAccess() ? PERMISSION_GRANTED : PERMISSION_DENIED; +} + +JNIEXPORT jintArray JNICALL +Java_dev_nucleusframework_screencapture_internal_NativeScreenCapture_nativePortalScreenshot( + JNIEnv *env, jclass clazz, jboolean interactive, jint timeoutMs, jintArray result, jobjectArray message) { + (void)clazz; + (void)interactive; + (void)timeoutMs; + return fail(env, result, message, STATUS_UNSUPPORTED, @"The screenshot portal is Linux-only"); +} diff --git a/screen-capture/src/main/native/macos/build.sh b/screen-capture/src/main/native/macos/build.sh new file mode 100755 index 000000000..7409466ba --- /dev/null +++ b/screen-capture/src/main/native/macos/build.sh @@ -0,0 +1,69 @@ +#!/bin/bash +# Compiles NucleusScreenCapture.m into per-architecture dylibs (arm64 + x86_64). +# The outputs are placed in the JAR resources so they ship with the library. +# +# ScreenCaptureKit is weak-linked: it is used from macOS 14 on, and the library still loads +# (on the Core Graphics path) where the framework is missing. +# +# Prerequisites: Xcode command-line tools (clang) with a macOS 14+ SDK. +# Usage: ./build.sh + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +SRC="$SCRIPT_DIR/NucleusScreenCapture.m" +RESOURCE_DIR="$SCRIPT_DIR/../../resources/nucleus/native" +OUT_DIR_ARM64="$RESOURCE_DIR/darwin-aarch64" +OUT_DIR_X64="$RESOURCE_DIR/darwin-x64" + +# Detect JAVA_HOME for JNI headers +if [ -z "${JAVA_HOME:-}" ]; then + JAVA_HOME=$(/usr/libexec/java_home 2>/dev/null || true) +fi +if [ -z "${JAVA_HOME:-}" ]; then + echo "ERROR: JAVA_HOME not set and /usr/libexec/java_home failed." >&2 + exit 1 +fi + +JNI_INCLUDE="$JAVA_HOME/include" +JNI_INCLUDE_DARWIN="$JAVA_HOME/include/darwin" + +if [ ! -d "$JNI_INCLUDE" ]; then + echo "ERROR: JNI headers not found at $JNI_INCLUDE" >&2 + exit 1 +fi + +mkdir -p "$OUT_DIR_ARM64" "$OUT_DIR_X64" + +COMMON_FLAGS=( + -dynamiclib + -I"$JNI_INCLUDE" -I"$JNI_INCLUDE_DARWIN" + -framework Foundation + -framework CoreFoundation + -framework CoreGraphics + -framework CoreVideo + -weak_framework ScreenCaptureKit + -mmacosx-version-min=10.15 + -fobjc-arc + -Wall + -Werror=unguarded-availability-new + -Oz + -flto + -fvisibility=hidden + -Wl,-dead_strip + -Wl,-x +) + +# Compile for arm64 +clang -arch arm64 "${COMMON_FLAGS[@]}" \ + -o "$OUT_DIR_ARM64/libnucleus_screencapture.dylib" "$SRC" +strip -x "$OUT_DIR_ARM64/libnucleus_screencapture.dylib" + +# Compile for x86_64 +clang -arch x86_64 "${COMMON_FLAGS[@]}" \ + -o "$OUT_DIR_X64/libnucleus_screencapture.dylib" "$SRC" +strip -x "$OUT_DIR_X64/libnucleus_screencapture.dylib" + +echo "Built per-architecture dylibs:" +ls -lh "$OUT_DIR_ARM64/libnucleus_screencapture.dylib" +ls -lh "$OUT_DIR_X64/libnucleus_screencapture.dylib" diff --git a/screen-capture/src/main/native/windows/build.bat b/screen-capture/src/main/native/windows/build.bat new file mode 100644 index 000000000..9190b1ae5 --- /dev/null +++ b/screen-capture/src/main/native/windows/build.bat @@ -0,0 +1,123 @@ +@echo off +REM Compiles nucleus_screencapture_windows.c into per-architecture DLLs (x64 + ARM64). +REM The outputs are placed in the JAR resources so they ship with the library. +REM +REM Prerequisites: Visual Studio Build Tools (MSVC) with ARM64 support. +REM Usage: build.bat + +setlocal enabledelayedexpansion + +set "SCRIPT_DIR=%~dp0" +set "SRC=%SCRIPT_DIR%nucleus_screencapture_windows.c" +set "RESOURCE_DIR=%SCRIPT_DIR%..\..\resources\nucleus\native" +set "OUT_DIR_X64=%RESOURCE_DIR%\win32-x64" +set "OUT_DIR_ARM64=%RESOURCE_DIR%\win32-aarch64" + +REM Check JAVA_HOME +if "%JAVA_HOME%"=="" ( + echo ERROR: JAVA_HOME is not set. >&2 + exit /b 1 +) +if not exist "%JAVA_HOME%\include\jni.h" ( + echo ERROR: JNI headers not found at %JAVA_HOME%\include >&2 + exit /b 1 +) + +set "JNI_INCLUDE=%JAVA_HOME%\include" +set "JNI_INCLUDE_WIN32=%JAVA_HOME%\include\win32" + +REM Locate vcvarsall.bat +set "VCVARSALL=" +REM Prefer vswhere: resolves any installed VS version (incl. 18+ and previews). +set "VSWHERE=%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" +if exist "%VSWHERE%" ( + for /f "usebackq tokens=*" %%i in (`"%VSWHERE%" -latest -prerelease -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath`) do ( + if exist "%%i\VC\Auxiliary\Build\vcvarsall.bat" set "VCVARSALL=%%i\VC\Auxiliary\Build\vcvarsall.bat" + ) +) +REM Fallback: scan well-known install locations if vswhere did not resolve a path. +if "%VCVARSALL%"=="" ( + for %%v in (18 2022 2019 2017) do ( + for %%e in (Enterprise Professional Community BuildTools) do ( + if exist "C:\Program Files\Microsoft Visual Studio\%%v\%%e\VC\Auxiliary\Build\vcvarsall.bat" ( + set "VCVARSALL=C:\Program Files\Microsoft Visual Studio\%%v\%%e\VC\Auxiliary\Build\vcvarsall.bat" + goto :found_vc + ) + if exist "C:\Program Files (x86)\Microsoft Visual Studio\%%v\%%e\VC\Auxiliary\Build\vcvarsall.bat" ( + set "VCVARSALL=C:\Program Files (x86)\Microsoft Visual Studio\%%v\%%e\VC\Auxiliary\Build\vcvarsall.bat" + goto :found_vc + ) + ) + ) +) +:found_vc +if "%VCVARSALL%"=="" ( + echo ERROR: Could not locate vcvarsall.bat. Install Visual Studio Build Tools. >&2 + exit /b 1 +) + +echo Using vcvarsall.bat: %VCVARSALL% + +REM Create output directories +if not exist "%OUT_DIR_X64%" mkdir "%OUT_DIR_X64%" +if not exist "%OUT_DIR_ARM64%" mkdir "%OUT_DIR_ARM64%" + +REM ---- Compile x64 ---- +REM Use setlocal/endlocal to isolate vcvarsall environment per architecture, +REM preventing PATH accumulation that exceeds cmd.exe line length on CI. +echo. +echo === Building x64 DLL === +setlocal +call "%VCVARSALL%" x64 +if errorlevel 1 ( + echo ERROR: vcvarsall x64 failed >&2 + exit /b 1 +) + +cl /LD /O1 /GS- /nologo ^ + /I"%JNI_INCLUDE%" /I"%JNI_INCLUDE_WIN32%" ^ + "%SRC%" ^ + /Fe:"%OUT_DIR_X64%\nucleus_screencapture.dll" ^ + /link kernel32.lib user32.lib gdi32.lib dwmapi.lib +if errorlevel 1 ( + echo ERROR: x64 compilation failed >&2 + exit /b 1 +) +endlocal + +REM Clean up intermediate files +del /q "%OUT_DIR_X64%\*.obj" "%OUT_DIR_X64%\*.lib" "%OUT_DIR_X64%\*.exp" 2>nul + +REM ---- Compile ARM64 ---- +echo. +echo === Building ARM64 DLL === +setlocal +call "%VCVARSALL%" x64_arm64 +if errorlevel 1 ( + echo WARNING: vcvarsall x64_arm64 failed. ARM64 cross-compilation may not be available. >&2 + endlocal + goto :done +) + +cl /LD /O1 /GS- /nologo ^ + /I"%JNI_INCLUDE%" /I"%JNI_INCLUDE_WIN32%" ^ + "%SRC%" ^ + /Fe:"%OUT_DIR_ARM64%\nucleus_screencapture.dll" ^ + /link kernel32.lib user32.lib gdi32.lib dwmapi.lib +if errorlevel 1 ( + echo WARNING: ARM64 compilation failed. >&2 + endlocal + goto :done +) +endlocal + +REM Clean up intermediate files +del /q "%OUT_DIR_ARM64%\*.obj" "%OUT_DIR_ARM64%\*.lib" "%OUT_DIR_ARM64%\*.exp" 2>nul + +:done +echo. +echo Built DLLs: +if exist "%OUT_DIR_X64%\nucleus_screencapture.dll" echo %OUT_DIR_X64%\nucleus_screencapture.dll +if exist "%OUT_DIR_ARM64%\nucleus_screencapture.dll" echo %OUT_DIR_ARM64%\nucleus_screencapture.dll + +endlocal diff --git a/screen-capture/src/main/native/windows/nucleus_screencapture_windows.c b/screen-capture/src/main/native/windows/nucleus_screencapture_windows.c new file mode 100644 index 000000000..612442d6a --- /dev/null +++ b/screen-capture/src/main/native/windows/nucleus_screencapture_windows.c @@ -0,0 +1,588 @@ +/* + * Windows backend of dev.nucleusframework.screencapture. + * + * Displays are captured with GDI (BitBlt of the screen DC, CAPTUREBLT so layered windows + * are included) and windows with PrintWindow(PW_RENDERFULLCONTENT), which asks DWM for the + * window's redirection surface — occluded parts and DirectX/ANGLE content included. + * + * Every entry point runs in a per-monitor-v2 DPI context for the calling thread only: the + * rectangles, the cursor position and the pixels are then physical, whatever the process + * awareness is, and the thread's context is restored before returning. + */ + +#define WIN32_LEAN_AND_MEAN +#ifndef _WIN32_WINNT +#define _WIN32_WINNT 0x0A00 +#endif +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../../../../native-common/nucleus_jni.h" + +#define STATUS_OK 0 +#define STATUS_UNSUPPORTED 1 +#define STATUS_DISPLAY_NOT_FOUND 3 +#define STATUS_WINDOW_NOT_FOUND 4 +#define STATUS_FAILED 6 +#define STATUS_INVALID_REGION 8 + +#define PERMISSION_NOT_REQUIRED 3 +#define BACKEND_GDI 1 + +#define MAX_MONITORS 64 +/* Largest capture: keeps w * h * 4 inside a DWORD and w * h inside a jint. */ +#define MAX_PIXELS 0x1FFFFFFFLL + +#ifndef PW_RENDERFULLCONTENT +#define PW_RENDERFULLCONTENT 0x00000002 +#endif + +typedef DPI_AWARENESS_CONTEXT(WINAPI *SetThreadDpiAwarenessContextFn)(DPI_AWARENESS_CONTEXT); +typedef HRESULT(WINAPI *GetDpiForMonitorFn)(HMONITOR, int, UINT *, UINT *); + +static INIT_ONCE g_init_once = INIT_ONCE_STATIC_INIT; +static SetThreadDpiAwarenessContextFn g_set_thread_dpi; +static GetDpiForMonitorFn g_get_dpi_for_monitor; + +static BOOL CALLBACK resolve_functions(PINIT_ONCE once, PVOID param, PVOID *ctx) { + (void)once; + (void)param; + (void)ctx; + HMODULE user32 = GetModuleHandleW(L"user32.dll"); + if (user32 != NULL) { + g_set_thread_dpi = + (SetThreadDpiAwarenessContextFn)(void *)GetProcAddress(user32, "SetThreadDpiAwarenessContext"); + } + HMODULE shcore = LoadLibraryW(L"shcore.dll"); + if (shcore != NULL) { + g_get_dpi_for_monitor = (GetDpiForMonitorFn)(void *)GetProcAddress(shcore, "GetDpiForMonitor"); + } + return TRUE; +} + +static void ensure_init(void) { + InitOnceExecuteOnce(&g_init_once, resolve_functions, NULL, NULL); +} + +/* Switches the calling thread to per-monitor-v2 (v1 before Windows 10 1703); returns the old context. */ +static DPI_AWARENESS_CONTEXT enter_physical_pixels(void) { + ensure_init(); + if (g_set_thread_dpi == NULL) return NULL; + DPI_AWARENESS_CONTEXT old = g_set_thread_dpi(DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2); + if (old == NULL) old = g_set_thread_dpi(DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE); + return old; +} + +static void leave_physical_pixels(DPI_AWARENESS_CONTEXT old) { + if (old != NULL && g_set_thread_dpi != NULL) g_set_thread_dpi(old); +} + +/* ---------------------------------------------------------------- result plumbing */ + +static void set_result(JNIEnv *env, jintArray result, int status, int width, int height) { + jint values[3] = {status, width, height}; + (*env)->SetIntArrayRegion(env, result, 0, 3, values); +} + +static void set_message(JNIEnv *env, jobjectArray message, const char *text) { + if (message == NULL || text == NULL) return; + jstring s = (*env)->NewStringUTF(env, text); + if (s == NULL) { + nucleus_jni_clear_exception(env); + return; + } + (*env)->SetObjectArrayElement(env, message, 0, s); + (*env)->DeleteLocalRef(env, s); +} + +static void fail(JNIEnv *env, jintArray result, jobjectArray message, int status, const char *what) { + char buffer[256]; + DWORD error = GetLastError(); + if (error != 0) { + snprintf(buffer, sizeof(buffer), "%s (Win32 error %lu)", what, (unsigned long)error); + } else { + snprintf(buffer, sizeof(buffer), "%s", what); + } + set_result(env, result, status, 0, 0); + set_message(env, message, buffer); +} + +/* ---------------------------------------------------------------- monitors */ + +typedef struct { + RECT rect; + WCHAR device[CCHDEVICENAME]; + BOOL primary; + UINT dpi; +} Monitor; + +typedef struct { + Monitor items[MAX_MONITORS]; + int count; +} MonitorList; + +static BOOL CALLBACK collect_monitor(HMONITOR handle, HDC dc, LPRECT rect, LPARAM param) { + (void)dc; + (void)rect; + MonitorList *list = (MonitorList *)param; + if (list->count >= MAX_MONITORS) return FALSE; + MONITORINFOEXW info; + ZeroMemory(&info, sizeof(info)); + info.cbSize = sizeof(info); + if (!GetMonitorInfoW(handle, (MONITORINFO *)&info)) return TRUE; + Monitor *m = &list->items[list->count++]; + m->rect = info.rcMonitor; + lstrcpynW(m->device, info.szDevice, CCHDEVICENAME); + m->primary = (info.dwFlags & MONITORINFOF_PRIMARY) != 0; + m->dpi = USER_DEFAULT_SCREEN_DPI; + if (g_get_dpi_for_monitor != NULL) { + UINT dx = 0, dy = 0; + if (SUCCEEDED(g_get_dpi_for_monitor(handle, 0 /* MDT_EFFECTIVE_DPI */, &dx, &dy)) && dx > 0) m->dpi = dx; + } + return TRUE; +} + +/* Must run in the physical-pixel context. */ +static void enumerate_monitors(MonitorList *list) { + list->count = 0; + EnumDisplayMonitors(NULL, NULL, collect_monitor, (LPARAM)list); +} + +/* The monitor's friendly name ("DELL U2720Q") through the display configuration API. */ +static BOOL friendly_name(const WCHAR *device, WCHAR *out, int capacity) { + UINT32 path_count = 0, mode_count = 0; + if (GetDisplayConfigBufferSizes(QDC_ONLY_ACTIVE_PATHS, &path_count, &mode_count) != ERROR_SUCCESS) return FALSE; + if (path_count == 0) return FALSE; + DISPLAYCONFIG_PATH_INFO *paths = (DISPLAYCONFIG_PATH_INFO *)calloc(path_count, sizeof(*paths)); + DISPLAYCONFIG_MODE_INFO *modes = (DISPLAYCONFIG_MODE_INFO *)calloc(mode_count > 0 ? mode_count : 1, sizeof(*modes)); + BOOL found = FALSE; + if (paths != NULL && modes != NULL && + QueryDisplayConfig(QDC_ONLY_ACTIVE_PATHS, &path_count, paths, &mode_count, modes, NULL) == ERROR_SUCCESS) { + for (UINT32 i = 0; i < path_count && !found; i++) { + DISPLAYCONFIG_SOURCE_DEVICE_NAME source; + ZeroMemory(&source, sizeof(source)); + source.header.type = DISPLAYCONFIG_DEVICE_INFO_GET_SOURCE_NAME; + source.header.size = sizeof(source); + source.header.adapterId = paths[i].sourceInfo.adapterId; + source.header.id = paths[i].sourceInfo.id; + if (DisplayConfigGetDeviceInfo(&source.header) != ERROR_SUCCESS) continue; + if (lstrcmpiW(source.viewGdiDeviceName, device) != 0) continue; + DISPLAYCONFIG_TARGET_DEVICE_NAME target; + ZeroMemory(&target, sizeof(target)); + target.header.type = DISPLAYCONFIG_DEVICE_INFO_GET_TARGET_NAME; + target.header.size = sizeof(target); + target.header.adapterId = paths[i].targetInfo.adapterId; + target.header.id = paths[i].targetInfo.id; + if (DisplayConfigGetDeviceInfo(&target.header) == ERROR_SUCCESS && target.monitorFriendlyDeviceName[0] != 0) { + lstrcpynW(out, target.monitorFriendlyDeviceName, capacity); + found = TRUE; + } + } + } + free(paths); + free(modes); + return found; +} + +/* ---------------------------------------------------------------- pixels */ + +typedef struct { + HDC screen; + HDC memory; + HBITMAP bitmap; + HGDIOBJ previous; + uint32_t *bits; + int width; + int height; +} Surface; + +static BOOL surface_open(Surface *s, int width, int height) { + ZeroMemory(s, sizeof(*s)); + s->width = width; + s->height = height; + s->screen = GetDC(NULL); + if (s->screen == NULL) return FALSE; + s->memory = CreateCompatibleDC(s->screen); + if (s->memory == NULL) return FALSE; + BITMAPINFO info; + ZeroMemory(&info, sizeof(info)); + info.bmiHeader.biSize = sizeof(BITMAPINFOHEADER); + info.bmiHeader.biWidth = width; + info.bmiHeader.biHeight = -height; /* top-down */ + info.bmiHeader.biPlanes = 1; + info.bmiHeader.biBitCount = 32; + info.bmiHeader.biCompression = BI_RGB; + void *bits = NULL; + s->bitmap = CreateDIBSection(s->screen, &info, DIB_RGB_COLORS, &bits, NULL, 0); + if (s->bitmap == NULL || bits == NULL) return FALSE; + s->bits = (uint32_t *)bits; + s->previous = SelectObject(s->memory, s->bitmap); + return TRUE; +} + +static void surface_close(Surface *s) { + if (s->memory != NULL && s->previous != NULL) SelectObject(s->memory, s->previous); + if (s->bitmap != NULL) DeleteObject(s->bitmap); + if (s->memory != NULL) DeleteDC(s->memory); + if (s->screen != NULL) ReleaseDC(NULL, s->screen); + ZeroMemory(s, sizeof(*s)); +} + +/* Draws the cursor as it is on screen, [origin] being the surface's screen position. */ +static void draw_cursor(HDC dc, int origin_x, int origin_y) { + CURSORINFO cursor; + ZeroMemory(&cursor, sizeof(cursor)); + cursor.cbSize = sizeof(cursor); + if (!GetCursorInfo(&cursor) || !(cursor.flags & CURSOR_SHOWING) || cursor.hCursor == NULL) return; + int hot_x = 0, hot_y = 0; + ICONINFO icon; + if (GetIconInfo(cursor.hCursor, &icon)) { + hot_x = (int)icon.xHotspot; + hot_y = (int)icon.yHotspot; + if (icon.hbmMask != NULL) DeleteObject(icon.hbmMask); + if (icon.hbmColor != NULL) DeleteObject(icon.hbmColor); + } + DrawIconEx(dc, cursor.ptScreenPos.x - hot_x - origin_x, cursor.ptScreenPos.y - hot_y - origin_y, cursor.hCursor, + 0, 0, 0, NULL, DI_NORMAL); +} + +/* + * Copies [width] x [height] pixels starting at ([x], [y]) of the surface into a new jintArray, + * alpha forced opaque (GDI leaves it undefined). + */ +static jintArray to_java(JNIEnv *env, const Surface *s, int x, int y, int width, int height) { + jintArray array = (*env)->NewIntArray(env, width * height); + if (array == NULL) return NULL; /* OutOfMemoryError pending */ + for (int row = 0; row < height; row++) { + uint32_t *line = s->bits + (size_t)(y + row) * (size_t)s->width + (size_t)x; + for (int col = 0; col < width; col++) line[col] |= 0xFF000000u; + (*env)->SetIntArrayRegion(env, array, row * width, width, (const jint *)line); + } + return array; +} + +static BOOL too_large(int width, int height) { + return (long long)width * (long long)height > MAX_PIXELS; +} + +/* Blits the screen rectangle into a new surface; FALSE on failure. */ +static BOOL blit_screen(Surface *s, int left, int top, int width, int height, BOOL cursor) { + if (!surface_open(s, width, height)) return FALSE; + if (!BitBlt(s->memory, 0, 0, width, height, s->screen, left, top, SRCCOPY | CAPTUREBLT)) return FALSE; + if (cursor) draw_cursor(s->memory, left, top); + GdiFlush(); + return TRUE; +} + +/* ---------------------------------------------------------------- PrintWindow off-thread */ + +/* + * PrintWindow sends WM_PRINT to the window's thread and waits for it without a timeout: a + * window whose thread does not pump (busy, or blocked on the very thread that asked for the + * capture) would block the caller for good. It therefore runs on a worker thread; when it + * does not finish in time the caller falls back to the screen and abandons the job, which the + * worker frees whenever PrintWindow eventually returns. + */ +#define PRINT_WINDOW_TIMEOUT_MS 2000 + +typedef struct { + volatile LONG refs; + HWND hwnd; + int width; + int height; + BOOL cursor; + int origin_x; + int origin_y; + BOOL ok; + HANDLE done; + Surface surface; +} PrintJob; + +static void print_job_release(PrintJob *job) { + if (InterlockedDecrement(&job->refs) != 0) return; + surface_close(&job->surface); + if (job->done != NULL) CloseHandle(job->done); + free(job); +} + +static void print_job_run(PrintJob *job) { + if (surface_open(&job->surface, job->width, job->height) && + PrintWindow(job->hwnd, job->surface.memory, PW_RENDERFULLCONTENT)) { + if (job->cursor) draw_cursor(job->surface.memory, job->origin_x, job->origin_y); + GdiFlush(); + job->ok = TRUE; + } +} + +static unsigned __stdcall print_job_thread(void *param) { + PrintJob *job = (PrintJob *)param; + DPI_AWARENESS_CONTEXT old = enter_physical_pixels(); + print_job_run(job); + leave_physical_pixels(old); + SetEvent(job->done); + print_job_release(job); + return 0; +} + +/* The printed window, or NULL (failed or timed out); release the job with print_job_release. */ +static PrintJob *print_window(HWND hwnd, int width, int height, BOOL cursor, int origin_x, int origin_y) { + PrintJob *job = (PrintJob *)calloc(1, sizeof(PrintJob)); + if (job == NULL) return NULL; + job->refs = 1; + job->hwnd = hwnd; + job->width = width; + job->height = height; + job->cursor = cursor; + job->origin_x = origin_x; + job->origin_y = origin_y; + + /* The window's own thread would wait on itself: print inline, the message is a direct call. */ + if (GetWindowThreadProcessId(hwnd, NULL) == GetCurrentThreadId()) { + print_job_run(job); + } else { + job->done = CreateEventW(NULL, TRUE, FALSE, NULL); + if (job->done == NULL) { + print_job_release(job); + return NULL; + } + InterlockedIncrement(&job->refs); + HANDLE thread = (HANDLE)_beginthreadex(NULL, 0, print_job_thread, job, 0, NULL); + if (thread == NULL) { + InterlockedDecrement(&job->refs); + print_job_release(job); + return NULL; + } + CloseHandle(thread); + if (WaitForSingleObject(job->done, PRINT_WINDOW_TIMEOUT_MS) != WAIT_OBJECT_0) { + print_job_release(job); + return NULL; + } + } + if (!job->ok) { + print_job_release(job); + return NULL; + } + return job; +} + +/* ---------------------------------------------------------------- JNI */ + +JNIEXPORT jint JNICALL +Java_dev_nucleusframework_screencapture_internal_NativeScreenCapture_nativeBackend(JNIEnv *env, jclass cls) { + (void)env; + (void)cls; + return BACKEND_GDI; +} + +JNIEXPORT jint JNICALL +Java_dev_nucleusframework_screencapture_internal_NativeScreenCapture_nativeListDisplays(JNIEnv *env, jclass cls, + jobject sink, + jobjectArray message) { + (void)cls; + jclass sink_class = (*env)->GetObjectClass(env, sink); + jmethodID add = (*env)->GetMethodID(env, sink_class, "add", "(Ljava/lang/String;Ljava/lang/String;IIIIIIFZ)V"); + (*env)->DeleteLocalRef(env, sink_class); + if (add == NULL) { + nucleus_jni_clear_exception(env); + set_message(env, message, "DisplayCollector.add not found"); + return STATUS_FAILED; + } + + MonitorList *list = (MonitorList *)calloc(1, sizeof(MonitorList)); + if (list == NULL) return STATUS_FAILED; + DPI_AWARENESS_CONTEXT old = enter_physical_pixels(); + enumerate_monitors(list); + leave_physical_pixels(old); + + int status = STATUS_OK; + for (int i = 0; i < list->count; i++) { + const Monitor *m = &list->items[i]; + WCHAR name[128]; + if (!friendly_name(m->device, name, 128)) lstrcpynW(name, m->device, 128); + jstring id = (*env)->NewString(env, (const jchar *)m->device, (jsize)lstrlenW(m->device)); + jstring label = (*env)->NewString(env, (const jchar *)name, (jsize)lstrlenW(name)); + if (id == NULL || label == NULL) { + status = STATUS_FAILED; + break; /* OutOfMemoryError pending: let it propagate */ + } + int width = m->rect.right - m->rect.left; + int height = m->rect.bottom - m->rect.top; + (*env)->CallVoidMethod(env, sink, add, id, label, (jint)m->rect.left, (jint)m->rect.top, (jint)width, + (jint)height, (jint)width, (jint)height, (jfloat)m->dpi / 96.0f, + (jboolean)(m->primary ? JNI_TRUE : JNI_FALSE)); + (*env)->DeleteLocalRef(env, id); + (*env)->DeleteLocalRef(env, label); + if (nucleus_jni_clear_exception(env)) { + set_message(env, message, "DisplayCollector.add threw"); + status = STATUS_FAILED; + break; + } + } + free(list); + return status; +} + +JNIEXPORT jintArray JNICALL +Java_dev_nucleusframework_screencapture_internal_NativeScreenCapture_nativeCaptureDisplay( + JNIEnv *env, jclass cls, jstring display_id, jint x, jint y, jint width, jint height, jboolean include_cursor, + jintArray result, jobjectArray message) { + (void)cls; + WCHAR device[CCHDEVICENAME]; + const jchar *chars = (*env)->GetStringChars(env, display_id, NULL); + if (chars == NULL) return NULL; + jsize length = (*env)->GetStringLength(env, display_id); + if (length >= CCHDEVICENAME) length = CCHDEVICENAME - 1; + memcpy(device, chars, (size_t)length * sizeof(WCHAR)); + device[length] = 0; + (*env)->ReleaseStringChars(env, display_id, chars); + + MonitorList *list = (MonitorList *)calloc(1, sizeof(MonitorList)); + if (list == NULL) { + fail(env, result, message, STATUS_FAILED, "Out of memory"); + return NULL; + } + DPI_AWARENESS_CONTEXT old = enter_physical_pixels(); + enumerate_monitors(list); + const Monitor *monitor = NULL; + for (int i = 0; i < list->count; i++) { + if (lstrcmpiW(list->items[i].device, device) == 0) { + monitor = &list->items[i]; + break; + } + } + + jintArray pixels = NULL; + if (monitor == NULL) { + SetLastError(0); + fail(env, result, message, STATUS_DISPLAY_NOT_FOUND, "No such display"); + } else { + int display_width = monitor->rect.right - monitor->rect.left; + int display_height = monitor->rect.bottom - monitor->rect.top; + long long left = 0, top = 0, right = display_width, bottom = display_height; + if (width > 0 && height > 0) { + left = x > 0 ? x : 0; + top = y > 0 ? y : 0; + right = (long long)x + width < display_width ? (long long)x + width : display_width; + bottom = (long long)y + height < display_height ? (long long)y + height : display_height; + } + int w = (int)(right - left); + int h = (int)(bottom - top); + if (right <= left || bottom <= top) { + SetLastError(0); + fail(env, result, message, STATUS_INVALID_REGION, "Region does not intersect the display"); + } else if (too_large(w, h)) { + SetLastError(0); + fail(env, result, message, STATUS_FAILED, "Region too large"); + } else { + Surface surface; + ZeroMemory(&surface, sizeof(surface)); + if (blit_screen(&surface, monitor->rect.left + (int)left, monitor->rect.top + (int)top, w, h, + include_cursor)) { + pixels = to_java(env, &surface, 0, 0, w, h); + if (pixels != NULL) set_result(env, result, STATUS_OK, w, h); + } else { + fail(env, result, message, STATUS_FAILED, "BitBlt failed (secure desktop or session locked?)"); + } + surface_close(&surface); + } + } + leave_physical_pixels(old); + free(list); + return pixels; +} + +JNIEXPORT jintArray JNICALL +Java_dev_nucleusframework_screencapture_internal_NativeScreenCapture_nativeCaptureWindow(JNIEnv *env, jclass cls, + jlong window_id, + jboolean include_cursor, + jintArray result, + jobjectArray message) { + (void)cls; + HWND hwnd = (HWND)(intptr_t)window_id; + SetLastError(0); + if (hwnd == NULL || !IsWindow(hwnd)) { + fail(env, result, message, STATUS_WINDOW_NOT_FOUND, "No such window"); + return NULL; + } + if (IsIconic(hwnd) || !IsWindowVisible(hwnd)) { + fail(env, result, message, STATUS_WINDOW_NOT_FOUND, "Window is minimized or hidden"); + return NULL; + } + + DPI_AWARENESS_CONTEXT old = enter_physical_pixels(); + jintArray pixels = NULL; + RECT window_rect, frame; + if (!GetWindowRect(hwnd, &window_rect)) { + fail(env, result, message, STATUS_WINDOW_NOT_FOUND, "GetWindowRect failed"); + leave_physical_pixels(old); + return NULL; + } + /* The visible frame, without the invisible resize borders and the shadow. */ + if (FAILED(DwmGetWindowAttribute(hwnd, DWMWA_EXTENDED_FRAME_BOUNDS, &frame, sizeof(frame)))) frame = window_rect; + if (!IntersectRect(&frame, &frame, &window_rect)) frame = window_rect; + + int full_width = window_rect.right - window_rect.left; + int full_height = window_rect.bottom - window_rect.top; + int w = frame.right - frame.left; + int h = frame.bottom - frame.top; + if (w <= 0 || h <= 0 || too_large(full_width, full_height)) { + SetLastError(0); + fail(env, result, message, STATUS_WINDOW_NOT_FOUND, "Window has no capturable area"); + leave_physical_pixels(old); + return NULL; + } + + Surface surface; + ZeroMemory(&surface, sizeof(surface)); + PrintJob *job = NULL; + if (!IsHungAppWindow(hwnd)) { + job = print_window(hwnd, full_width, full_height, include_cursor, window_rect.left, window_rect.top); + } + if (job != NULL) { + pixels = to_java(env, &job->surface, frame.left - window_rect.left, frame.top - window_rect.top, w, h); + if (pixels != NULL) set_result(env, result, STATUS_OK, w, h); + print_job_release(job); + } else if (!(*env)->ExceptionCheck(env)) { + /* Hung, slow or refusing window: what is visible of it on screen. */ + if (blit_screen(&surface, frame.left, frame.top, w, h, include_cursor)) { + pixels = to_java(env, &surface, 0, 0, w, h); + if (pixels != NULL) set_result(env, result, STATUS_OK, w, h); + } else { + fail(env, result, message, STATUS_FAILED, "PrintWindow and BitBlt failed"); + } + surface_close(&surface); + } + leave_physical_pixels(old); + return pixels; +} + +JNIEXPORT jint JNICALL +Java_dev_nucleusframework_screencapture_internal_NativeScreenCapture_nativePermissionStatus(JNIEnv *env, jclass cls) { + (void)env; + (void)cls; + return PERMISSION_NOT_REQUIRED; +} + +JNIEXPORT jint JNICALL +Java_dev_nucleusframework_screencapture_internal_NativeScreenCapture_nativeRequestPermission(JNIEnv *env, jclass cls) { + (void)env; + (void)cls; + return PERMISSION_NOT_REQUIRED; +} + +JNIEXPORT jintArray JNICALL +Java_dev_nucleusframework_screencapture_internal_NativeScreenCapture_nativePortalScreenshot( + JNIEnv *env, jclass cls, jboolean interactive, jint timeout_ms, jintArray result, jobjectArray message) { + (void)cls; + (void)interactive; + (void)timeout_ms; + SetLastError(0); + fail(env, result, message, STATUS_UNSUPPORTED, "xdg-desktop-portal is Linux only"); + return NULL; +} diff --git a/screen-capture/src/main/resources/META-INF/native-image/dev.nucleusframework/nucleus.screen-capture/reachability-metadata.json b/screen-capture/src/main/resources/META-INF/native-image/dev.nucleusframework/nucleus.screen-capture/reachability-metadata.json new file mode 100644 index 000000000..34db02b15 --- /dev/null +++ b/screen-capture/src/main/resources/META-INF/native-image/dev.nucleusframework/nucleus.screen-capture/reachability-metadata.json @@ -0,0 +1,33 @@ +{ + "reflection": [ + { + "type": "dev.nucleusframework.screencapture.internal.DisplayCollector", + "jniAccessible": true, + "methods": [ + { + "name": "add", + "parameterTypes": [ + "java.lang.String", + "java.lang.String", + "int", + "int", + "int", + "int", + "int", + "int", + "float", + "boolean" + ] + } + ] + }, + { + "type": "dev.nucleusframework.screencapture.internal.NativeScreenCapture", + "jniAccessible": true + }, + { + "type": "java.lang.String", + "jniAccessible": true + } + ] +} diff --git a/screen-capture/src/test/kotlin/dev/nucleusframework/screencapture/ScreenCaptureLiveTest.kt b/screen-capture/src/test/kotlin/dev/nucleusframework/screencapture/ScreenCaptureLiveTest.kt new file mode 100644 index 000000000..9baa6fa59 --- /dev/null +++ b/screen-capture/src/test/kotlin/dev/nucleusframework/screencapture/ScreenCaptureLiveTest.kt @@ -0,0 +1,164 @@ +package dev.nucleusframework.screencapture + +import dev.nucleusframework.core.runtime.Platform +import org.junit.jupiter.api.Assumptions.assumeTrue +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicInteger +import kotlin.random.Random +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +/** + * Captures the real desktop. Runs where a backend exists and capture is allowed; on macOS a + * runner without Screen Recording rights only checks enumeration. + */ +class ScreenCaptureLiveTest { + private fun assumeCapturable() { + assumeTrue(ScreenCapture.isSupported, "no capture backend") + assumeTrue(ScreenCapture.backend != CaptureBackend.XdgDesktopPortal, "portal is interactive") + assumeTrue(ScreenCapture.displays().isNotEmpty(), "no display") + val permission = ScreenCapture.permissionStatus() + assumeTrue( + permission == CapturePermission.Granted || permission == CapturePermission.NotRequired, + "$permission", + ) + } + + @Test + fun `backend matches the platform`() { + val expected = + when (Platform.Current) { + Platform.Windows -> setOf(CaptureBackend.Gdi) + Platform.MacOS -> setOf(CaptureBackend.ScreenCaptureKit, CaptureBackend.CoreGraphics) + Platform.Linux -> setOf(CaptureBackend.X11, CaptureBackend.XdgDesktopPortal, CaptureBackend.Unavailable) + Platform.Unknown -> setOf(CaptureBackend.Unavailable) + } + assertTrue(ScreenCapture.backend in expected, "${ScreenCapture.backend}") + } + + @Test + fun `displays are consistent`() { + assumeCapturable() + val displays = ScreenCapture.displays() + assertEquals(1, displays.count { it.isPrimary }, "$displays") + assertTrue(displays.first().isPrimary) + assertEquals(displays.size, displays.map { it.id }.toSet().size, "duplicate ids: $displays") + for (d in displays) { + assertTrue(d.widthPx > 0 && d.heightPx > 0, "$d") + assertTrue(d.scaleFactor >= 1f, "$d") + } + } + + @Test + fun `full capture has the display's pixel size`() { + assumeCapturable() + for (display in ScreenCapture.displays()) { + val image = ScreenCapture.captureDisplay(display) + assertEquals(display.widthPx, image.width, "$display") + assertEquals(display.heightPx, image.height, "$display") + assertTrue(image.toArgbArray().all { it ushr 24 == 0xFF }, "alpha must be opaque") + } + } + + @Test + fun `region is a crop of the full capture`() { + assumeCapturable() + val display = ScreenCapture.primaryDisplay()!! + // Static content is not guaranteed; compare the region against a capture taken right after. + val region = CaptureRegion(display.widthPx / 4, display.heightPx / 4, 97, 53) + val part = ScreenCapture.captureDisplay(display, region) + assertEquals(97, part.width) + assertEquals(53, part.height) + val full = ScreenCapture.captureDisplay(display).crop(region) + val same = part.toArgbArray().zip(full.toArgbArray()).count { (a, b) -> a == b } + assertTrue(same > part.width * part.height * 9 / 10, "region differs from the full capture: $same") + } + + @Test + fun `regions are clipped and empty intersections rejected`() { + assumeCapturable() + val display = ScreenCapture.primaryDisplay()!! + val clipped = ScreenCapture.captureDisplay(display, CaptureRegion(display.widthPx - 10, -5, 50, 20)) + assertEquals(10, clipped.width) + assertEquals(15, clipped.height) + val outside = CaptureRegion(display.widthPx, 0, 10, 10) + val error = assertFailsWith { ScreenCapture.captureDisplay(display, outside) } + assertEquals(CaptureFailure.InvalidRegion, error.failure) + val far = CaptureRegion(Int.MAX_VALUE - 5, Int.MAX_VALUE - 5, 5, 5) + assertEquals( + CaptureFailure.InvalidRegion, + assertFailsWith { ScreenCapture.captureDisplay(display, far) }.failure, + ) + val negative = CaptureRegion(Int.MIN_VALUE, Int.MIN_VALUE, Int.MAX_VALUE, Int.MAX_VALUE) + assertEquals( + CaptureFailure.InvalidRegion, + assertFailsWith { ScreenCapture.captureDisplay(display, negative) }.failure, + ) + } + + @Test + fun `unknown display and window are reported`() { + assumeCapturable() + val real = ScreenCapture.primaryDisplay()!! + val ghost = CaptureDisplay("nucleus-no-such-display", "ghost", real.bounds, 10, 10, 1f, false) + assertEquals( + CaptureFailure.DisplayNotFound, + assertFailsWith { ScreenCapture.captureDisplay(ghost) }.failure, + ) + for (id in listOf(0L, 1L, 0x7FFF_FFF0L, -1L)) { + assertEquals( + CaptureFailure.WindowNotFound, + assertFailsWith { ScreenCapture.captureWindow(id) }.failure, + "window $id", + ) + } + } + + @Test + fun `cursor capture stays within the display size`() { + assumeCapturable() + val display = ScreenCapture.primaryDisplay()!! + val image = ScreenCapture.captureDisplay(display, includeCursor = true) + assertEquals(display.widthPx, image.width) + } + + @Test + fun `concurrent random captures`() { + assumeCapturable() + val displays = ScreenCapture.displays() + val pool = Executors.newFixedThreadPool(8) + val done = AtomicInteger() + val failures = java.util.concurrent.ConcurrentLinkedQueue() + repeat(8) { worker -> + pool.execute { + val random = Random(worker) + repeat(ITERATIONS) { + val display = displays[random.nextInt(displays.size)] + val x = random.nextInt(-200, display.widthPx + 200) + val y = random.nextInt(-200, display.heightPx + 200) + val region = CaptureRegion(x, y, random.nextInt(1, 400), random.nextInt(1, 400)) + try { + val image = ScreenCapture.captureDisplay(display, region, includeCursor = random.nextBoolean()) + check(image.width <= region.width && image.height <= region.height) { "$region -> $image" } + } catch (e: ScreenCaptureException) { + if (e.failure != CaptureFailure.InvalidRegion) failures += e + } catch (e: Throwable) { + failures += e + } + done.incrementAndGet() + } + } + } + pool.shutdown() + assertTrue(pool.awaitTermination(5, TimeUnit.MINUTES)) + failures.peek()?.let { throw AssertionError("${failures.size} failures", it) } + assertEquals(8 * ITERATIONS, done.get()) + } + + private companion object { + const val ITERATIONS = 150 + } +} diff --git a/screen-capture/src/test/kotlin/dev/nucleusframework/screencapture/ScreenImageTest.kt b/screen-capture/src/test/kotlin/dev/nucleusframework/screencapture/ScreenImageTest.kt new file mode 100644 index 000000000..05f080dbd --- /dev/null +++ b/screen-capture/src/test/kotlin/dev/nucleusframework/screencapture/ScreenImageTest.kt @@ -0,0 +1,124 @@ +package dev.nucleusframework.screencapture + +import java.io.ByteArrayInputStream +import java.io.DataInputStream +import java.util.zip.CRC32 +import java.util.zip.InflaterInputStream +import kotlin.random.Random +import kotlin.test.Test +import kotlin.test.assertContentEquals +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith + +class ScreenImageTest { + private fun image( + width: Int, + height: Int, + seed: Int = 1, + ): ScreenImage { + val random = Random(seed) + return ScreenImage(width, height, IntArray(width * height) { random.nextInt() or 0xFF000000.toInt() }, 1f) + } + + @Test + fun `png round-trips every pixel`() { + for ((w, h) in listOf(1 to 1, 7 to 3, 257 to 129, 1 to 300, 300 to 1)) { + val source = image(w, h, seed = w * 31 + h) + val decoded = decodePng(source.toPng()) + assertEquals(w, decoded.first) + assertEquals(h, decoded.second) + assertContentEquals(source.toArgbArray(), decoded.third, "${w}x$h") + } + } + + @Test + fun `bgra bytes follow the argb ints`() { + val source = ScreenImage(2, 1, intArrayOf(0xFF112233.toInt(), 0xFFAABBCC.toInt()), 1f) + assertContentEquals( + byteArrayOf(0x33, 0x22, 0x11, 0xFF.toByte(), 0xCC.toByte(), 0xBB.toByte(), 0xAA.toByte(), 0xFF.toByte()), + source.toBgraBytes(), + ) + } + + @Test + fun `crop copies the region`() { + val source = image(10, 8) + val region = CaptureRegion(3, 2, 4, 5) + val cropped = source.crop(region) + assertEquals(4, cropped.width) + assertEquals(5, cropped.height) + for (y in 0 until 5) { + for (x in 0 until 4) assertEquals(source.pixelAt(x + 3, y + 2), cropped.pixelAt(x, y)) + } + assertFailsWith { source.crop(CaptureRegion(8, 0, 3, 1)) } + assertFailsWith { source.crop(CaptureRegion(-1, 0, 3, 1)) } + } + + @Test + fun `invalid shapes are rejected`() { + assertFailsWith { CaptureRegion(0, 0, 0, 1) } + assertFailsWith { CaptureRegion(0, 0, 1, -1) } + assertFailsWith { ScreenImage(2, 2, IntArray(3), 1f) } + assertFailsWith { image(2, 2).pixelAt(2, 0) } + } +} + +/** A strict decoder for what [ScreenImage.toPng] writes: 8-bit RGB, any filter, chunk CRCs verified. */ +internal fun decodePng(bytes: ByteArray): Triple { + val input = DataInputStream(ByteArrayInputStream(bytes)) + val signature = ByteArray(8).also { input.readFully(it) } + check(signature.contentEquals(byteArrayOf(0x89.toByte(), 80, 78, 71, 13, 10, 26, 10))) { "bad signature" } + var width = 0 + var height = 0 + val idat = java.io.ByteArrayOutputStream() + while (true) { + val length = input.readInt() + val type = ByteArray(4).also { input.readFully(it) } + val data = ByteArray(length).also { input.readFully(it) } + val crc = input.readInt() + check( + CRC32() + .apply { + update(type) + update(data) + }.value + .toInt() == crc, + ) { "bad CRC" } + when (String(type, Charsets.US_ASCII)) { + "IHDR" -> { + val header = DataInputStream(ByteArrayInputStream(data)) + width = header.readInt() + height = header.readInt() + check(header.readByte().toInt() == 8 && header.readByte().toInt() == 2) { "not 8-bit RGB" } + } + "IDAT" -> idat.write(data) + "IEND" -> break + } + } + val raw = InflaterInputStream(ByteArrayInputStream(idat.toByteArray())).readBytes() + val stride = width * 3 + check(raw.size == height * (stride + 1)) { "bad IDAT size ${raw.size}" } + val out = IntArray(width * height) + var previous = IntArray(stride) + for (y in 0 until height) { + val filter = raw[y * (stride + 1)].toInt() + val line = IntArray(stride) + for (i in 0 until stride) { + val value = raw[y * (stride + 1) + 1 + i].toInt() and 0xFF + val left = if (i >= 3) line[i - 3] else 0 + val up = previous[i] + line[i] = + when (filter) { + 0 -> value + 1 -> value + left + 2 -> value + up + else -> error("unexpected filter $filter") + } and 0xFF + } + for (x in 0 until width) { + out[y * width + x] = (0xFF shl 24) or (line[x * 3] shl 16) or (line[x * 3 + 1] shl 8) or line[x * 3 + 2] + } + previous = line + } + return Triple(width, height, out) +} diff --git a/scripts/screen-capture-linux-e2e.sh b/scripts/screen-capture-linux-e2e.sh new file mode 100644 index 000000000..b60a28320 --- /dev/null +++ b/scripts/screen-capture-linux-e2e.sh @@ -0,0 +1,108 @@ +#!/bin/bash +# E2E + torture suite for the Linux screen-capture backend (screen-capture/src/main/native/linux). +# +# X11: Xvfb at depths 24 / 16 / 8, RandR monitors, a 4K screen, cursor, window capture (with and +# without an XComposite redirect), random-region torture from 8 threads with RSS / fd checks, and +# the X server dying mid-capture. Every display capture is checked pixel-exact against a known +# pattern and, at depth 24, against `xwd -root`. +# Portal: a fake org.freedesktop.portal.Screenshot on a private session bus (ok, cancel, error, +# timeout, bogus / missing / garbage files, percent-encoded paths, symlinks, ignored +# handle_token, a response before the method reply, concurrency). +# +# Requires: gcc, a JDK, Xvfb, xwd + ImageMagick `convert`, dbus-run-session, python3-dbus, +# python3-gi, and the X11 / Xrandr / Xcomposite / dbus development headers. +# Usage: scripts/screen-capture-linux-e2e.sh (prints RESULT lines; exits non-zero on failure) +set -u +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +T="$ROOT/scripts/screen-capture-linux-e2e" +bash "$ROOT/screen-capture/src/main/native/linux/build.sh" >/dev/null || exit 1 +case "$(uname -m)" in x86_64) ARCH=x64 ;; *) ARCH=$(uname -m) ;; esac +LIB="$ROOT/screen-capture/src/main/resources/nucleus/native/linux-$ARCH/libnucleus_screencapture.so" +W=$(mktemp -d); trap 'rm -rf "$W"' EXIT +main() { +gcc -O2 "$T/xtool.c" -o $W/xtool -lX11 -lXrandr -lXcomposite || return 1 +javac -d $W/classes "$T"/harness/dev/nucleusframework/screencapture/internal/*.java "$T"/harness/dev/nucleusframework/core/runtime/*.java || return 1 +JV="java --enable-native-access=ALL-UNNAMED -Xms1g -Xmx1g -XX:+AlwaysPreTouch -Dlib=$LIB -cp $W/classes" +H=dev.nucleusframework.screencapture.internal.Harness +J="$JV $H" +X=$W/xtool +PIDS=() +cleanup() { for p in "${PIDS[@]}"; do kill $p 2>/dev/null; done; PIDS=(); sleep 0.3; } + +start_x() { # display depth [size] + LASTD=$1 + Xvfb :$1 -screen 0 ${3:-1920x1080}x$2 -nolisten tcp >/dev/null 2>&1 & PIDS+=($!) + for i in $(seq 50); do DISPLAY=:$1 $X warp 0 0 2>/dev/null && return 0; sleep 0.1; done; echo "Xvfb :$1 failed"; return 1 +} +solid() { # x y w h rgb -> sets WID + coproc SOLID { exec $X window "$@"; }; PIDS+=($SOLID_PID); read -r WID <&${SOLID[0]} +} + +for depth in 24 16; do + echo "=== X11 depth $depth" + start_x $((80 + depth)) $depth; export DISPLAY=:$LASTD + $X pattern > /dev/null & PIDS+=($!); sleep 0.5 + solid 1500 800 200 150 3366CC; sleep 0.3 + DEAD=$($X destroyed) + $J x11 $depth $X "1500,800,200,150,3366CC,$WID" $DEAD + echo "=== X11 depth $depth, two RandR monitors" + $X monitors + $JV -Diterations=2000 $H x11 $depth $X "1500,800,200,150,3366CC,$WID" $DEAD monitors 2>&1 || true + cleanup +done + +echo "=== X11 4K (3840x2160x24) timing" +start_x 82 24 3840x2160; export DISPLAY=:$LASTD +$X pattern > /dev/null & PIDS+=($!); sleep 1.5 +solid 1500 800 200 150 3366CC; sleep 0.3 +$JV -Diterations=1200 -Dphases=8 $H x11 24 $X "1500,800,200,150,3366CC,$WID" $($X destroyed) 2>&1 +cleanup + +echo "=== X11 8-bit PseudoColor" +start_x 83 8 800x600; export DISPLAY=:$LASTD +solid 100 100 200 150 3366CC; sleep 0.3 +$J smoke8 "100,100,200,150,3366CC,$WID" $($X destroyed) +cleanup + +echo "=== occluded window, no compositor (fallback: visible pixels only)" +start_x 84 24; export DISPLAY=:$LASTD +solid 100 100 200 150 3366CC; W1=$WID; sleep 0.2 +solid 150 150 50 50 FF0000; sleep 0.3 +$JV -DallowOccluded=true $H window-only "100,100,200,150,3366CC,$W1" $($X destroyed) 2>&1 | grep -E "window|RESULT|compositor" +cleanup +echo "=== occluded window, composite redirect (XCompositeNameWindowPixmap)" +start_x 85 24; export DISPLAY=:$LASTD +$X redirect > /dev/null & PIDS+=($!); sleep 0.3 +solid 100 100 200 150 3366CC; W1=$WID; sleep 0.2 +solid 150 150 50 50 FF0000; sleep 0.3 +$J window-only "100,100,200,150,3366CC,$W1" $($X destroyed) +echo "=== partly off-screen window" +solid -50 -40 200 150 00FF00; sleep 0.3 +$J window-only "0,0,200,150,00FF00,$WID" $($X destroyed) 2>&1 | grep -E "window|RESULT|FAIL" +cleanup + +echo "=== X server dies mid-capture" +start_x 86 24 1280x720; export DISPLAY=:$LASTD +XPID=${PIDS[-1]} +( sleep 1.5; kill -9 $XPID ) & +$J xkill 4000 +cleanup + +echo "=== no DISPLAY" +unset DISPLAY +env -u DISPLAY -u WAYLAND_DISPLAY $J nodisplay + +echo "=== portal" +rm -rf $W/shots; mkdir -p $W/shots; echo ok > $W/mode +dbus-run-session -- bash -c "PORTAL_MODE_FILE=$W/mode PORTAL_DIR=$W/shots python3 $T/fake_portal.py & sleep 1.5; $JV -DportalMode=$W/mode -DportalDir=$W/shots $H portal" +echo "=== portal missing (bus without portal)" +dbus-run-session -- $J portal-missing +echo "=== portal missing (no session bus)" +env -u DISPLAY -u WAYLAND_DISPLAY -u DBUS_SESSION_BUS_ADDRESS XDG_RUNTIME_DIR=/nonexistent $J portal-missing +} + +main 2>&1 | grep -vE "^WARNING|^$" | tee "$W/log" +PASSED=$(grep -c "RESULT: PASS" "$W/log") +FAILURES=$(grep -c "RESULT: FAIL" "$W/log") +echo "=== $PASSED passed, $FAILURES failed" +[ "$FAILURES" = 0 ] && [ "$PASSED" -gt 0 ] diff --git a/scripts/screen-capture-linux-e2e/fake_portal.py b/scripts/screen-capture-linux-e2e/fake_portal.py new file mode 100644 index 000000000..624e070c3 --- /dev/null +++ b/scripts/screen-capture-linux-e2e/fake_portal.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""Fake org.freedesktop.portal.Screenshot for tests. Mode is read from $PORTAL_MODE_FILE on every call.""" +import os, random, struct, sys, urllib.parse, zlib +import dbus, dbus.service, dbus.lowlevel +from dbus.mainloop.glib import DBusGMainLoop +from gi.repository import GLib + +MODE_FILE = os.environ["PORTAL_MODE_FILE"] +SHOTS = os.environ["PORTAL_DIR"] +W, H = 640, 480 + +def pattern(x, y): + return (x & 255, y & 255, ((x >> 8) * 64 + (y >> 8) * 16 + ((x + y) & 15)) & 255) + +def png(path, alpha): + raw = bytearray() + for y in range(H): + raw.append(0) + for x in range(W): + raw.extend(pattern(x, y)) + if alpha: + raw.append(128) + def chunk(t, d): + c = struct.pack(">I", len(d)) + t + d + return c + struct.pack(">I", zlib.crc32(t + d) & 0xffffffff) + data = b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", W, H, 8, 6 if alpha else 2, 0, 0, 0)) + data += chunk(b"IDAT", zlib.compress(bytes(raw), 1)) + chunk(b"IEND", b"") + with open(path, "wb") as f: + f.write(data) + +def uri(path): + return "file://" + urllib.parse.quote(path) + +class Portal(dbus.service.Object): + def __init__(self, bus): + super().__init__(bus, "/org/freedesktop/portal/desktop") + self.bus = bus + + def respond(self, path, code, results): + msg = dbus.lowlevel.SignalMessage(path, "org.freedesktop.portal.Request", "Response") + msg.append(dbus.UInt32(code), dbus.Dictionary(results, signature="sv"), signature="ua{sv}") + self.bus.send_message(msg) + return False + + @dbus.service.method("org.freedesktop.portal.Screenshot", in_signature="sa{sv}", out_signature="o", + sender_keyword="sender") + def Screenshot(self, parent, options, sender): + mode = open(MODE_FILE).read().strip() + token = str(options.get("handle_token", "t%d" % random.randint(0, 1 << 30))) + if mode == "ignore_token": + token = "portal%d" % random.randint(0, 1 << 30) + path = "/org/freedesktop/portal/desktop/request/%s/%s" % (sender[1:].replace(".", "_"), token) + if mode == "dbus_error": + raise dbus.exceptions.DBusException("boom", name="org.freedesktop.portal.Error.Failed") + if mode == "access_denied": + raise dbus.exceptions.DBusException("denied", name="org.freedesktop.DBus.Error.AccessDenied") + name = "Screenshot-%d.png" % random.randint(0, 1 << 30) + delay = 0 + results = {} + code = 0 + if mode in ("ok", "ignore_token", "early") or mode.startswith("delay:"): + f = os.path.join(SHOTS, name); png(f, False); results = {"uri": uri(f)} + if mode.startswith("delay:"): + delay = int(mode.split(":")[1]) + elif mode == "ok_rgba_spaces": + f = os.path.join(SHOTS, "Scr een é %25 " + name); png(f, True); results = {"uri": uri(f)} + elif mode == "symlink": + t = os.path.join(SHOTS, "target.png"); png(t, False) + l = os.path.join(SHOTS, "link-%d.bin" % random.randint(0, 1 << 30)) + os.symlink(t, l); results = {"uri": uri(l)} + elif mode == "cancel": + code = 1 + elif mode == "error": + code = 2 + elif mode == "bogus_uri": + results = {"uri": "http://example.com/x.png"} + elif mode == "missing_file": + results = {"uri": uri(os.path.join(SHOTS, "nope.png"))} + elif mode == "not_png": + f = os.path.join(SHOTS, "garbage-%d.bin" % random.randint(0, 1 << 30)) + open(f, "wb").write(os.urandom(4096)); results = {"uri": uri(f)} + elif mode == "no_uri": + results = {} + elif mode == "wrong_path": + f = os.path.join(SHOTS, name + ".bin"); png(f, False); results = {"uri": uri(f)} + path_sig = "/org/freedesktop/portal/desktop/request/other/xyz" + GLib.idle_add(self.respond, path_sig, 0, results) + return dbus.ObjectPath(path) + elif mode == "silent": + return dbus.ObjectPath(path) + if mode == "early": + self.respond(path, code, results) + elif delay: + GLib.timeout_add(delay, self.respond, path, code, results) + else: + GLib.idle_add(self.respond, path, code, results) + return dbus.ObjectPath(path) + +DBusGMainLoop(set_as_default=True) +bus = dbus.SessionBus() +name = dbus.service.BusName("org.freedesktop.portal.Desktop", bus) +portal = Portal(bus) +print("fake portal ready", flush=True) +GLib.MainLoop().run() diff --git a/scripts/screen-capture-linux-e2e/harness/dev/nucleusframework/core/runtime/JniExceptionReporter.java b/scripts/screen-capture-linux-e2e/harness/dev/nucleusframework/core/runtime/JniExceptionReporter.java new file mode 100644 index 000000000..7d1021f13 --- /dev/null +++ b/scripts/screen-capture-linux-e2e/harness/dev/nucleusframework/core/runtime/JniExceptionReporter.java @@ -0,0 +1,11 @@ +package dev.nucleusframework.core.runtime; + +/** Stand-in for core-runtime's reporter, which nucleus_jni_clear_exception() calls. */ +public final class JniExceptionReporter { + public static volatile int reported = 0; + + public static void report(Throwable t) { + reported++; + System.out.println("JniExceptionReporter: " + t); + } +} diff --git a/scripts/screen-capture-linux-e2e/harness/dev/nucleusframework/screencapture/internal/DisplayCollector.java b/scripts/screen-capture-linux-e2e/harness/dev/nucleusframework/screencapture/internal/DisplayCollector.java new file mode 100644 index 000000000..f14267b5e --- /dev/null +++ b/scripts/screen-capture-linux-e2e/harness/dev/nucleusframework/screencapture/internal/DisplayCollector.java @@ -0,0 +1,18 @@ +package dev.nucleusframework.screencapture.internal; + +import java.util.ArrayList; +import java.util.List; + +public final class DisplayCollector { + public static final class D { + public String id, name; public int x, y, w, h, pw, ph; public float scale; public boolean primary; + public String toString() { return id + "[" + x + "," + y + " " + w + "x" + h + " px=" + pw + "x" + ph + " s=" + scale + (primary ? " primary" : "") + "]"; } + } + public final List list = new ArrayList<>(); + public boolean throwOnAdd; + public void add(String id, String name, int x, int y, int w, int h, int pw, int ph, float scale, boolean primary) { + if (throwOnAdd) throw new IllegalStateException("boom from add"); + D d = new D(); d.id = id; d.name = name; d.x = x; d.y = y; d.w = w; d.h = h; d.pw = pw; d.ph = ph; d.scale = scale; d.primary = primary; + list.add(d); + } +} diff --git a/scripts/screen-capture-linux-e2e/harness/dev/nucleusframework/screencapture/internal/Harness.java b/scripts/screen-capture-linux-e2e/harness/dev/nucleusframework/screencapture/internal/Harness.java new file mode 100644 index 000000000..fad51ff5f --- /dev/null +++ b/scripts/screen-capture-linux-e2e/harness/dev/nucleusframework/screencapture/internal/Harness.java @@ -0,0 +1,507 @@ +package dev.nucleusframework.screencapture.internal; + +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +/** E2E + torture harness for libnucleus_screencapture.so. Exits non-zero on the first failed check. */ +public final class Harness { + static int failures = 0; + static int depth = 24; + static String xtool; + // Solid window placed by the driver script: x, y, w, h, rgb, xid + static int wx, wy, ww, wh, wrgb; + static long wid; + + static void check(boolean ok, String what) { + if (!ok) { + failures++; + System.out.println("FAIL: " + what); + } + } + + static int pattern(int x, int y) { + int r = x & 255, g = y & 255, b = ((x >> 8) * 64 + (y >> 8) * 16 + ((x + y) & 15)) & 255; + return (r << 16) | (g << 8) | b; + } + + static int expand(int v, int bits) { + if (bits >= 8) return v; + // Bit replication, as xwd / ImageMagick / pixman widen a channel. + int q = v >> (8 - bits), out = 0, filled = 0; + while (filled < 8) { out = (out << bits) | q; filled += bits; } + return out >> (filled - 8); + } + + static int quantize(int rgb) { + if (depth == 24) return rgb | 0xFF000000; + int r = expand((rgb >> 16) & 255, 5), g = expand((rgb >> 8) & 255, 6), b = expand(rgb & 255, 5); + return 0xFF000000 | (r << 16) | (g << 8) | b; + } + + /** What the root window shows at (x, y). */ + static int expected(int x, int y) { + if (wid != 0 && x >= wx && x < wx + ww && y >= wy && y < wy + wh) return quantize(wrgb); + return quantize(pattern(x, y)); + } + + static List displays() { + DisplayCollector c = new DisplayCollector(); + String[] m = new String[1]; + int st = NativeScreenCapture.nativeListDisplays(c, m); + check(st == 0, "listDisplays status " + st + " " + m[0]); + return c.list; + } + + /** Checks a capture of display d at region (clipped) against the oracle; returns mismatches. */ + static long verify(DisplayCollector.D d, int rx, int ry, int[] px, int w, int h) { + long bad = 0; + for (int y = 0; y < h; y++) + for (int x = 0; x < w; x++) + if (px[y * w + x] != expected(d.x + rx + x, d.y + ry + y)) bad++; + return bad; + } + + static int[] clip(DisplayCollector.D d, long rx, long ry, long rw, long rh) { + long l = Math.max(0, rx), t = Math.max(0, ry), r = Math.min(d.w, rx + rw), b = Math.min(d.h, ry + rh); + if (r <= l || b <= t) return null; + return new int[] {(int) l, (int) t, (int) (r - l), (int) (b - t)}; + } + + static void run(String... cmd) throws Exception { + Process p = new ProcessBuilder(cmd).inheritIO().start(); + p.waitFor(); + } + + static long rssKb() throws Exception { + for (String line : Files.readAllLines(Path.of("/proc/self/status"))) + if (line.startsWith("VmRSS:")) return Long.parseLong(line.replaceAll("\\D", "")); + return -1; + } + + static long fds() throws Exception { + try (var s = Files.list(Path.of("/proc/self/fd"))) { + return s.count(); + } + } + + public static void main(String[] args) throws Exception { + System.load(System.getProperty("lib")); + String mode = args[0]; + switch (mode) { + case "x11": x11(args); break; + case "nodisplay": noDisplay(); break; + case "portal": portal(args); break; + case "portal-missing": portalMissing(); break; + case "xkill": xkill(args); break; + case "window-only": windowOnly(args); break; + case "smoke8": smoke8(args); break; + default: throw new IllegalArgumentException(mode); + } + System.out.println(failures == 0 ? "RESULT: PASS " + mode : "RESULT: FAIL " + mode + " (" + failures + ")"); + System.exit(failures == 0 ? 0 : 1); + } + + // ---------------------------------------------------------------- X11 + + static void x11(String[] args) throws Exception { + depth = Integer.parseInt(args[1]); + xtool = args[2]; + String[] win = args[3].split(","); + wx = Integer.parseInt(win[0]); wy = Integer.parseInt(win[1]); ww = Integer.parseInt(win[2]); wh = Integer.parseInt(win[3]); + wrgb = Integer.parseInt(win[4], 16); wid = Long.parseLong(win[5]); + long deadWindow = Long.parseLong(args[4]); + boolean expectMonitors = args.length > 5 && args[5].equals("monitors"); + int iterations = Integer.parseInt(System.getProperty("iterations", "5000")); + int threads = Integer.parseInt(System.getProperty("threads", "8")); + + check(NativeScreenCapture.nativeBackend() == 4, "backend is X11"); + check(NativeScreenCapture.nativePermissionStatus() == 3, "permission NOT_REQUIRED"); + check(NativeScreenCapture.nativeRequestPermission() == 3, "request NOT_REQUIRED"); + + List ds = displays(); + System.out.println("displays: " + ds); + check(!ds.isEmpty(), "at least one display"); + check(ds.stream().filter(d -> d.primary).count() == 1, "exactly one primary"); + if (expectMonitors) { + check(ds.size() == 3, "RandR monitors: screen + LEFT + RIGHT"); + check(ds.stream().anyMatch(d -> d.id.equals("LEFT") && d.x == 0 && !d.primary), "LEFT monitor"); + check(ds.stream().anyMatch(d -> d.id.equals("RIGHT") && d.x > 0 && d.primary), "RIGHT primary monitor"); + } + + // Full capture of every display, pixel-exact against the oracle. + for (DisplayCollector.D d : ds) { + int[] res = new int[3]; String[] m = new String[1]; + int[] px = NativeScreenCapture.nativeCaptureDisplay(d.id, 0, 0, 0, 0, false, res, m); + check(px != null && res[0] == 0 && res[1] == d.w && res[2] == d.h, "full capture " + d.id + " " + res[0] + " " + m[0]); + if (px != null) { + long bad = verify(d, 0, 0, px, res[1], res[2]); + System.out.println("full " + d.id + " " + res[1] + "x" + res[2] + " mismatches=" + bad); + check(bad == 0, "full capture exact " + d.id + " bad=" + bad); + } + } + + // Independent oracle: xwd of the root window (depth 24 only, needs ImageMagick). + xwdCrossCheck(ds); + + // Unknown display id. + { + int[] res = new int[3]; String[] m = new String[1]; + int[] px = NativeScreenCapture.nativeCaptureDisplay("NOPE-1", 0, 0, 0, 0, false, res, m); + check(px == null && res[0] == 3, "unknown display -> 3, got " + res[0]); + px = NativeScreenCapture.nativeCaptureDisplay("", 0, 0, 0, 0, false, res, m); + check(px == null && res[0] == 3, "empty display id -> 3, got " + res[0]); + } + + // Edge regions. + DisplayCollector.D d0 = ds.get(0); + long[][] edges = { + {-100, -100, 200, 200}, {d0.w - 10, d0.h - 10, 1000, 1000}, {d0.w, 0, 10, 10}, {0, d0.h, 10, 10}, + {-50, -50, 10, 10}, {Integer.MIN_VALUE, Integer.MIN_VALUE, Integer.MAX_VALUE, Integer.MAX_VALUE}, + {Integer.MAX_VALUE, Integer.MAX_VALUE, Integer.MAX_VALUE, Integer.MAX_VALUE}, {0, 0, 1, 1}, + {-1, -1, Integer.MAX_VALUE, Integer.MAX_VALUE}, {d0.w - 1, d0.h - 1, 1, 1}, {5, 5, -3, 7}, + }; + for (long[] e : edges) { + int[] res = new int[3]; String[] m = new String[1]; + int[] px = NativeScreenCapture.nativeCaptureDisplay(d0.id, (int) e[0], (int) e[1], (int) e[2], (int) e[3], false, res, m); + int[] c = (e[2] <= 0 || e[3] <= 0) ? new int[] {0, 0, d0.w, d0.h} : clip(d0, e[0], e[1], e[2], e[3]); + if (c == null) { + check(px == null && res[0] == 8, "edge " + java.util.Arrays.toString(e) + " -> INVALID_REGION, got " + res[0]); + } else { + check(px != null && res[0] == 0 && res[1] == c[2] && res[2] == c[3], "edge " + java.util.Arrays.toString(e) + " size " + res[1] + "x" + res[2] + " st " + res[0]); + if (px != null) check(verify(d0, c[0], c[1], px, res[1], res[2]) == 0, "edge exact " + java.util.Arrays.toString(e)); + } + } + + // Cursor. + cursorCheck(d0); + + // Timing: full 1080p (or whatever the first display is). + { + int[] res = new int[3]; String[] m = new String[1]; + for (int i = 0; i < 5; i++) NativeScreenCapture.nativeCaptureDisplay(d0.id, 0, 0, 0, 0, false, res, m); + long t0 = System.nanoTime(); + int n = 40; + for (int i = 0; i < n; i++) NativeScreenCapture.nativeCaptureDisplay(d0.id, 0, 0, 0, 0, false, res, m); + System.out.printf("timing: %dx%d full capture avg %.2f ms%n", res[1], res[2], (System.nanoTime() - t0) / 1e6 / n); + t0 = System.nanoTime(); + for (int i = 0; i < n; i++) NativeScreenCapture.nativeCaptureDisplay(d0.id, 10, 10, 64, 64, false, res, m); + System.out.printf("timing: 64x64 region avg %.2f ms%n", (System.nanoTime() - t0) / 1e6 / n); + } + + // Windows. + windowChecks(deadWindow); + + // Callback that throws must not crash nor leak an exception. + { + DisplayCollector c = new DisplayCollector(); + c.throwOnAdd = true; + String[] m = new String[1]; + int st = NativeScreenCapture.nativeListDisplays(c, m); + check(st == 6, "throwing collector -> FAILED, got " + st + " " + m[0]); + check(dev.nucleusframework.core.runtime.JniExceptionReporter.reported == 1, "collector exception reported once"); + } + + torture(ds, deadWindow, iterations, threads); + } + + static void xwdCrossCheck(List ds) throws Exception { + Process p = new ProcessBuilder("sh", "-c", "xwd -root -silent | convert xwd:- rgb:-").start(); + byte[] rgb; + try (InputStream in = p.getInputStream()) { rgb = in.readAllBytes(); } + p.waitFor(); + // The root "display" = union; capture each display and compare its slice. + int rootW = 0, rootH = 0; + for (DisplayCollector.D d : ds) { rootW = Math.max(rootW, d.x + d.w); rootH = Math.max(rootH, d.y + d.h); } + if (rgb.length != (long) rootW * rootH * 3) { + System.out.println("xwd cross-check skipped: got " + rgb.length + " bytes for " + rootW + "x" + rootH); + return; + } + for (DisplayCollector.D d : ds) { + int[] res = new int[3]; String[] m = new String[1]; + int[] px = NativeScreenCapture.nativeCaptureDisplay(d.id, 0, 0, 0, 0, false, res, m); + long bad = 0; + for (int y = 0; y < res[2]; y++) + for (int x = 0; x < res[1]; x++) { + int o = ((d.y + y) * rootW + d.x + x) * 3; + int v = 0xFF000000 | ((rgb[o] & 255) << 16) | ((rgb[o + 1] & 255) << 8) | (rgb[o + 2] & 255); + if (px[y * res[1] + x] != v) bad++; + } + System.out.println("xwd cross-check " + d.id + ": mismatches=" + bad + " of " + ((long) res[1] * res[2])); + check(bad == 0, "xwd cross-check " + d.id); + } + } + + static void cursorCheck(DisplayCollector.D d) throws Exception { + int cx = d.x + 300, cy = d.y + 200; + run(xtool, "warp", String.valueOf(cx), String.valueOf(cy)); + int[] res = new int[3]; String[] m = new String[1]; + int[] with = NativeScreenCapture.nativeCaptureDisplay(d.id, 200, 100, 200, 200, true, res, m); + check(with != null && res[0] == 0, "cursor capture ok"); + int[] without = NativeScreenCapture.nativeCaptureDisplay(d.id, 200, 100, 200, 200, false, res, m); + check(without != null && verify(d, 200, 100, without, 200, 200) == 0, "no-cursor capture exact"); + if (with == null) return; + long diff = 0, outside = 0; + for (int y = 0; y < 200; y++) + for (int x = 0; x < 200; x++) + if (with[y * 200 + x] != without[y * 200 + x]) { + diff++; + int ax = d.x + 200 + x, ay = d.y + 100 + y; + if (Math.abs(ax - cx) > 64 || Math.abs(ay - cy) > 64) outside++; + } + System.out.println("cursor: " + diff + " pixels changed, " + outside + " outside the hotspot box"); + check(diff > 0, "cursor drawn"); + check(outside == 0, "cursor drawn at the pointer"); + // Cursor half off the region's left/top edge must not crash. + run(xtool, "warp", String.valueOf(d.x + 2), String.valueOf(d.y + 2)); + int[] edge = NativeScreenCapture.nativeCaptureDisplay(d.id, 0, 0, 50, 50, true, res, m); + check(edge != null, "cursor at display edge"); + } + + static void windowChecks(long deadWindow) { + int[] res = new int[3]; String[] m = new String[1]; + int[] px = NativeScreenCapture.nativeCaptureWindow(wid, false, res, m); + check(px != null && res[0] == 0 && res[1] == ww && res[2] == wh, "window capture size " + res[1] + "x" + res[2] + " st " + res[0] + " " + m[0]); + if (px != null) { + long bad = 0; + for (int v : px) if (v != quantize(wrgb)) bad++; + System.out.println("window " + wid + ": " + res[1] + "x" + res[2] + " mismatches=" + bad); + if (Boolean.getBoolean("allowOccluded")) System.out.println("(no compositor: covered pixels are the occluder's, as documented)"); + else check(bad == 0, "window capture exact"); + } + long[] invalid = {0, -1, deadWindow, 0x1FFFFFFFFL, 12345, 0x7FFFFFFF, Long.MAX_VALUE, Long.MIN_VALUE}; + for (long id : invalid) { + m[0] = null; + px = NativeScreenCapture.nativeCaptureWindow(id, true, res, m); + check(px == null && res[0] == 4, "invalid window " + id + " -> 4, got " + res[0] + " " + m[0]); + } + } + + static void torture(List ds, long deadWindow, int iterations, int threads) throws Exception { + ExecutorService pool = Executors.newFixedThreadPool(threads); + AtomicLong ops = new AtomicLong(), verified = new AtomicLong(), errors = new AtomicLong(); + int phases = Integer.getInteger("phases", 3); + long[] rss = new long[phases], fd = new long[phases]; + Runnable round = () -> {}; + for (int phase = 0; phase < phases; phase++) { + int n = phase == 0 ? Math.max(200, iterations / 2) : iterations; + List> fs = new ArrayList<>(); + for (int t = 0; t < threads; t++) { + final int seed = phase * 1000 + t; + final int per = n / threads; + fs.add(pool.submit(() -> { + Random rnd = new Random(seed); + int[] res = new int[3]; String[] m = new String[1]; + for (int i = 0; i < per; i++) { + ops.incrementAndGet(); + int op = rnd.nextInt(10); + DisplayCollector.D d = ds.get(rnd.nextInt(ds.size())); + if (op == 0) { + DisplayCollector c = new DisplayCollector(); String[] mm = new String[1]; + if (NativeScreenCapture.nativeListDisplays(c, mm) != 0 || c.list.size() != ds.size()) errors.incrementAndGet(); + } else if (op == 1) { + int[] px = NativeScreenCapture.nativeCaptureWindow(rnd.nextBoolean() ? wid : deadWindow, rnd.nextBoolean(), res, m); + if (px == null && res[0] != 4) errors.incrementAndGet(); + } else if (op == 2) { + NativeScreenCapture.nativeCaptureDisplay("missing-" + i, 0, 0, 0, 0, false, res, m); + if (res[0] != 3) errors.incrementAndGet(); + } else { + int rx = rnd.nextInt(d.w + 400) - 200, ry = rnd.nextInt(d.h + 400) - 200; + int rw = 1 + rnd.nextInt(op == 9 ? d.w + 400 : 256), rh = 1 + rnd.nextInt(op == 9 ? d.h + 400 : 256); + int[] px = NativeScreenCapture.nativeCaptureDisplay(d.id, rx, ry, rw, rh, false, res, m); + int[] c = clip(d, rx, ry, rw, rh); + if (c == null) { + if (px != null || res[0] != 8) errors.incrementAndGet(); + } else if (px == null || res[1] != c[2] || res[2] != c[3] || verify(d, c[0], c[1], px, c[2], c[3]) != 0) { + errors.incrementAndGet(); + System.out.println("torture mismatch " + d.id + " region " + rx + "," + ry + " " + rw + "x" + rh + " st=" + res[0] + " " + m[0]); + } else { + verified.incrementAndGet(); + } + } + } + })); + } + for (Future f : fs) f.get(10, TimeUnit.MINUTES); + System.gc(); + Thread.sleep(300); + rss[phase] = rssKb(); + fd[phase] = fds(); + System.out.println("torture phase " + phase + ": ops=" + ops.get() + " verified=" + verified.get() + " errors=" + errors.get() + " rss=" + rss[phase] + "kB fds=" + fd[phase]); + } + pool.shutdown(); + check(errors.get() == 0, "torture errors " + errors.get()); + int a = phases - 2, b = phases - 1; + check(fd[b] <= fd[a], "fd leak " + fd[a] + " -> " + fd[b]); + System.out.println("torture rss growth between the last equal phases: " + (rss[b] - rss[a]) + " kB"); + check(rss[b] - rss[a] < 16 * 1024, "rss growth " + (rss[b] - rss[a]) + " kB"); + } + + static void parseWindow(String spec) { + String[] win = spec.split(","); + wx = Integer.parseInt(win[0]); wy = Integer.parseInt(win[1]); ww = Integer.parseInt(win[2]); wh = Integer.parseInt(win[3]); + wrgb = Integer.parseInt(win[4], 16); wid = Long.parseLong(win[5]); + } + + /** Window capture only (e.g. with an occluding window and a composite redirect). */ + static void windowOnly(String[] args) { + parseWindow(args[1]); + windowChecks(Long.parseLong(args[2])); + } + + /** 8-bit PseudoColor: no crash, colormap resolved. */ + static void smoke8(String[] args) { + parseWindow(args[1]); + List ds = displays(); + int[] res = new int[3]; String[] m = new String[1]; + int[] px = NativeScreenCapture.nativeCaptureDisplay(ds.get(0).id, wx, wy, ww, wh, false, res, m); + check(px != null && res[0] == 0, "8-bit display capture " + res[0] + " " + m[0]); + if (px != null) { + long bad = 0; + for (int v : px) if (v != (0xFF000000 | wrgb)) bad++; + System.out.println("8-bit region over the solid window: mismatches=" + bad + " sample=" + Integer.toHexString(px[0])); + check(bad == 0, "8-bit colormap lookup exact"); + } + windowChecks(Long.parseLong(args[2])); + } + + // ---------------------------------------------------------------- No X server + + static void noDisplay() { + check(NativeScreenCapture.nativeBackend() == 0, "backend NONE without DISPLAY"); + DisplayCollector c = new DisplayCollector(); String[] m = new String[1]; + check(NativeScreenCapture.nativeListDisplays(c, m) == 1, "list -> UNSUPPORTED: " + m[0]); + int[] res = new int[3]; + check(NativeScreenCapture.nativeCaptureDisplay("screen", 0, 0, 0, 0, false, res, m) == null && res[0] == 1, "capture -> UNSUPPORTED " + res[0]); + check(NativeScreenCapture.nativeCaptureWindow(1234, false, res, m) == null && res[0] == 1, "window -> UNSUPPORTED " + res[0]); + } + + // X server killed while captures run: must not exit the JVM. + static void xkill(String[] args) throws Exception { + int[] res = new int[3]; String[] m = new String[1]; + long ok = 0, failed = 0; + long end = System.currentTimeMillis() + Long.parseLong(args[1]); + while (System.currentTimeMillis() < end) { + int[] px = NativeScreenCapture.nativeCaptureDisplay("screen", 0, 0, 0, 0, true, res, m); + if (px != null) ok++; else failed++; + } + System.out.println("xkill: ok=" + ok + " failed=" + failed + " last=" + res[0] + " " + m[0]); + check(ok > 0 && failed > 0, "captures before and after the server died"); + } + + // ---------------------------------------------------------------- Portal + + static void setMode(String mode) throws Exception { + Files.writeString(Path.of(System.getProperty("portalMode")), mode); + } + + static int[] shot(int timeout, int[] res, String[] m) { + return NativeScreenCapture.nativePortalScreenshot(false, timeout, res, m); + } + + static void portal(String[] args) throws Exception { + Path shots = Path.of(System.getProperty("portalDir")); + int[] res = new int[3]; String[] m = new String[1]; + + setMode("ok"); + int[] px = shot(5000, res, m); + check(px != null && res[0] == 0 && res[1] == 640 && res[2] == 480, "portal ok " + res[0] + " " + m[0]); + if (px != null) { + long bad = 0; + for (int y = 0; y < 480; y++) for (int x = 0; x < 640; x++) if (px[y * 640 + x] != (0xFF000000 | pattern(x, y))) bad++; + System.out.println("portal ok: 640x480 mismatches=" + bad); + check(bad == 0, "portal pixels exact"); + } + try (var s = Files.list(shots)) { check(s.noneMatch(f -> f.toString().endsWith(".png")), "portal file deleted after decode"); } + + setMode("ok_rgba_spaces"); + px = shot(5000, res, m); + check(px != null && res[0] == 0 && res[1] == 640, "portal rgba + percent-encoded path " + res[0] + " " + m[0]); + if (px != null) { + long bad = 0; + for (int y = 0; y < 480; y++) for (int x = 0; x < 640; x++) if (px[y * 640 + x] != (0xFF000000 | pattern(x, y))) bad++; + check(bad == 0, "portal rgba pixels exact, alpha forced opaque (bad=" + bad + ")"); + } + try (var s = Files.list(shots)) { check(s.noneMatch(f -> f.toString().endsWith(".png")), "rgba file deleted"); } + + setMode("cancel"); + check(shot(5000, res, m) == null && res[0] == 5, "portal cancel -> 5, got " + res[0]); + setMode("error"); + check(shot(5000, res, m) == null && res[0] == 6, "portal error -> 6, got " + res[0] + " " + m[0]); + setMode("silent"); + long t0 = System.nanoTime(); + check(shot(1500, res, m) == null && res[0] == 7, "portal silent -> 7, got " + res[0]); + long took = (System.nanoTime() - t0) / 1_000_000; + System.out.println("portal timeout took " + took + " ms (asked 1500)"); + check(took >= 1400 && took < 3000, "timeout honoured: " + took); + setMode("bogus_uri"); + check(shot(5000, res, m) == null && res[0] == 6, "bogus uri -> 6, got " + res[0] + " " + m[0]); + setMode("missing_file"); + check(shot(5000, res, m) == null && res[0] == 6, "missing file -> 6, got " + res[0] + " " + m[0]); + setMode("not_png"); + check(shot(5000, res, m) == null && res[0] == 6, "garbage file -> 6, got " + res[0] + " " + m[0]); + setMode("no_uri"); + check(shot(5000, res, m) == null && res[0] == 6, "no uri -> 6, got " + res[0] + " " + m[0]); + setMode("delay:800"); + check(shot(5000, res, m) != null && res[0] == 0, "delayed response ok " + res[0] + " " + m[0]); + setMode("wrong_path"); + check(shot(1500, res, m) == null && res[0] == 7, "response on another path ignored -> 7, got " + res[0]); + setMode("ignore_token"); + check(shot(5000, res, m) != null && res[0] == 0, "portal ignoring handle_token still answered " + res[0] + " " + m[0]); + setMode("dbus_error"); + check(shot(5000, res, m) == null && res[0] == 6, "method error -> 6, got " + res[0] + " " + m[0]); + setMode("access_denied"); + check(shot(5000, res, m) == null && res[0] == 2, "AccessDenied -> 2, got " + res[0] + " " + m[0]); + setMode("early"); + check(shot(5000, res, m) != null && res[0] == 0, "response before the method reply " + res[0] + " " + m[0]); + setMode("symlink"); + px = shot(5000, res, m); + check(px != null && res[0] == 0, "symlinked screenshot decoded " + res[0] + " " + m[0]); + check(Files.exists(shots.resolve("target.png")), "symlink target kept"); + + // Concurrency + leaks. + setMode("ok"); + int threads = 8, per = Integer.parseInt(System.getProperty("portalIterations", "40")); + ExecutorService pool = Executors.newFixedThreadPool(threads); + AtomicLong errs = new AtomicLong(); + long fd0 = 0, rss0 = 0; + for (int phase = 0; phase < 2; phase++) { + List> fs = new ArrayList<>(); + for (int t = 0; t < threads; t++) fs.add(pool.submit(() -> { + int[] r = new int[3]; String[] mm = new String[1]; + for (int i = 0; i < per; i++) { + int[] p = NativeScreenCapture.nativePortalScreenshot(false, 10000, r, mm); + if (p == null || r[0] != 0 || p[641] != (0xFF000000 | pattern(1, 1))) { + errs.incrementAndGet(); + System.out.println("portal torture: " + r[0] + " " + mm[0]); + } + } + })); + for (Future f : fs) f.get(10, TimeUnit.MINUTES); + System.gc(); + Thread.sleep(300); + if (phase == 0) { fd0 = fds(); rss0 = rssKb(); } + System.out.println("portal torture phase " + phase + ": calls=" + (long) threads * per * (phase + 1) + " errors=" + errs.get() + " fds=" + fds() + " rss=" + rssKb() + "kB"); + } + pool.shutdown(); + check(errs.get() == 0, "portal torture errors"); + check(fds() <= fd0, "portal fd leak"); + check(rssKb() - rss0 < 64 * 1024, "portal rss growth " + (rssKb() - rss0)); + try (var s = Files.list(shots)) { check(s.filter(f -> f.toString().endsWith(".png") && !f.getFileName().toString().equals("target.png")).count() == 0, "no screenshot left behind"); } + } + + static void portalMissing() { + int[] res = new int[3]; String[] m = new String[1]; + check(NativeScreenCapture.nativePortalScreenshot(false, 3000, res, m) == null && res[0] == 1, "no portal -> UNSUPPORTED, got " + res[0] + " " + m[0]); + System.out.println("portal-missing message: " + m[0]); + } +} diff --git a/scripts/screen-capture-linux-e2e/harness/dev/nucleusframework/screencapture/internal/NativeScreenCapture.java b/scripts/screen-capture-linux-e2e/harness/dev/nucleusframework/screencapture/internal/NativeScreenCapture.java new file mode 100644 index 000000000..a909c9658 --- /dev/null +++ b/scripts/screen-capture-linux-e2e/harness/dev/nucleusframework/screencapture/internal/NativeScreenCapture.java @@ -0,0 +1,11 @@ +package dev.nucleusframework.screencapture.internal; + +public final class NativeScreenCapture { + public static native int nativeBackend(); + public static native int nativeListDisplays(DisplayCollector sink, String[] message); + public static native int[] nativeCaptureDisplay(String id, int x, int y, int w, int h, boolean cursor, int[] result, String[] message); + public static native int[] nativeCaptureWindow(long id, boolean cursor, int[] result, String[] message); + public static native int nativePermissionStatus(); + public static native int nativeRequestPermission(); + public static native int[] nativePortalScreenshot(boolean interactive, int timeoutMs, int[] result, String[] message); +} diff --git a/scripts/screen-capture-linux-e2e/xtool.c b/scripts/screen-capture-linux-e2e/xtool.c new file mode 100644 index 000000000..8fdad7e60 --- /dev/null +++ b/scripts/screen-capture-linux-e2e/xtool.c @@ -0,0 +1,120 @@ +// Test helper: paints known content on an X server. +// xtool pattern full-screen window painted with pattern(x, y); prints XID; runs until killed +// xtool window X Y W H RRGGBB solid window; prints XID; runs until killed +// xtool monitors defines RandR monitors LEFT (x +#include +#include +#include +#include +#include +#include +#include + +static unsigned pattern(int x, int y) { + unsigned r = x & 255, g = y & 255, b = ((x >> 8) * 64 + (y >> 8) * 16 + ((x + y) & 15)) & 255; + return (r << 16) | (g << 8) | b; +} + +static int ctz(unsigned long m) { int s = 0; while (m && !((m >> s) & 1)) s++; return s; } +static int bits(unsigned long m) { int s = ctz(m), b = 0; while ((m >> (s + b)) & 1) b++; return b; } + +static unsigned long to_pixel(Visual *v, unsigned rgb) { + unsigned long p = 0; + unsigned long masks[3] = {v->red_mask, v->green_mask, v->blue_mask}; + unsigned vals[3] = {(rgb >> 16) & 255, (rgb >> 8) & 255, rgb & 255}; + for (int i = 0; i < 3; i++) { + int s = ctz(masks[i]), b = bits(masks[i]); + unsigned q = b >= 8 ? vals[i] << (b - 8) : vals[i] >> (8 - b); + p |= ((unsigned long)q << s) & masks[i]; + } + return p; +} + +static Window make(Display *d, int x, int y, int w, int h) { + XSetWindowAttributes a; + a.override_redirect = True; + a.background_pixmap = None; + Window win = XCreateWindow(d, DefaultRootWindow(d), x, y, w, h, 0, CopyFromParent, InputOutput, CopyFromParent, + CWOverrideRedirect | CWBackPixmap, &a); + XSelectInput(d, win, ExposureMask); + XMapRaised(d, win); + return win; +} + +int main(int argc, char **argv) { + Display *d = XOpenDisplay(NULL); + if (!d) { fprintf(stderr, "no display\n"); return 1; } + int scr = DefaultScreen(d); + Visual *vis = DefaultVisual(d, scr); + int depth = DefaultDepth(d, scr); + if (argc >= 2 && strcmp(argv[1], "pattern") == 0) { + int w = DisplayWidth(d, scr), h = DisplayHeight(d, scr); + Window win = make(d, 0, 0, w, h); + XImage *img = XCreateImage(d, vis, depth, ZPixmap, 0, NULL, w, h, 32, 0); + img->data = malloc((size_t)img->bytes_per_line * h); + for (int y = 0; y < h; y++) + for (int x = 0; x < w; x++) XPutPixel(img, x, y, to_pixel(vis, pattern(x, y))); + GC gc = XCreateGC(d, win, 0, NULL); + printf("%lu\n", win); fflush(stdout); + for (;;) { + XEvent e; + XPutImage(d, win, gc, img, 0, 0, 0, 0, w, h); + XSync(d, False); + XNextEvent(d, &e); + } + } else if (argc >= 7 && strcmp(argv[1], "window") == 0) { + int x = atoi(argv[2]), y = atoi(argv[3]), w = atoi(argv[4]), h = atoi(argv[5]); + unsigned rgb = (unsigned)strtoul(argv[6], NULL, 16); + Window win = make(d, x, y, w, h); + unsigned long px; + if (vis->class == TrueColor) px = to_pixel(vis, rgb); + else { + XColor c; c.red = ((rgb >> 16) & 255) * 257; c.green = ((rgb >> 8) & 255) * 257; c.blue = (rgb & 255) * 257; + XAllocColor(d, DefaultColormap(d, scr), &c); px = c.pixel; + } + GC gc = XCreateGC(d, win, 0, NULL); + XSetForeground(d, gc, px); + printf("%lu\n", win); fflush(stdout); + for (;;) { + XEvent e; + XFillRectangle(d, win, gc, 0, 0, w, h); + XSync(d, False); + XNextEvent(d, &e); + } + } else if (argc >= 2 && strcmp(argv[1], "monitors") == 0) { + int w = DisplayWidth(d, scr), h = DisplayHeight(d, scr); + XRRMonitorInfo *m = XRRAllocateMonitor(d, 0); + m->name = XInternAtom(d, "LEFT", False); m->x = 0; m->y = 0; m->width = w / 2; m->height = h; + m->mwidth = 300; m->mheight = 300; m->primary = False; m->automatic = False; m->noutput = 0; + XRRSetMonitor(d, DefaultRootWindow(d), m); + m->name = XInternAtom(d, "RIGHT", False); m->x = w / 2; m->width = w - w / 2; m->primary = True; + XRRSetMonitor(d, DefaultRootWindow(d), m); + XSync(d, False); + int n; XRRMonitorInfo *all = XRRGetMonitors(d, DefaultRootWindow(d), True, &n); + printf("monitors=%d\n", n); + XRRFreeMonitors(all); + } else if (argc >= 4 && strcmp(argv[1], "warp") == 0) { + XWarpPointer(d, None, DefaultRootWindow(d), 0, 0, 0, 0, atoi(argv[2]), atoi(argv[3])); + XSync(d, False); + } else if (argc >= 2 && strcmp(argv[1], "redirect") == 0) { + XCompositeRedirectSubwindows(d, DefaultRootWindow(d), CompositeRedirectAutomatic); + XSync(d, False); + printf("redirected\n"); fflush(stdout); + for (;;) pause(); + } else if (argc >= 2 && strcmp(argv[1], "destroyed") == 0) { + Window win = XCreateSimpleWindow(d, DefaultRootWindow(d), 0, 0, 10, 10, 0, 0, 0); + XSync(d, False); + XDestroyWindow(d, win); + XSync(d, False); + printf("%lu\n", win); + } else { + fprintf(stderr, "usage\n"); + return 2; + } + XCloseDisplay(d); + return 0; +} diff --git a/scripts/screen-capture-windows-e2e.ps1 b/scripts/screen-capture-windows-e2e.ps1 new file mode 100644 index 000000000..926f25844 --- /dev/null +++ b/scripts/screen-capture-windows-e2e.ps1 @@ -0,0 +1,140 @@ +<# +.SYNOPSIS + Windows E2E of screen-capture: runs examples/screen-capture-demo in self-test mode. + +.DESCRIPTION + The demo draws a pixel-exact pattern and checks window, display, region, occluded, + minimized, own-UI-thread and blocked-UI-thread captures against it, then tortures the API + from many threads. This script adds what the demo cannot do on its own: + - parks the cursor at the centre of the primary display (cursor compositing check); + - opens a window whose thread never pumps (a hung foreign window, PrintWindow's trap); + - samples the demo's GDI / USER objects and kernel handles around the torture phase: + a leak there grows with every capture. + Exits with the number of failures. + +.EXAMPLE + powershell -NoProfile -ExecutionPolicy Bypass -File scripts\screen-capture-windows-e2e.ps1 -Torture 2000 +#> +param( + [int]$Torture = 800, + [int]$TimeoutSeconds = 900, + [string]$OutDir = (Join-Path $env:TEMP "screen-capture-e2e") +) + +$ErrorActionPreference = "Stop" +$root = Split-Path -Parent $PSScriptRoot + +Add-Type -AssemblyName System.Windows.Forms +Add-Type @" +using System; +using System.Runtime.InteropServices; +public static class ScE2e { + [DllImport("user32.dll")] public static extern uint GetGuiResources(IntPtr process, uint flags); + [DllImport("kernel32.dll")] public static extern IntPtr OpenProcess(uint access, bool inherit, int pid); + [DllImport("kernel32.dll")] public static extern bool CloseHandle(IntPtr h); + public static uint[] Gui(int pid) { + IntPtr h = OpenProcess(0x1000 /* QUERY_LIMITED_INFORMATION */, false, pid); + if (h == IntPtr.Zero) return new uint[] { 0, 0 }; + try { return new uint[] { GetGuiResources(h, 0), GetGuiResources(h, 1) }; } finally { CloseHandle(h); } + } +} +"@ + +New-Item -ItemType Directory -Force -Path $OutDir | Out-Null +$log = Join-Path $OutDir "selftest.log" +Remove-Item $log -ErrorAction SilentlyContinue + +# A window whose thread spins without pumping messages: PrintWindow would wait on it forever. +$hungScript = Join-Path $OutDir "hung-window.ps1" +@' +Add-Type -AssemblyName System.Windows.Forms +$f = New-Object System.Windows.Forms.Form +$f.Text = "ScreenCaptureE2eHung" +$f.StartPosition = "Manual" +$f.WindowState = "Normal" +$f.Location = New-Object System.Drawing.Point(40, 40) +$f.BackColor = [System.Drawing.Color]::FromArgb(255, 20, 160, 60) +$f.Show() +$f.WindowState = "Normal" +[System.Windows.Forms.Application]::DoEvents() +Start-Sleep -Milliseconds 200 +[System.Windows.Forms.Application]::DoEvents() +[Int64]$f.Handle | Set-Content -Encoding ASCII $args[0] +$end = [DateTime]::Now.AddSeconds(600) +while ([DateTime]::Now -lt $end) { } +'@ | Set-Content -Encoding ASCII $hungScript +$hwndFile = Join-Path $OutDir "hung-window.hwnd" +Remove-Item $hwndFile -ErrorAction SilentlyContinue +$hung = Start-Process powershell -ArgumentList "-NoProfile", "-ExecutionPolicy", "Bypass", "-File", $hungScript, $hwndFile -PassThru -WindowStyle Minimized +$hwnd = 0 +for ($i = 0; $i -lt 150 -and $hwnd -eq 0; $i++) { + Start-Sleep -Milliseconds 200 + if (Test-Path $hwndFile) { $hwnd = [Int64](Get-Content $hwndFile -Raw).Trim() } +} +Write-Host "hung window: $hwnd (pid $($hung.Id))" + +$primary = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds +[System.Windows.Forms.Cursor]::Position = New-Object System.Drawing.Point( + ($primary.X + [int]($primary.Width / 2)), ($primary.Y + [int]($primary.Height / 2))) + +$env:SCREEN_CAPTURE_DEMO_SELFTEST = "1" +$env:SCREEN_CAPTURE_DEMO_LOG = $log +$env:SCREEN_CAPTURE_DEMO_OUT = $OutDir +$env:SCREEN_CAPTURE_DEMO_TORTURE = "$Torture" +$env:SCREEN_CAPTURE_DEMO_HUNG_HWND = "$hwnd" +# The parked cursor shows (it may be moved since: the check finds it wherever it is). +$env:SCREEN_CAPTURE_DEMO_CURSOR_EXPECTED = "1" + +$gradle = Start-Process -FilePath (Join-Path $root "gradlew.bat") ` + -ArgumentList ":examples:screen-capture-demo:run", "--console=plain", "-q" ` + -WorkingDirectory $root -PassThru -WindowStyle Hidden ` + -RedirectStandardOutput (Join-Path $OutDir "gradle.out") -RedirectStandardError (Join-Path $OutDir "gradle.err") + +$deadline = (Get-Date).AddSeconds($TimeoutSeconds) +$appPid = 0 +$before = $null +$after = $null +while ((Get-Date) -lt $deadline) { + Start-Sleep -Milliseconds 500 + if (-not (Test-Path $log)) { if ($gradle.HasExited) { break } else { continue } } + $text = Get-Content $log -Raw + if ($appPid -eq 0 -and $text -match "START pid=(\d+)") { $appPid = [int]$Matches[1] } + if ($appPid -ne 0 -and $null -eq $before -and $text -match "PHASE torture-begin") { + Start-Sleep -Milliseconds 1000 + $gui = [ScE2e]::Gui($appPid) + $before = @{ gdi = $gui[0]; user = $gui[1]; handles = (Get-Process -Id $appPid).HandleCount } + } + if ($appPid -ne 0 -and $null -eq $after -and $text -match "PHASE torture-end") { + Start-Sleep -Milliseconds 1500 + $gui = [ScE2e]::Gui($appPid) + $after = @{ gdi = $gui[0]; user = $gui[1]; handles = (Get-Process -Id $appPid).HandleCount } + } + if ($text -match "DONE failures=") { break } +} +$gradle.WaitForExit(120000) | Out-Null +Stop-Process -Id $hung.Id -Force -ErrorAction SilentlyContinue + +$failures = 0 +if (Test-Path $log) { + $lines = Get-Content $log + $lines | Where-Object { $_ -match " (FAIL|INFO|SKIP|DISPLAY) " } | ForEach-Object { Write-Host $_ } + $failures += ($lines | Where-Object { $_ -match " FAIL " }).Count + Write-Host ("passed checks: " + ($lines | Where-Object { $_ -match " PASS " }).Count) + if (-not ($lines -match "DONE failures=")) { Write-Host "FAIL self-test did not finish"; $failures++ } +} else { + Write-Host "FAIL no self-test log"; $failures++ +} + +if ($null -ne $before -and $null -ne $after) { + Write-Host ("resources before torture: gdi={0} user={1} handles={2}" -f $before.gdi, $before.user, $before.handles) + Write-Host ("resources after torture: gdi={0} user={1} handles={2}" -f $after.gdi, $after.user, $after.handles) + # Threads, sockets and JIT come and go; a per-capture leak would be thousands. + if ($after.gdi - $before.gdi -gt 20) { Write-Host "FAIL GDI objects leak"; $failures++ } + if ($after.user - $before.user -gt 20) { Write-Host "FAIL USER objects leak"; $failures++ } + if ($after.handles - $before.handles -gt 200) { Write-Host "FAIL kernel handles leak"; $failures++ } +} else { + Write-Host "FAIL resources were not sampled"; $failures++ +} + +Write-Host "failures=$failures" +exit $failures diff --git a/settings.gradle.kts b/settings.gradle.kts index b28a4ca8b..253da5978 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -42,6 +42,7 @@ include(":native-http-ktor") include(":linux-hidpi") include(":spellcheck") include(":system-color") +include(":screen-capture") include(":decorated-window-core") include(":decorated-window-tao") include(":nucleus-application") @@ -102,4 +103,5 @@ include(":examples:watermark-demo") include(":examples:widget-demo") include(":examples:macos-appex-demo") include(":examples:hot-update-demo") +include(":examples:screen-capture-demo") includeBuild("plugin-build") From eb37f2da908bfc0aa7adb56fd39f2c2ee5010d3c Mon Sep 17 00:00:00 2001 From: Elie Gambache Date: Sat, 26 Sep 2026 22:04:59 +0300 Subject: [PATCH 2/4] fix(screen-capture/macos): exact regions, minimized windows, window scale Found by running the Windows E2E protocol on macOS 26 (Retina, 2x): - A region starting on an odd pixel was resampled: sourceRect is in points. A dithered gradient also came back up to 2 levels off the full capture, because the WindowServer re-renders the rect it is given with a 32 px tile anchored there. The region is now captured from a 32 pt grid-aligned rect and cropped out, which is bit-exact on both. - ScreenCaptureKit lists minimized windows with their last contents: a window that is not on screen is now WindowNotFound, as documented. - captureWindow always reported scaleFactor 1. The native result gains a scale slot (thousandths): the window's backing scale on macOS, its monitor's DPI scale on Windows, 1 on X11. - NucleusScreenCapture.m had a stray cp1252 byte; it is UTF-8 now. - CaptureDisplay documents that macOS scaled modes capture the backing store's pixels, not the panel's. scripts/screen-capture-macos-e2e.sh runs the demo's self-test like the Windows script: it hands the demo its CGWindowID (Tao does not expose it yet), parks the cursor with a real motion event, opens a never-pumping window and samples fds, threads, Mach ports and RSS around the torture. The self-test now also checks the occluded window pixel-exact and the window scale on macOS. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/main/kotlin/screencapturedemo/Main.kt | 21 ++++- .../main/kotlin/screencapturedemo/SelfTest.kt | 13 ++- .../screencapture/CaptureDisplay.kt | 4 +- .../screencapture/ScreenCapture.kt | 6 +- .../screencapture/internal/NativeCall.kt | 10 ++- .../main/native/macos/NucleusScreenCapture.m | 80 +++++++++++++++++-- .../windows/nucleus_screencapture_windows.c | 12 +++ scripts/screen-capture-macos-e2e.sh | 77 ++++++++++++++++++ scripts/screen-capture-macos-e2e/helper.swift | 34 ++++++++ 9 files changed, 240 insertions(+), 17 deletions(-) create mode 100755 scripts/screen-capture-macos-e2e.sh create mode 100644 scripts/screen-capture-macos-e2e/helper.swift diff --git a/examples/screen-capture-demo/src/main/kotlin/screencapturedemo/Main.kt b/examples/screen-capture-demo/src/main/kotlin/screencapturedemo/Main.kt index 105ec439f..8fea65e57 100644 --- a/examples/screen-capture-demo/src/main/kotlin/screencapturedemo/Main.kt +++ b/examples/screen-capture-demo/src/main/kotlin/screencapturedemo/Main.kt @@ -70,7 +70,7 @@ fun main(args: Array) = val test = SelfTest( window = window, - windowId = windowIdOf(window), + windowId = windowIdOf(window) ?: awaitMacWindowId(), showCover = { visible -> withContext(Dispatchers.Main) { coverVisible = visible } }, setMinimized = { minimized -> withContext(Dispatchers.Main) { window.setMinimized(minimized) } @@ -97,14 +97,29 @@ fun main(args: Array) = } } -/** The id [ScreenCapture.captureWindow] takes for a Tao window; `0` where the demo has none. */ -private fun windowIdOf(window: TaoWindow): Long = +/** The id [ScreenCapture.captureWindow] takes for a Tao window; `0` where it has none, `null` on macOS. */ +private fun windowIdOf(window: TaoWindow): Long? = when (Platform.Current) { Platform.Windows -> window.nativeHandle Platform.Linux -> window.x11WindowId ?: 0L + Platform.MacOS -> null else -> 0L } +/** + * macOS: Tao does not expose the NSWindow's CGWindowID yet; the E2E script looks it up by pid and + * title and writes it to SCREEN_CAPTURE_DEMO_WINDOW_ID_FILE. + */ +private suspend fun awaitMacWindowId(): Long { + val file = System.getenv("SCREEN_CAPTURE_DEMO_WINDOW_ID_FILE")?.let(::File) ?: return 0L + repeat(300) { + val id = runCatching { file.readText().trim().toLongOrNull() }.getOrNull() + if (id != null) return id + kotlinx.coroutines.delay(100) + } + return 0L +} + @Composable private fun DemoContent(window: TaoWindow?) { val scope = rememberCoroutineScope() diff --git a/examples/screen-capture-demo/src/main/kotlin/screencapturedemo/SelfTest.kt b/examples/screen-capture-demo/src/main/kotlin/screencapturedemo/SelfTest.kt index 34d71ae7d..c1b5401ff 100644 --- a/examples/screen-capture-demo/src/main/kotlin/screencapturedemo/SelfTest.kt +++ b/examples/screen-capture-demo/src/main/kotlin/screencapturedemo/SelfTest.kt @@ -141,6 +141,12 @@ internal class SelfTest( val image: ScreenImage val ms = measureTimeMillis { image = ScreenCapture.captureWindow(windowId) } log("INFO window capture ${image.width}x${image.height} in ${ms}ms") + if (Platform.Current != Platform.Linux) { + val displayScale = displayUnderWindow(ScreenCapture.displays()).scaleFactor + check("window scale", kotlin.math.abs(image.scaleFactor - displayScale) < 0.01f) { + "${image.scaleFactor} vs $displayScale" + } + } expectPattern("window", image) saveSample("window", image) } @@ -183,7 +189,10 @@ internal class SelfTest( val shot = ScreenCapture.captureDisplay(display, shifted) val expected = (-dx) to (-dy) if (dx > 0 || dy > 0) continue // the marker is cut; checked through the full image below - if (TargetPattern.find(shot) != listOf(expected)) misaligned++ + if (TargetPattern.find(shot) != listOf(expected)) { + misaligned++ + log("INFO shifted dx=$dx dy=$dy $shifted expected=$expected found=${TargetPattern.find(shot)}") + } } } check("shifted regions aligned", misaligned == 0) { "misaligned=$misaligned" } @@ -226,7 +235,7 @@ internal class SelfTest( TargetPattern.find(screen).isEmpty(), ) { "${TargetPattern.find(screen)}" } val image = ScreenCapture.captureWindow(windowId) - if (Platform.Current == Platform.Windows) { + if (Platform.Current == Platform.Windows || Platform.Current == Platform.MacOS) { expectPattern("occluded window", image) } else { log("INFO occluded window found=${TargetPattern.find(image)}") diff --git a/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/CaptureDisplay.kt b/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/CaptureDisplay.kt index 5d8c1f0ef..d46016660 100644 --- a/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/CaptureDisplay.kt +++ b/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/CaptureDisplay.kt @@ -12,7 +12,9 @@ package dev.nucleusframework.screencapture * space on macOS (origin at the primary display's top-left). `null` on Wayland, where the * compositor does not expose it. * @property widthPx width of a full capture of this display, in physical pixels; `0` while - * unknown (Wayland, before the first capture). + * unknown (Wayland, before the first capture). On macOS these are the backing store's + * pixels: in a scaled display mode ("looks like 1710 × 1112" at 2x) a capture is + * 3420 × 2224 even when the panel itself has fewer pixels. * @property heightPx height of a full capture of this display, in physical pixels; `0` while * unknown. * @property scaleFactor physical pixels per [bounds] unit on macOS, the display's DPI scale diff --git a/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/ScreenCapture.kt b/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/ScreenCapture.kt index 1409a912c..b2c3aa025 100644 --- a/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/ScreenCapture.kt +++ b/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/ScreenCapture.kt @@ -143,7 +143,9 @@ public object ScreenCapture { /** * Captures a top-level window, including the parts other windows cover, when the platform * can: an `HWND` on Windows, a `CGWindowID` (`NSWindow.windowNumber`) on macOS, an XID on - * X11. Not available on Wayland ([isWindowCaptureSupported]). + * X11. Not available on Wayland ([isWindowCaptureSupported]). The image's + * [ScreenImage.scaleFactor] is the window's backing scale on macOS, its monitor's DPI scale on + * Windows, and `1` on X11. * * @throws ScreenCaptureException see [ScreenCaptureException.failure]. */ @@ -157,7 +159,7 @@ public object ScreenCapture { } val call = NativeCall("capture window $windowId") val pixels = NativeScreenCapture.nativeCaptureWindow(windowId, includeCursor, call.result, call.message) - return call.image(pixels, 1f) + return call.image(pixels, call.reportedScale(1f)) } /** Whether the app may capture the screen, without prompting. */ diff --git a/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/internal/NativeCall.kt b/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/internal/NativeCall.kt index 5621ba4f0..86b7bb15d 100644 --- a/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/internal/NativeCall.kt +++ b/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/internal/NativeCall.kt @@ -10,7 +10,7 @@ import dev.nucleusframework.screencapture.ScreenImage internal class NativeCall( private val what: String, ) { - /** `[status, width, height]`. */ + /** `[status, width, height, scale × 1000]`; the scale is `0` where the native side reports none. */ val result = IntArray(RESULT_SIZE) /** A diagnostic from the native side, on failure. */ @@ -33,6 +33,10 @@ internal class NativeCall( return ScreenImage(result[1], result[2], checkNotNull(pixels), scaleFactor) } + /** The scale the native side reported for the captured source, or [fallback]. */ + fun reportedScale(fallback: Float): Float = + if (result[SCALE_INDEX] > 0) result[SCALE_INDEX] / SCALE_UNIT else fallback + private fun failureOf(status: Int): CaptureFailure = when (status) { NativeScreenCapture.STATUS_UNSUPPORTED -> CaptureFailure.Unsupported @@ -46,7 +50,9 @@ internal class NativeCall( } private companion object { - const val RESULT_SIZE = 3 + const val RESULT_SIZE = 4 + const val SCALE_INDEX = 3 + const val SCALE_UNIT = 1000f } } diff --git a/screen-capture/src/main/native/macos/NucleusScreenCapture.m b/screen-capture/src/main/native/macos/NucleusScreenCapture.m index 17dabefc1..3502c93ef 100644 --- a/screen-capture/src/main/native/macos/NucleusScreenCapture.m +++ b/screen-capture/src/main/native/macos/NucleusScreenCapture.m @@ -79,6 +79,13 @@ static void setResult(JNIEnv *env, jintArray result, jint status, jint width, ji (*env)->SetIntArrayRegion(env, result, 0, 3, values); } +// Reports the captured source's scale (px per point), in the result's fourth slot. +static void setScale(JNIEnv *env, jintArray result, CGFloat scale) { + if (result == NULL || (*env)->GetArrayLength(env, result) < 4 || !(scale > 0)) return; + jint value = (jint)lround(scale * 1000); + (*env)->SetIntArrayRegion(env, result, 3, 1, &value); +} + static void setMessage(JNIEnv *env, jobjectArray message, NSString *text) { if (message == NULL || text == nil || (*env)->GetArrayLength(env, message) < 1) return; jstring str = (*env)->NewStringUTF(env, text.UTF8String ?: ""); @@ -134,6 +141,37 @@ static jintArray imageToPixels(JNIEnv *env, CGImageRef image, jintArray result, return pixels; } +// A region is captured from a grid-aligned rect around it and cropped back out: +// - a region starting mid-point (an odd pixel at 2x) is otherwise resampled; +// - the WindowServer re-renders the rect it is asked for, and dithered content (a gradient +// wallpaper) follows a 32 px tile anchored at that rect: an unaligned origin gives pixels up +// to 2 levels off the full capture's. +// ponytail: fixed 32 pt grid (>= 32 px at every scale), measured on macOS 26 at 2x; widen it if +// a display shows a larger dither tile. +#define CAPTURE_GRID_POINTS 32.0 + +static CGRect pointRectAround(CGRect regionPx, CGFloat scale) { + CGFloat left = floor(regionPx.origin.x / scale / CAPTURE_GRID_POINTS) * CAPTURE_GRID_POINTS; + CGFloat top = floor(regionPx.origin.y / scale / CAPTURE_GRID_POINTS) * CAPTURE_GRID_POINTS; + CGFloat right = ceil(CGRectGetMaxX(regionPx) / scale); + CGFloat bottom = ceil(CGRectGetMaxY(regionPx) / scale); + return CGRectMake(left, top, right - left, bottom - top); +} + +// +1 image: the [regionPx] pixels of [image], a capture of [points]. Consumes [image]. +static CGImageRef cropToRegion(CGImageRef image, CGRect points, CGRect regionPx, CGFloat scale) { + CGRect crop = CGRectMake(regionPx.origin.x - lround(points.origin.x * scale), + regionPx.origin.y - lround(points.origin.y * scale), + regionPx.size.width, regionPx.size.height); + if (crop.origin.x == 0 && crop.origin.y == 0 && + CGImageGetWidth(image) == (size_t)crop.size.width && CGImageGetHeight(image) == (size_t)crop.size.height) { + return image; + } + CGImageRef cropped = CGImageCreateWithImageInRect(image, crop); + CGImageRelease(image); + return cropped; +} + // --------------------------------------------------------------------------------------------- // ScreenCaptureKit plumbing // --------------------------------------------------------------------------------------------- @@ -154,7 +192,7 @@ - (void)dealloc { @end // Waits for [sem]. ScreenCaptureKit completes on its own queue, so a plain wait is safe on any -// thread — including the main thread, whose run loop is deliberately not pumped here: turning +// thread — including the main thread, whose run loop is deliberately not pumped here: turning // it would dispatch the app's own events re-entrantly from inside a capture call. static BOOL waitFor(dispatch_semaphore_t sem) { return dispatch_semaphore_wait(sem, dispatch_time(DISPATCH_TIME_NOW, (int64_t)(CAPTURE_TIMEOUT_SECONDS * NSEC_PER_SEC))) == 0; @@ -384,15 +422,17 @@ static jintArray captureDisplayWithKit( SCContentFilter *filter = [[SCContentFilter alloc] initWithDisplay:target excludingWindows:@[]]; SCStreamConfiguration *config = [SCStreamConfiguration new]; configureCommon(config, includeCursor); - config.width = (size_t)regionPx.size.width; - config.height = (size_t)regionPx.size.height; // sourceRect is in points, relative to the display. - config.sourceRect = CGRectMake(regionPx.origin.x / scale, regionPx.origin.y / scale, - regionPx.size.width / scale, regionPx.size.height / scale); + CGRect points = pointRectAround(regionPx, scale); + config.width = (size_t)lround(points.size.width * scale); + config.height = (size_t)lround(points.size.height * scale); + config.sourceRect = points; CGImageRef image = NULL; status = screenshot(filter, config, &image, &why); if (status != STATUS_OK) return fail(env, result, message, status, why); + image = cropToRegion(image, points, regionPx, scale); + if (image == NULL) return fail(env, result, message, STATUS_FAILED, @"Cannot crop the capture to the region"); jintArray pixels = imageToPixels(env, image, result, message); CGImageRelease(image); return pixels; @@ -409,9 +449,9 @@ static jintArray captureDisplayWithCoreGraphics( if (!CGPreflightScreenCaptureAccess()) { return fail(env, result, message, STATUS_PERMISSION_DENIED, @"Screen recording permission is not granted"); } - CGRect points = CGRectMake(regionPx.origin.x / scale, regionPx.origin.y / scale, - regionPx.size.width / scale, regionPx.size.height / scale); + CGRect points = pointRectAround(regionPx, scale); CGImageRef image = create(displayId, points); + if (image != NULL) image = cropToRegion(image, points, regionPx, scale); if (image == NULL) { return fail(env, result, message, STATUS_FAILED, @"CGDisplayCreateImageForRect returned nothing"); } @@ -490,6 +530,11 @@ static jintArray captureWindowWithKit( return fail(env, result, message, STATUS_WINDOW_NOT_FOUND, [NSString stringWithFormat:@"ScreenCaptureKit does not list window %u", windowId]); } + // Minimized and hidden windows are listed too, with their last contents. + if (!target.onScreen) { + return fail(env, result, message, STATUS_WINDOW_NOT_FOUND, + [NSString stringWithFormat:@"Window %u is not on screen", windowId]); + } SCContentFilter *filter = [[SCContentFilter alloc] initWithDesktopIndependentWindow:target]; CGRect rect = filter.contentRect; @@ -511,9 +556,29 @@ static jintArray captureWindowWithKit( if (status != STATUS_OK) return fail(env, result, message, status, why); jintArray pixels = imageToPixels(env, image, result, message); CGImageRelease(image); + if (pixels != NULL) setScale(env, result, scale); return pixels; } +// Px per point of a Core Graphics window image: its width over the window's bounds, in points. +static CGFloat coreGraphicsWindowScale(CGWindowID windowId, size_t widthPx) { + CGFloat scale = 0; + const void *values[] = {(const void *)(uintptr_t)windowId}; + CFArrayRef ids = CFArrayCreate(NULL, values, 1, NULL); + CFArrayRef info = ids != NULL ? CGWindowListCreateDescriptionFromArray(ids) : NULL; + if (info != NULL && CFArrayGetCount(info) > 0) { + CFDictionaryRef entry = CFArrayGetValueAtIndex(info, 0); + CFDictionaryRef boundsDict = CFDictionaryGetValue(entry, kCGWindowBounds); + CGRect bounds; + if (boundsDict != NULL && CGRectMakeWithDictionaryRepresentation(boundsDict, &bounds) && bounds.size.width > 0) { + scale = (CGFloat)widthPx / bounds.size.width; + } + } + if (info != NULL) CFRelease(info); + if (ids != NULL) CFRelease(ids); + return scale; +} + static jintArray captureWindowWithCoreGraphics( JNIEnv *env, CGWindowID windowId, jintArray result, jobjectArray message) { CreateWindowListImageFn create = cgCreateWindowListImage(); @@ -531,6 +596,7 @@ static jintArray captureWindowWithCoreGraphics( [NSString stringWithFormat:@"No capturable window %u", windowId]); } jintArray pixels = imageToPixels(env, image, result, message); + if (pixels != NULL) setScale(env, result, coreGraphicsWindowScale(windowId, CGImageGetWidth(image))); CGImageRelease(image); return pixels; } diff --git a/screen-capture/src/main/native/windows/nucleus_screencapture_windows.c b/screen-capture/src/main/native/windows/nucleus_screencapture_windows.c index 612442d6a..683e592e3 100644 --- a/screen-capture/src/main/native/windows/nucleus_screencapture_windows.c +++ b/screen-capture/src/main/native/windows/nucleus_screencapture_windows.c @@ -90,6 +90,13 @@ static void set_result(JNIEnv *env, jintArray result, int status, int width, int (*env)->SetIntArrayRegion(env, result, 0, 3, values); } +/* The captured source's scale (dpi / 96), in the result's fourth slot as thousandths. */ +static void set_scale(JNIEnv *env, jintArray result, UINT dpi) { + if (result == NULL || (*env)->GetArrayLength(env, result) < 4 || dpi == 0) return; + jint value = (jint)((dpi * 1000 + 48) / 96); + (*env)->SetIntArrayRegion(env, result, 3, 1, &value); +} + static void set_message(JNIEnv *env, jobjectArray message, const char *text) { if (message == NULL || text == NULL) return; jstring s = (*env)->NewStringUTF(env, text); @@ -558,6 +565,11 @@ Java_dev_nucleusframework_screencapture_internal_NativeScreenCapture_nativeCaptu } surface_close(&surface); } + if (pixels != NULL && g_get_dpi_for_monitor != NULL) { + UINT dx = 0, dy = 0; + HMONITOR monitor = MonitorFromWindow(hwnd, MONITOR_DEFAULTTONEAREST); + if (SUCCEEDED(g_get_dpi_for_monitor(monitor, 0 /* MDT_EFFECTIVE_DPI */, &dx, &dy))) set_scale(env, result, dx); + } leave_physical_pixels(old); return pixels; } diff --git a/scripts/screen-capture-macos-e2e.sh b/scripts/screen-capture-macos-e2e.sh new file mode 100755 index 000000000..a8a7e2d59 --- /dev/null +++ b/scripts/screen-capture-macos-e2e.sh @@ -0,0 +1,77 @@ +#!/bin/bash +# macOS E2E of screen-capture: runs examples/screen-capture-demo in self-test mode, the same +# protocol as screen-capture-windows-e2e.ps1. On top of the demo's own checks this script: +# - hands the demo its CGWindowID (Tao does not expose it yet); +# - parks the cursor at the centre of the main display (cursor compositing check); +# - opens a window whose app never pumps its run loop again (a hung foreign window); +# - samples the demo's file descriptors, threads, Mach ports and RSS around the torture phase. +# Needs the Screen Recording permission for the terminal running it. Exits with the failure count. +# +# scripts/screen-capture-macos-e2e.sh [torture=800] [timeoutSeconds=900] +set -uo pipefail +TORTURE=${1:-800} +TIMEOUT=${2:-900} +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +OUT=${OUT_DIR:-${TMPDIR:-/tmp}/screen-capture-e2e} +mkdir -p "$OUT" +LOG="$OUT/selftest.log"; IDFILE="$OUT/window.id"; HUNGFILE="$OUT/hung.id" +rm -f "$LOG" "$IDFILE" "$HUNGFILE" + +HELPER="$OUT/helper" +swiftc -O -o "$HELPER" "$ROOT/scripts/screen-capture-macos-e2e/helper.swift" || exit 1 + +"$HELPER" hung "$HUNGFILE" & HUNG_PID=$! +for _ in $(seq 50); do [ -s "$HUNGFILE" ] && break; sleep 0.2; done +HUNG_ID=$(cat "$HUNGFILE" 2>/dev/null || echo 0) +echo "hung window: $HUNG_ID (pid $HUNG_PID)" +"$HELPER" park + +sample() { # fds threads ports rssKb + echo "$(lsof -p "$1" 2>/dev/null | wc -l | tr -d ' ') $(($(ps -M -p "$1" | wc -l) - 1)) \ +$(top -l 1 -pid "$1" -stats ports | tail -1 | tr -dc '0-9') $(ps -o rss= -p "$1" | tr -d ' ')" +} + +SCREEN_CAPTURE_DEMO_SELFTEST=1 SCREEN_CAPTURE_DEMO_LOG="$LOG" SCREEN_CAPTURE_DEMO_OUT="$OUT" \ +SCREEN_CAPTURE_DEMO_TORTURE="$TORTURE" SCREEN_CAPTURE_DEMO_HUNG_HWND="$HUNG_ID" \ +SCREEN_CAPTURE_DEMO_CURSOR_EXPECTED=1 SCREEN_CAPTURE_DEMO_WINDOW_ID_FILE="$IDFILE" \ + "$ROOT/gradlew" -p "$ROOT" :examples:screen-capture-demo:run --console=plain -q >"$OUT/gradle.out" 2>&1 & +GRADLE=$! + +deadline=$((SECONDS + TIMEOUT)); APP=0; BEFORE=""; AFTER="" +while [ $SECONDS -lt $deadline ]; do + sleep 0.5 + if [ ! -s "$IDFILE" ]; then + id=$("$HELPER" windowid 0 "Screen Capture Demo" 2>/dev/null) && echo "$id" >"$IDFILE" && echo "demo window: $id" + fi + [ -f "$LOG" ] || { kill -0 $GRADLE 2>/dev/null && continue || break; } + [ $APP = 0 ] && APP=$(sed -n 's/.*START pid=\([0-9]*\).*/\1/p' "$LOG" | head -1) && APP=${APP:-0} + if [ $APP != 0 ] && [ -z "$BEFORE" ] && grep -q "PHASE torture-begin" "$LOG"; then sleep 1; BEFORE=$(sample $APP); fi + if [ $APP != 0 ] && [ -z "$AFTER" ] && grep -q "PHASE torture-end" "$LOG"; then sleep 1.5; AFTER=$(sample $APP); fi + grep -q "DONE failures=" "$LOG" && break +done +wait $GRADLE +kill -9 $HUNG_PID 2>/dev/null + +FAILURES=0 +if [ -f "$LOG" ]; then + grep -E " (FAIL|INFO|SKIP|DISPLAY) " "$LOG" + FAILURES=$(grep -c " FAIL " "$LOG") + echo "passed checks: $(grep -c " PASS " "$LOG")" + grep -q "DONE failures=" "$LOG" || { echo "FAIL self-test did not finish"; FAILURES=$((FAILURES + 1)); } +else + echo "FAIL no self-test log"; FAILURES=$((FAILURES + 1)) +fi +if [ -n "$BEFORE" ] && [ -n "$AFTER" ]; then + read -r f0 t0 p0 r0 <<<"$BEFORE"; read -r f1 t1 p1 r1 <<<"$AFTER" + echo "resources before torture: fds=$f0 threads=$t0 ports=$p0 rssMb=$((r0 / 1024))" + echo "resources after torture: fds=$f1 threads=$t1 ports=$p1 rssMb=$((r1 / 1024))" + # Threads and JIT come and go; a per-capture leak would be thousands (or GBs of pixels). + [ $((f1 - f0)) -gt 50 ] && { echo "FAIL file descriptor leak"; FAILURES=$((FAILURES + 1)); } + [ $((p1 - p0)) -gt 200 ] && { echo "FAIL Mach port leak"; FAILURES=$((FAILURES + 1)); } + [ $((t1 - t0)) -gt 50 ] && { echo "FAIL thread leak"; FAILURES=$((FAILURES + 1)); } + [ $((r1 - r0)) -gt $((1024 * 1024)) ] && { echo "FAIL RSS grew by over 1 GB"; FAILURES=$((FAILURES + 1)); } +else + echo "FAIL resources were not sampled"; FAILURES=$((FAILURES + 1)) +fi +echo "failures=$FAILURES" +exit $FAILURES diff --git a/scripts/screen-capture-macos-e2e/helper.swift b/scripts/screen-capture-macos-e2e/helper.swift new file mode 100644 index 000000000..c64bcf2d9 --- /dev/null +++ b/scripts/screen-capture-macos-e2e/helper.swift @@ -0,0 +1,34 @@ +// Helper of screen-capture-macos-e2e.sh. +// helper windowid CGWindowID of the on-screen window of <pid> (0: any) named <title> +// helper park cursor to the centre of the main display +// helper hung <file> opens a window, writes its CGWindowID to <file>, never pumps again +import AppKit + +let args = CommandLine.arguments +switch args.count > 1 ? args[1] : "" { +case "windowid": + let pid = Int32(args[2])! + let list = CGWindowListCopyWindowInfo([.optionOnScreenOnly], kCGNullWindowID) as? [[String: Any]] ?? [] + for w in list where (pid == 0 || (w[kCGWindowOwnerPID as String] as? Int32) == pid) && (w[kCGWindowName as String] as? String) == args[3] { + print(w[kCGWindowNumber as String] as! Int); exit(0) + } + exit(1) +case "park": + let b = CGDisplayBounds(CGMainDisplayID()) + // A real motion event: a warp alone leaves a cursor hidden by setHiddenUntilMouseMoves hidden. + CGEvent(mouseEventSource: nil, mouseType: .mouseMoved, mouseCursorPosition: CGPoint(x: b.midX, y: b.midY), mouseButton: .left)? + .post(tap: .cghidEventTap) +case "hung": + let app = NSApplication.shared + app.setActivationPolicy(.regular) + let w = NSWindow(contentRect: NSRect(x: 40, y: 40, width: 300, height: 200), styleMask: [.titled], backing: .buffered, defer: false) + w.title = "ScreenCaptureE2eHung" + w.backgroundColor = NSColor(srgbRed: 20 / 255, green: 160 / 255, blue: 60 / 255, alpha: 1) + w.orderFrontRegardless() + let end = Date().addingTimeInterval(0.5) + while Date() < end { RunLoop.current.run(mode: .default, before: end) } + try! "\(w.windowNumber)".write(toFile: args[2], atomically: true, encoding: .ascii) + while true {} // never pumps again: a hung app +default: + exit(2) +} From 832ec8279948a5ddd4cf647db497674caefa76b0 Mon Sep 17 00:00:00 2001 From: Elie Gambache <elyahou.hadass@gmail.com> Date: Sat, 26 Sep 2026 22:53:23 +0300 Subject: [PATCH 3/4] feat(screen-capture): list capturable windows captureWindow took an id the app had no way to find beyond its own windows. ScreenCapture.windows() lists the top-level windows it can capture, front to back: visible, not minimized, not on another virtual desktop, with title, app name, pid and bounds in the same space as CaptureDisplay.bounds. - Windows: EnumWindows with the Alt+Tab rules (no tool, cloaked or untitled windows, not the shell's surfaces); InternalGetWindowText never sends WM_GETTEXT, so a hung window cannot block the listing. - macOS: CGWindowListCopyWindowInfo, on-screen windows below the Dock level. The demo now finds its own CGWindowID there, so the E2E script no longer hands it over. - X11: _NET_CLIENT_LIST_STACKING, or the root's children without a window manager; Java is called with the X11 lock released. - Titles cross JNI as UTF-8 bytes: NewStringUTF takes modified UTF-8, which an emoji in a tab title is not. CaptureFailure and CaptureBackend document that later versions may add values. The self-test checks the listing (own window, size, minimized, occluded, hung) and calls it throughout the torture; the Linux harness checks it on both the XQueryTree and no-display paths. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> --- CLAUDE.md | 2 +- README.md | 2 +- .../src/main/kotlin/screencapturedemo/Main.kt | 58 ++++-- .../main/kotlin/screencapturedemo/SelfTest.kt | 29 +++ screen-capture/api/screen-capture.api | 12 ++ .../screencapture/CaptureWindow.kt | 37 ++++ .../screencapture/ScreenCapture.kt | 112 ++++++----- .../screencapture/ScreenCaptureException.kt | 6 +- .../internal/NativeScreenCapture.kt | 38 ++++ .../linux/nucleus_screencapture_linux.c | 179 ++++++++++++++++++ .../main/native/macos/NucleusScreenCapture.m | 68 +++++++ .../windows/nucleus_screencapture_windows.c | 126 ++++++++++++ .../reachability-metadata.json | 19 ++ .../screencapture/ScreenCaptureLiveTest.kt | 15 ++ .../screencapture/internal/Harness.java | 7 + .../internal/NativeScreenCapture.java | 1 + .../internal/WindowCollector.java | 18 ++ scripts/screen-capture-macos-e2e.sh | 10 +- scripts/screen-capture-macos-e2e/helper.swift | 8 - 19 files changed, 666 insertions(+), 81 deletions(-) create mode 100644 screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/CaptureWindow.kt create mode 100644 scripts/screen-capture-linux-e2e/harness/dev/nucleusframework/screencapture/internal/WindowCollector.java diff --git a/CLAUDE.md b/CLAUDE.md index 02cce898f..3b59b084d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -27,7 +27,7 @@ Published releases are `2.5.x` (latest tag `v2.5.0`). Do not treat `IDEAL_API.md - `darkmode-detector` - Reactive OS dark mode detection via JNI - `system-color` - Reactive system accent color and high contrast detection via JNI - `system-info` - CPU, memory, GPU, temperature, network, processes -- `screen-capture` - AWT-free screen capture in physical pixels: `ScreenCapture.displays()` / `captureDisplay(display, region, includeCursor)` / `captureWindow(id)` / permission. Windows: GDI `BitBlt` in a per-monitor-v2 thread DPI context; windows through `PrintWindow(PW_RENDERFULLCONTENT)` on a **worker thread with a 2 s timeout** (it sends `WM_PRINT` and waits forever on a thread that does not pump — including an app's own UI thread blocked on the capturing thread), falling back to the visible part from the screen; the window's own thread prints inline. macOS: ScreenCaptureKit (14+) / `CGDisplayCreateImageForRect` via `dlsym` (obsoleted in the 15 SDK), never pumps the main run loop. Linux: X11 (`XGetImage`, RandR monitors, XFixes cursor, XComposite for covered windows; everything `dlopen`ed, X errors trapped per call) or, on Wayland, `org.freedesktop.portal.Screenshot` (whole desktop, cropped; the portal file is deleted). E2E: `scripts/screen-capture-windows-e2e.ps1` (`examples/screen-capture-demo` self-test: pixel-exact pattern, occluded / minimized / blocked-UI / hung-foreign windows, torture + GDI/handle leak sampling) and `scripts/screen-capture-linux-e2e.sh` (Xvfb depths 24/16/8 vs `xwd`, fake portal) +- `screen-capture` - AWT-free screen capture in physical pixels: `ScreenCapture.displays()` / `captureDisplay(display, region, includeCursor)` / `captureWindow(id)` / `windows()` (capturable top-level windows, front to back; titles cross JNI as UTF-8 `byte[]`, since `NewStringUTF` rejects emoji) / permission. Windows: GDI `BitBlt` in a per-monitor-v2 thread DPI context; windows through `PrintWindow(PW_RENDERFULLCONTENT)` on a **worker thread with a 2 s timeout** (it sends `WM_PRINT` and waits forever on a thread that does not pump — including an app's own UI thread blocked on the capturing thread), falling back to the visible part from the screen; the window's own thread prints inline. macOS: ScreenCaptureKit (14+) / `CGDisplayCreateImageForRect` via `dlsym` (obsoleted in the 15 SDK), never pumps the main run loop; a region is captured from a **32 pt grid-aligned** rect and cropped (an odd-pixel origin is resampled, and dithered content follows a tile anchored at the requested rect — unaligned it comes back ±2 levels off the full capture); window ids are `CGWindowID`s, found through `windows()` since Tao does not expose them. Linux: X11 (`XGetImage`, RandR monitors, XFixes cursor, XComposite for covered windows; everything `dlopen`ed, X errors trapped per call) or, on Wayland, `org.freedesktop.portal.Screenshot` (whole desktop, cropped; the portal file is deleted). E2E: `scripts/screen-capture-windows-e2e.ps1` (`examples/screen-capture-demo` self-test: pixel-exact pattern, occluded / minimized / blocked-UI / hung-foreign windows, torture + GDI/handle leak sampling) `scripts/screen-capture-macos-e2e.sh` (same self-test; hung window, cursor parking and fd/thread/Mach port/RSS sampling done by a Swift helper; run under `caffeinate` on an untouched Mac — display sleep, mouse moves, Mission Control or a playing video all fail it) and `scripts/screen-capture-linux-e2e.sh` (Xvfb depths 24/16/8 vs `xwd`, fake portal) - `energy-manager` - Energy efficiency & screen-awake APIs - `autolaunch` - Start at login (Win32/MSIX/SMAppService/systemd/Flatpak portal) - `scheduler` / `scheduler-testing` - OS-scheduled background tasks (Task Scheduler / launchd / systemd) + test doubles diff --git a/README.md b/README.md index 4e45696a8..f7618f63a 100644 --- a/README.md +++ b/README.md @@ -220,7 +220,7 @@ Each module is published independently to Maven Central — use them together or | `nucleus.darkmode-detector` | Reactive OS dark mode detection | | `nucleus.system-color` | Reactive accent color & high contrast detection | | `nucleus.system-info` | CPU, memory, GPU (NVIDIA/AMD/Intel), temperature, network, processes | -| `nucleus.screen-capture` | Native screen & window capture in physical pixels, no AWT (GDI / ScreenCaptureKit / X11 / xdg-desktop-portal) | +| `nucleus.screen-capture` | Native screen & window capture in physical pixels, window listing, no AWT (GDI / ScreenCaptureKit / X11 / xdg-desktop-portal) | | `nucleus.decorated-window-tao` | Windowing backend (Rust `tao`, no AWT) | | `nucleus.decorated-window-core` | Shared window types, layout, chrome (design-system agnostic) | | `nucleus.decorated-window-jewel` | Jewel (IntelliJ theme) integration | diff --git a/examples/screen-capture-demo/src/main/kotlin/screencapturedemo/Main.kt b/examples/screen-capture-demo/src/main/kotlin/screencapturedemo/Main.kt index 8fea65e57..0d7dceea2 100644 --- a/examples/screen-capture-demo/src/main/kotlin/screencapturedemo/Main.kt +++ b/examples/screen-capture-demo/src/main/kotlin/screencapturedemo/Main.kt @@ -34,6 +34,7 @@ import dev.nucleusframework.application.DecoratedWindow import dev.nucleusframework.application.nucleusApplication import dev.nucleusframework.core.runtime.Platform import dev.nucleusframework.screencapture.CaptureRegion +import dev.nucleusframework.screencapture.CaptureWindow import dev.nucleusframework.screencapture.ScreenCapture import dev.nucleusframework.screencapture.ScreenCaptureException import dev.nucleusframework.screencapture.ScreenImage @@ -58,19 +59,19 @@ fun main(args: Array<String>) = NucleusDecoratedWindowTheme(isDark = true) { DecoratedWindow( onCloseRequest = ::exitApplication, - title = "Screen Capture Demo", + title = WINDOW_TITLE, // Self-test: nothing else may cover the pattern (the cover window, shown later, goes above). alwaysOnTop = selfTest, state = rememberWindowState(size = DpSize(760.dp, 620.dp), position = WindowPosition(120.dp, 120.dp)), ) { - TitleBar { BasicText("Screen Capture Demo", style = TextStyle(color = Color.White)) } + TitleBar { BasicText(WINDOW_TITLE, style = TextStyle(color = Color.White)) } val window = LocalTaoWindow.current if (window != null && selfTest) { LaunchedEffect(window) { val test = SelfTest( window = window, - windowId = windowIdOf(window) ?: awaitMacWindowId(), + windowId = withContext(Dispatchers.IO) { awaitWindowId(window) }, showCover = { visible -> withContext(Dispatchers.Main) { coverVisible = visible } }, setMinimized = { minimized -> withContext(Dispatchers.Main) { window.setMinimized(minimized) } @@ -97,24 +98,31 @@ fun main(args: Array<String>) = } } -/** The id [ScreenCapture.captureWindow] takes for a Tao window; `0` where it has none, `null` on macOS. */ -private fun windowIdOf(window: TaoWindow): Long? = +private const val WINDOW_TITLE = "Screen Capture Demo" + +/** + * The id [ScreenCapture.captureWindow] takes for a Tao window; `0` where it has none. Blocking. + * + * macOS: Tao does not expose the NSWindow's CGWindowID yet, so the window is found in + * [ScreenCapture.windows] by pid and title. + */ +private fun windowIdOf(window: TaoWindow): Long = when (Platform.Current) { Platform.Windows -> window.nativeHandle Platform.Linux -> window.x11WindowId ?: 0L - Platform.MacOS -> null + Platform.MacOS -> + ScreenCapture + .windows() + .firstOrNull { it.pid == ProcessHandle.current().pid() && it.title == WINDOW_TITLE } + ?.id ?: 0L else -> 0L } -/** - * macOS: Tao does not expose the NSWindow's CGWindowID yet; the E2E script looks it up by pid and - * title and writes it to SCREEN_CAPTURE_DEMO_WINDOW_ID_FILE. - */ -private suspend fun awaitMacWindowId(): Long { - val file = System.getenv("SCREEN_CAPTURE_DEMO_WINDOW_ID_FILE")?.let(::File) ?: return 0L - repeat(300) { - val id = runCatching { file.readText().trim().toLongOrNull() }.getOrNull() - if (id != null) return id +/** [windowIdOf], waiting for the window to reach the screen. */ +private suspend fun awaitWindowId(window: TaoWindow): Long { + repeat(50) { + val id = runCatching { windowIdOf(window) }.getOrDefault(0L) + if (id != 0L) return id kotlinx.coroutines.delay(100) } return 0L @@ -125,6 +133,7 @@ private fun DemoContent(window: TaoWindow?) { val scope = rememberCoroutineScope() var preview by remember { mutableStateOf<ImageBitmap?>(null) } var last by remember { mutableStateOf<ScreenImage?>(null) } + var windows by remember { mutableStateOf<List<CaptureWindow>>(emptyList()) } var status by remember { mutableStateOf("backend=${ScreenCapture.backend} permission=${ScreenCapture.permissionStatus()}") } @@ -167,9 +176,22 @@ private fun DemoContent(window: TaoWindow?) { capture("region") { ScreenCapture.captureDisplay(primary, CaptureRegion(0, 0, 400, 300)) } } } - val id = window?.let(::windowIdOf) ?: 0L - if (id != 0L && ScreenCapture.isWindowCaptureSupported) { - DemoButton("This window") { capture("window") { ScreenCapture.captureWindow(id) } } + if (window != null && ScreenCapture.isWindowCaptureSupported) { + DemoButton("This window") { capture("window") { ScreenCapture.captureWindow(windowIdOf(window)) } } + DemoButton("List windows") { + scope.launch { + windows = + withContext( + Dispatchers.IO, + ) { runCatching { ScreenCapture.windows() } }.getOrDefault(emptyList()) + } + } + } + for (target in windows.take(12)) { + val label = listOf(target.appName, target.title).filter { it.isNotEmpty() }.joinToString(" — ") + DemoButton(label.take(40).ifEmpty { "#${target.id}" }) { + capture(label) { ScreenCapture.captureWindow(target.id) } + } } last?.let { image -> DemoButton("Save PNG") { diff --git a/examples/screen-capture-demo/src/main/kotlin/screencapturedemo/SelfTest.kt b/examples/screen-capture-demo/src/main/kotlin/screencapturedemo/SelfTest.kt index c1b5401ff..a4b7509b5 100644 --- a/examples/screen-capture-demo/src/main/kotlin/screencapturedemo/SelfTest.kt +++ b/examples/screen-capture-demo/src/main/kotlin/screencapturedemo/SelfTest.kt @@ -4,6 +4,7 @@ import dev.nucleusframework.core.runtime.Platform import dev.nucleusframework.screencapture.CaptureDisplay import dev.nucleusframework.screencapture.CaptureFailure import dev.nucleusframework.screencapture.CaptureRegion +import dev.nucleusframework.screencapture.CaptureWindow import dev.nucleusframework.screencapture.ScreenCapture import dev.nucleusframework.screencapture.ScreenCaptureException import dev.nucleusframework.screencapture.ScreenImage @@ -149,6 +150,26 @@ internal class SelfTest( } expectPattern("window", image) saveSample("window", image) + checkWindowList(image) + } + + /** This window is listed, with its id, pid and a size matching its capture. */ + private fun checkWindowList(capture: ScreenImage) { + val windows: List<CaptureWindow> + val ms = measureTimeMillis { windows = ScreenCapture.windows() } + log("INFO windows() listed ${windows.size} in ${ms}ms") + val own = windows.firstOrNull { it.id == windowId } + check("window list has this window", own != null) { "id=$windowId in ${windows.take(5)}" } + if (own == null) return + log("INFO window list entry $own") + check("window list title", own.title == "Screen Capture Demo") { own.title } + check("window list pid", own.pid == 0L || own.pid == ProcessHandle.current().pid()) { "${own.pid}" } + val scale = if (Platform.Current == Platform.MacOS) capture.scaleFactor else 1f + check( + "window list size matches the capture", + kotlin.math.abs(own.bounds.width * scale - capture.width) <= scale && + kotlin.math.abs(own.bounds.height * scale - capture.height) <= scale, + ) { "${own.bounds} x$scale vs ${capture.width}x${capture.height}" } } private fun displayUnderWindow(displays: List<CaptureDisplay>): CaptureDisplay = @@ -234,6 +255,7 @@ internal class SelfTest( "cover hides the pattern on screen", TargetPattern.find(screen).isEmpty(), ) { "${TargetPattern.find(screen)}" } + check("occluded window is listed", ScreenCapture.windows().any { it.id == windowId }) val image = ScreenCapture.captureWindow(windowId) if (Platform.Current == Platform.Windows || Platform.Current == Platform.MacOS) { expectPattern("occluded window", image) @@ -251,6 +273,7 @@ internal class SelfTest( setMinimized(true) delay(1000) try { + check("minimized window is not listed", ScreenCapture.windows().none { it.id == windowId }) val error = runCatching { ScreenCapture.captureWindow(windowId) }.exceptionOrNull() check( "minimized window is not capturable", @@ -307,6 +330,8 @@ internal class SelfTest( val hung = System.getenv("SCREEN_CAPTURE_DEMO_HUNG_HWND")?.toLongOrNull() ?: return log("SKIP hung window: not provided") + val listMs = measureTimeMillis { check("hung window is listed", ScreenCapture.windows().any { it.id == hung }) } + check("window list returns in time with a hung window", listMs < 1000) { "${listMs}ms" } repeat(3) { attempt -> var outcome = "" val ms = @@ -423,6 +448,10 @@ internal class SelfTest( repeat(tortureIterations / 2) { runCatching { ScreenCapture.captureWindow(random.nextLong() or 0x7000_0000_0000_0000L) } if (ScreenCapture.displays().size != displays.size) errors += "display count changed" + runCatching { ScreenCapture.windows() }.onFailure { + errors += + "windows() ${it::class.simpleName} ${it.message}" + } } } pool.shutdown() diff --git a/screen-capture/api/screen-capture.api b/screen-capture/api/screen-capture.api index 5a7feef50..fa1e8743f 100644 --- a/screen-capture/api/screen-capture.api +++ b/screen-capture/api/screen-capture.api @@ -66,6 +66,17 @@ public final class dev/nucleusframework/screencapture/CaptureRegion { public fun toString ()Ljava/lang/String; } +public final class dev/nucleusframework/screencapture/CaptureWindow { + public fun equals (Ljava/lang/Object;)Z + public final fun getAppName ()Ljava/lang/String; + public final fun getBounds ()Ldev/nucleusframework/screencapture/CaptureRegion; + public final fun getId ()J + public final fun getPid ()J + public final fun getTitle ()Ljava/lang/String; + public fun hashCode ()I + public fun toString ()Ljava/lang/String; +} + public final class dev/nucleusframework/screencapture/ScreenCapture { public static final field INSTANCE Ldev/nucleusframework/screencapture/ScreenCapture; public final fun captureDisplay (Ldev/nucleusframework/screencapture/CaptureDisplay;Ldev/nucleusframework/screencapture/CaptureRegion;Z)Ldev/nucleusframework/screencapture/ScreenImage; @@ -79,6 +90,7 @@ public final class dev/nucleusframework/screencapture/ScreenCapture { public final fun permissionStatus ()Ldev/nucleusframework/screencapture/CapturePermission; public final fun primaryDisplay ()Ldev/nucleusframework/screencapture/CaptureDisplay; public final fun requestPermission ()Ldev/nucleusframework/screencapture/CapturePermission; + public final fun windows ()Ljava/util/List; } public final class dev/nucleusframework/screencapture/ScreenCaptureException : java/lang/RuntimeException { diff --git a/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/CaptureWindow.kt b/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/CaptureWindow.kt new file mode 100644 index 000000000..310f9a33d --- /dev/null +++ b/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/CaptureWindow.kt @@ -0,0 +1,37 @@ +package dev.nucleusframework.screencapture + +/** + * A top-level window [ScreenCapture.captureWindow] can capture, as listed by + * [ScreenCapture.windows]. + * + * @property id what [ScreenCapture.captureWindow] takes: the `HWND` on Windows, the + * `CGWindowID` on macOS, the client window's XID on X11. + * @property title the window's title; empty when it has none. On macOS titles are only + * reported with the Screen Recording permission. + * @property appName the owning application's name as the platform reports it: the process + * name on Windows (`chrome`), the app name on macOS (`Google Chrome`), the `WM_CLASS` + * class on X11 (`Google-chrome`); empty when unknown. + * @property pid the owning process, `0` when unknown (an X11 client without `_NET_WM_PID`). + * @property bounds the window's rectangle in the same desktop space as [CaptureDisplay.bounds]: + * physical pixels on Windows (the visible frame, without the invisible resize borders) and + * X11 (the client area), points on macOS (title bar included). + */ +public class CaptureWindow internal constructor( + public val id: Long, + public val title: String, + public val appName: String, + public val pid: Long, + public val bounds: CaptureRegion, +) { + override fun equals(other: Any?): Boolean = + other is CaptureWindow && + id == other.id && + title == other.title && + appName == other.appName && + pid == other.pid && + bounds == other.bounds + + override fun hashCode(): Int = 31 * id.hashCode() + bounds.hashCode() + + override fun toString(): String = "CaptureWindow(id=$id, title=$title, app=$appName, pid=$pid, bounds=$bounds)" +} diff --git a/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/ScreenCapture.kt b/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/ScreenCapture.kt index b2c3aa025..0c58e00e6 100644 --- a/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/ScreenCapture.kt +++ b/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/ScreenCapture.kt @@ -5,12 +5,17 @@ import dev.nucleusframework.screencapture.internal.DisplayCollector import dev.nucleusframework.screencapture.internal.NativeCall import dev.nucleusframework.screencapture.internal.NativeScreenCapture import dev.nucleusframework.screencapture.internal.RawDisplay +import dev.nucleusframework.screencapture.internal.WindowCollector import dev.nucleusframework.screencapture.internal.clip import dev.nucleusframework.screencapture.internal.permissionOf import java.util.logging.Level import java.util.logging.Logger -/** The mechanism [ScreenCapture] captures with on this machine. */ +/** + * The mechanism [ScreenCapture] captures with on this machine. + * + * Later versions may add values: keep an `else` branch in a `when` over it. + */ public enum class CaptureBackend { /** Windows GDI, rendered in a per-monitor DPI context (physical pixels). */ Gdi, @@ -68,13 +73,12 @@ public object ScreenCapture { private val logger = Logger.getLogger(ScreenCapture::class.java.name) private const val PORTAL_DISPLAY_ID = "portal" private const val PORTAL_TIMEOUT_MS = 60_000 - private const val LINUX_BACKEND_PROPERTY = "nucleus.screencapture.linuxBackend" @Volatile private var portalImageSize: Pair<Int, Int>? = null /** The backend captures go through; [CaptureBackend.Unavailable] when there is none. */ - public val backend: CaptureBackend by lazy { resolveBackend() } + public val backend: CaptureBackend by lazy { resolveBackend(logger) } /** `true` when [backend] is not [CaptureBackend.Unavailable]. */ public val isSupported: Boolean get() = backend != CaptureBackend.Unavailable @@ -104,6 +108,22 @@ public object ScreenCapture { .sortedByDescending { it.isPrimary } } + /** + * The top-level windows [captureWindow] can capture, front to back: visible, not + * minimized, not on another virtual desktop. The app's own windows are included. + * + * Empty where window capture is not available ([isWindowCaptureSupported]). + * + * @throws ScreenCaptureException when the backend cannot enumerate them. + */ + public fun windows(): List<CaptureWindow> { + if (!isWindowCaptureSupported) return emptyList() + val collector = WindowCollector() + val call = NativeCall("list windows") + call.check(NativeScreenCapture.nativeListWindows(collector, call.message)) + return collector.windows + } + /** The primary display, or `null` when none is reported. */ public fun primaryDisplay(): CaptureDisplay? = displays().firstOrNull() @@ -183,48 +203,6 @@ public object ScreenCapture { else -> permissionStatus() } - private fun resolveBackend(): CaptureBackend { - if (Platform.Current == Platform.Unknown) return CaptureBackend.Unavailable - val loaded = - try { - NativeScreenCapture.isLoaded - } catch (e: LinkageError) { - logger.log(Level.WARNING, "Screen capture native library failed to load", e) - false - } - if (!loaded) { - logger.warning("Screen capture native library is not available on ${Platform.Current}") - return CaptureBackend.Unavailable - } - if (Platform.Current == Platform.Linux && useLinuxPortal()) return CaptureBackend.XdgDesktopPortal - return when (NativeScreenCapture.nativeBackend()) { - NativeScreenCapture.BACKEND_GDI -> CaptureBackend.Gdi - NativeScreenCapture.BACKEND_SCREEN_CAPTURE_KIT -> CaptureBackend.ScreenCaptureKit - NativeScreenCapture.BACKEND_CORE_GRAPHICS -> CaptureBackend.CoreGraphics - NativeScreenCapture.BACKEND_X11 -> CaptureBackend.X11 - // Linux without an X server may still have a portal. - else -> - if (Platform.Current == - Platform.Linux - ) { - CaptureBackend.XdgDesktopPortal - } else { - CaptureBackend.Unavailable - } - } - } - - /** - * On a Wayland session the X server is XWayland, whose root window does not hold the - * native Wayland windows — only the portal sees the real desktop. - */ - private fun useLinuxPortal(): Boolean = - when (System.getProperty(LINUX_BACKEND_PROPERTY)?.lowercase()) { - "x11" -> false - "portal" -> true - else -> Platform.isWayland - } - private fun portalDisplay(): CaptureDisplay { val size = portalImageSize return CaptureDisplay( @@ -253,6 +231,50 @@ public object ScreenCapture { } } +private const val LINUX_BACKEND_PROPERTY = "nucleus.screencapture.linuxBackend" + +private fun resolveBackend(logger: Logger): CaptureBackend { + if (Platform.Current == Platform.Unknown) return CaptureBackend.Unavailable + val loaded = + try { + NativeScreenCapture.isLoaded + } catch (e: LinkageError) { + logger.log(Level.WARNING, "Screen capture native library failed to load", e) + false + } + if (!loaded) { + logger.warning("Screen capture native library is not available on ${Platform.Current}") + return CaptureBackend.Unavailable + } + if (Platform.Current == Platform.Linux && useLinuxPortal()) return CaptureBackend.XdgDesktopPortal + return when (NativeScreenCapture.nativeBackend()) { + NativeScreenCapture.BACKEND_GDI -> CaptureBackend.Gdi + NativeScreenCapture.BACKEND_SCREEN_CAPTURE_KIT -> CaptureBackend.ScreenCaptureKit + NativeScreenCapture.BACKEND_CORE_GRAPHICS -> CaptureBackend.CoreGraphics + NativeScreenCapture.BACKEND_X11 -> CaptureBackend.X11 + // Linux without an X server may still have a portal. + else -> + if (Platform.Current == + Platform.Linux + ) { + CaptureBackend.XdgDesktopPortal + } else { + CaptureBackend.Unavailable + } + } +} + +/** + * On a Wayland session the X server is XWayland, whose root window does not hold the + * native Wayland windows — only the portal sees the real desktop. + */ +private fun useLinuxPortal(): Boolean = + when (System.getProperty(LINUX_BACKEND_PROPERTY)?.lowercase()) { + "x11" -> false + "portal" -> true + else -> Platform.isWayland + } + private fun RawDisplay.toDisplay(): CaptureDisplay = CaptureDisplay( id = id, diff --git a/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/ScreenCaptureException.kt b/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/ScreenCaptureException.kt index c4840f8c5..775edaae1 100644 --- a/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/ScreenCaptureException.kt +++ b/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/ScreenCaptureException.kt @@ -1,6 +1,10 @@ package dev.nucleusframework.screencapture -/** Why a capture failed. */ +/** + * Why a capture failed. + * + * Later versions may add values: keep an `else` branch in a `when` over it. + */ public enum class CaptureFailure { /** The platform or session has no capture backend (e.g. Wayland without a portal). */ Unsupported, diff --git a/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/internal/NativeScreenCapture.kt b/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/internal/NativeScreenCapture.kt index 52366fbc5..bf9b84a04 100644 --- a/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/internal/NativeScreenCapture.kt +++ b/screen-capture/src/main/kotlin/dev/nucleusframework/screencapture/internal/NativeScreenCapture.kt @@ -1,6 +1,8 @@ package dev.nucleusframework.screencapture.internal import dev.nucleusframework.core.runtime.NativeLibraryLoader +import dev.nucleusframework.screencapture.CaptureRegion +import dev.nucleusframework.screencapture.CaptureWindow /** * JNI bridge, one library per platform with the same symbols. @@ -45,6 +47,13 @@ internal object NativeScreenCapture { message: Array<String?>, ): Int + /** Reports every capturable top-level window to [sink], front to back; returns a status code. */ + @JvmStatic + external fun nativeListWindows( + sink: WindowCollector, + message: Array<String?>, + ): Int + /** * Captures [displayId]. A region with `width <= 0` means the whole display; otherwise it is * in display pixels and the native side clips it to the display. @@ -107,6 +116,35 @@ internal class DisplayCollector { } } +/** Receives [NativeScreenCapture.nativeListWindows] records; called from native code. */ +internal class WindowCollector { + val windows = mutableListOf<CaptureWindow>() + + // Titles arrive as UTF-8 bytes: JNI's NewStringUTF takes modified UTF-8, which an emoji + // in a title is not, and X11 titles are not always valid UTF-8 at all. + @Suppress("LongParameterList") + fun add( + id: Long, + title: ByteArray, + appName: ByteArray, + pid: Long, + x: Int, + y: Int, + width: Int, + height: Int, + ) { + if (width <= 0 || height <= 0) return + windows += + CaptureWindow( + id = id, + title = title.toString(Charsets.UTF_8), + appName = appName.toString(Charsets.UTF_8), + pid = pid, + bounds = CaptureRegion(x, y, width, height), + ) + } +} + internal data class RawDisplay( val id: String, val name: String, diff --git a/screen-capture/src/main/native/linux/nucleus_screencapture_linux.c b/screen-capture/src/main/native/linux/nucleus_screencapture_linux.c index 63297f543..08f42a827 100644 --- a/screen-capture/src/main/native/linux/nucleus_screencapture_linux.c +++ b/screen-capture/src/main/native/linux/nucleus_screencapture_linux.c @@ -28,6 +28,7 @@ #include <X11/Xlib.h> #include <X11/Xutil.h> +#include <X11/Xatom.h> #include <X11/extensions/Xrandr.h> #include <X11/extensions/Xfixes.h> #include <X11/extensions/Xcomposite.h> @@ -114,6 +115,11 @@ static struct { int (*Free)(void *); int (*FreePixmap)(Display *, Pixmap); Bool (*QueryExtension)(Display *, const char *, int *, int *, int *); + /* Window list only: optional, X.loaded does not depend on them. */ + Atom (*InternAtom)(Display *, const char *, Bool); + int (*GetWindowProperty)(Display *, Window, Atom, long, long, Bool, Atom, Atom *, int *, unsigned long *, + unsigned long *, unsigned char **); + Status (*QueryTree)(Display *, Window, Window *, Window *, Window **, unsigned int *); } X; static struct { @@ -165,6 +171,9 @@ static void load_x11(void) { LOAD(lib, X, Free, "XFree"); LOAD(lib, X, FreePixmap, "XFreePixmap"); LOAD(lib, X, QueryExtension, "XQueryExtension"); + LOAD(lib, X, InternAtom, "XInternAtom"); + LOAD(lib, X, GetWindowProperty, "XGetWindowProperty"); + LOAD(lib, X, QueryTree, "XQueryTree"); X.loaded = X.OpenDisplay && X.CloseDisplay && X.SetErrorHandler && X.Sync && X.GetImage && X.GetWindowAttributes && X.TranslateCoordinates && X.QueryColors && X.GetAtomName && X.Free && X.FreePixmap; @@ -686,6 +695,176 @@ EXPORT JNIEXPORT jint JNICALL Java_dev_nucleusframework_screencapture_internal_N return status; } +/* ------------------------------------------------------------------------ */ +/* Window list */ +/* ------------------------------------------------------------------------ */ + +#define MAX_LISTED_WINDOWS 4096 +#define MAX_TEXT 1024 + +typedef struct { + Window id; + char title[MAX_TEXT]; + char app[MAX_TEXT]; + long pid; + int x, y, w, h; +} ListedWindow; + +/* The property's value (XFree it), or NULL. [type] AnyPropertyType accepts any type. */ +static unsigned char *window_property(Display *dpy, Window w, Atom property, Atom type, unsigned long *count) { + Atom actual = None; + int format = 0; + unsigned long after = 0; + unsigned char *data = NULL; + *count = 0; + if (property == None) return NULL; + if (X.GetWindowProperty(dpy, w, property, 0, MAX_TEXT, False, type, &actual, &format, count, &after, &data) != + Success || + x_take_error(dpy) != 0 || data == NULL) { + return NULL; + } + if (actual == None || *count == 0) { + X.Free(data); + return NULL; + } + return data; +} + +static void copy_text(char *out, const unsigned char *data, unsigned long length) { + if (length >= MAX_TEXT) length = MAX_TEXT - 1; + memcpy(out, data, length); + out[length] = 0; +} + +typedef struct { + Atom net_wm_name, utf8, wm_name, wm_class, net_wm_pid; +} Atoms; + +/* Reads the window's attributes; returns 0 when it is not a viewable, capturable window. */ +static int describe_window(Display *dpy, Window root, Window w, int from_tree, const Atoms *atoms, ListedWindow *out) { + XWindowAttributes attrs; + memset(&attrs, 0, sizeof(attrs)); + if (!X.GetWindowAttributes(dpy, w, &attrs) || x_take_error(dpy) != 0) return 0; + if (attrs.map_state != IsViewable || attrs.class != InputOutput || attrs.width <= 0 || attrs.height <= 0) return 0; + /* Without a window manager the root's children include menus and tooltips. */ + if (from_tree && attrs.override_redirect) return 0; + int x = 0, y = 0; + Window child; + if (!X.TranslateCoordinates(dpy, w, root, 0, 0, &x, &y, &child) || x_take_error(dpy) != 0) return 0; + + memset(out, 0, sizeof(*out)); + out->id = w; + out->x = x; + out->y = y; + out->w = attrs.width; + out->h = attrs.height; + unsigned long count = 0; + unsigned char *data = window_property(dpy, w, atoms->net_wm_name, atoms->utf8, &count); + if (data == NULL) data = window_property(dpy, w, atoms->wm_name, AnyPropertyType, &count); + if (data != NULL) { + copy_text(out->title, data, count); + X.Free(data); + } + /* WM_CLASS is "instance\0class\0": the class is the app's name. */ + data = window_property(dpy, w, atoms->wm_class, XA_STRING, &count); + if (data != NULL) { + size_t instance = strnlen((const char *)data, count); + if (instance + 1 < count) copy_text(out->app, data + instance + 1, strnlen((const char *)data + instance + 1, count - instance - 1)); + X.Free(data); + } + data = window_property(dpy, w, atoms->net_wm_pid, XA_CARDINAL, &count); + if (data != NULL) { + out->pid = *(long *)data; /* format 32 properties are longs */ + X.Free(data); + } + return 1; +} + +static jbyteArray text_bytes(JNIEnv *env, const char *text) { + jsize length = (jsize)strlen(text); + jbyteArray bytes = (*env)->NewByteArray(env, length); + if (bytes != NULL && length > 0) (*env)->SetByteArrayRegion(env, bytes, 0, length, (const jbyte *)text); + return bytes; +} + +EXPORT JNIEXPORT jint JNICALL Java_dev_nucleusframework_screencapture_internal_NativeScreenCapture_nativeListWindows( + JNIEnv *env, jclass cls, jobject sink, jobjectArray message) { + (void)cls; + pthread_once(&g_x11_once, load_x11); + if (!X.loaded || X.InternAtom == NULL || X.GetWindowProperty == NULL || X.QueryTree == NULL) { + set_message(env, message, "libX11 is not available"); + return ST_UNSUPPORTED; + } + jclass sink_class = (*env)->GetObjectClass(env, sink); + jmethodID add = (*env)->GetMethodID(env, sink_class, "add", "(J[B[BJIIII)V"); + (*env)->DeleteLocalRef(env, sink_class); + if (add == NULL) { + nucleus_jni_clear_exception(env); + set_message(env, message, "WindowCollector.add not found"); + return ST_FAILED; + } + ListedWindow *listed = (ListedWindow *)calloc(MAX_LISTED_WINDOWS, sizeof(ListedWindow)); + if (listed == NULL) return ST_FAILED; + + Display *dpy = x_session_open(); + if (dpy == NULL) { + free(listed); + set_message(env, message, "Cannot open X display '%s'", getenv("DISPLAY") != NULL ? getenv("DISPLAY") : ""); + return ST_UNSUPPORTED; + } + Window root = DefaultRootWindow(dpy); + Atoms atoms = { + X.InternAtom(dpy, "_NET_WM_NAME", False), X.InternAtom(dpy, "UTF8_STRING", False), + XA_WM_NAME, XA_WM_CLASS, X.InternAtom(dpy, "_NET_WM_PID", False), + }; + /* EWMH client windows, bottom to top; without a window manager, the root's children. */ + unsigned long count = 0; + int from_tree = 0; + unsigned char *clients = window_property(dpy, root, X.InternAtom(dpy, "_NET_CLIENT_LIST_STACKING", False), + XA_WINDOW, &count); + Window *windows = (Window *)clients; + Window *children = NULL; + if (windows == NULL) { + Window root_return, parent; + unsigned int n = 0; + if (X.QueryTree(dpy, root, &root_return, &parent, &children, &n) && x_take_error(dpy) == 0) { + windows = children; + count = n; + from_tree = 1; + } + } + int listed_count = 0; + for (unsigned long i = count; i-- > 0 && listed_count < MAX_LISTED_WINDOWS;) { + if (describe_window(dpy, root, windows[i], from_tree, &atoms, &listed[listed_count])) listed_count++; + } + if (clients != NULL) X.Free(clients); + if (children != NULL) X.Free(children); + x_session_close(dpy); + + /* Java is called with the X11 lock released. */ + int status = ST_OK; + for (int i = 0; i < listed_count; i++) { + ListedWindow *w = &listed[i]; + jbyteArray title = text_bytes(env, w->title); + jbyteArray app = title != NULL ? text_bytes(env, w->app) : NULL; + if (title == NULL || app == NULL) { + if (title != NULL) (*env)->DeleteLocalRef(env, title); + status = ST_FAILED; /* OutOfMemoryError pending: let it propagate */ + break; + } + (*env)->CallVoidMethod(env, sink, add, (jlong)w->id, title, app, (jlong)w->pid, w->x, w->y, w->w, w->h); + (*env)->DeleteLocalRef(env, title); + (*env)->DeleteLocalRef(env, app); + if (nucleus_jni_clear_exception(env)) { + set_message(env, message, "WindowCollector.add threw"); + status = ST_FAILED; + break; + } + } + free(listed); + return status; +} + EXPORT JNIEXPORT jintArray JNICALL Java_dev_nucleusframework_screencapture_internal_NativeScreenCapture_nativeCaptureDisplay( JNIEnv *env, jclass cls, jstring display_id, jint rx, jint ry, jint rw, jint rh, jboolean include_cursor, diff --git a/screen-capture/src/main/native/macos/NucleusScreenCapture.m b/screen-capture/src/main/native/macos/NucleusScreenCapture.m index 3502c93ef..abe1e1db8 100644 --- a/screen-capture/src/main/native/macos/NucleusScreenCapture.m +++ b/screen-capture/src/main/native/macos/NucleusScreenCapture.m @@ -619,6 +619,74 @@ static jintArray captureWindowWithCoreGraphics( } } +// UTF-8 bytes for WindowCollector.add: NewStringUTF takes modified UTF-8, which an emoji is not. +static jbyteArray utf8Bytes(JNIEnv *env, NSString *text) { + NSData *data = [text ?: @"" dataUsingEncoding:NSUTF8StringEncoding] ?: [NSData data]; + jbyteArray bytes = (*env)->NewByteArray(env, (jsize)data.length); + if (bytes != NULL && data.length > 0) { + (*env)->SetByteArrayRegion(env, bytes, 0, (jsize)data.length, (const jbyte *)data.bytes); + } + return bytes; +} + +JNIEXPORT jint JNICALL +Java_dev_nucleusframework_screencapture_internal_NativeScreenCapture_nativeListWindows( + JNIEnv *env, jclass clazz, jobject sink, jobjectArray message) { + (void)clazz; + @autoreleasepool { + jclass sinkClass = (*env)->GetObjectClass(env, sink); + jmethodID add = (*env)->GetMethodID(env, sinkClass, "add", "(J[B[BJIIII)V"); + (*env)->DeleteLocalRef(env, sinkClass); + if (add == NULL) { + nucleus_jni_clear_exception(env); + setMessage(env, message, @"WindowCollector.add not found"); + return STATUS_FAILED; + } + // Front to back. Not deprecated, unlike the window images; titles need Screen Recording. + CFArrayRef list = CGWindowListCopyWindowInfo( + kCGWindowListOptionOnScreenOnly | kCGWindowListExcludeDesktopElements, kCGNullWindowID); + if (list == NULL) { + setMessage(env, message, @"CGWindowListCopyWindowInfo returned nothing"); + return STATUS_FAILED; + } + NSArray *windows = CFBridgingRelease(list); + // App windows, floating panels included; the Dock, the menu bar and above are not. + CGWindowLevel dockLevel = CGWindowLevelForKey(kCGDockWindowLevelKey); + for (NSDictionary *window in windows) { + NSInteger layer = [window[(__bridge NSString *)kCGWindowLayer] integerValue]; + if (layer < 0 || layer >= dockLevel) continue; + NSNumber *alpha = window[(__bridge NSString *)kCGWindowAlpha]; + if (alpha != nil && alpha.doubleValue <= 0) continue; + CGRect bounds; + NSDictionary *boundsDict = window[(__bridge NSString *)kCGWindowBounds]; + if (boundsDict == nil || + !CGRectMakeWithDictionaryRepresentation((__bridge CFDictionaryRef)boundsDict, &bounds) || + bounds.size.width < 1 || bounds.size.height < 1) { + continue; + } + jbyteArray title = utf8Bytes(env, window[(__bridge NSString *)kCGWindowName]); + jbyteArray owner = title != NULL ? utf8Bytes(env, window[(__bridge NSString *)kCGWindowOwnerName]) : NULL; + if (title == NULL || owner == NULL) { + if (title != NULL) (*env)->DeleteLocalRef(env, title); + return STATUS_FAILED; // OutOfMemoryError pending: let it propagate + } + (*env)->CallVoidMethod( + env, sink, add, + (jlong)[window[(__bridge NSString *)kCGWindowNumber] longLongValue], title, owner, + (jlong)[window[(__bridge NSString *)kCGWindowOwnerPID] longLongValue], + (jint)lround(bounds.origin.x), (jint)lround(bounds.origin.y), + (jint)lround(bounds.size.width), (jint)lround(bounds.size.height)); + (*env)->DeleteLocalRef(env, title); + (*env)->DeleteLocalRef(env, owner); + if (nucleus_jni_clear_exception(env)) { + setMessage(env, message, @"WindowCollector.add threw"); + return STATUS_FAILED; + } + } + return STATUS_OK; + } +} + // --------------------------------------------------------------------------------------------- // Permission // --------------------------------------------------------------------------------------------- diff --git a/screen-capture/src/main/native/windows/nucleus_screencapture_windows.c b/screen-capture/src/main/native/windows/nucleus_screencapture_windows.c index 683e592e3..79d38712e 100644 --- a/screen-capture/src/main/native/windows/nucleus_screencapture_windows.c +++ b/screen-capture/src/main/native/windows/nucleus_screencapture_windows.c @@ -574,6 +574,132 @@ Java_dev_nucleusframework_screencapture_internal_NativeScreenCapture_nativeCaptu return pixels; } +/* ---------------------------------------------------------------- window list */ + +#define MAX_LISTED_WINDOWS 4096 + +typedef struct { + HWND items[MAX_LISTED_WINDOWS]; + int count; +} WindowList; + +static BOOL CALLBACK collect_window(HWND hwnd, LPARAM param) { + WindowList *list = (WindowList *)param; + if (list->count < MAX_LISTED_WINDOWS) list->items[list->count++] = hwnd; + return TRUE; +} + +/* + * What the Alt+Tab switcher would show: visible, not minimized, not cloaked (another virtual + * desktop, a suspended UWP app), not a tool window, titled, and not the shell's own surfaces. + */ +static BOOL is_listed_window(HWND hwnd) { + if (!IsWindowVisible(hwnd) || IsIconic(hwnd)) return FALSE; + if (GetWindowLongW(hwnd, GWL_EXSTYLE) & WS_EX_TOOLWINDOW) return FALSE; + DWORD cloaked = 0; + if (SUCCEEDED(DwmGetWindowAttribute(hwnd, DWMWA_CLOAKED, &cloaked, sizeof(cloaked))) && cloaked != 0) return FALSE; + WCHAR cls[64]; + if (GetClassNameW(hwnd, cls, 64) > 0 && + (lstrcmpW(cls, L"Progman") == 0 || lstrcmpW(cls, L"WorkerW") == 0 || lstrcmpW(cls, L"Shell_TrayWnd") == 0 || + lstrcmpW(cls, L"Shell_SecondaryTrayWnd") == 0)) { + return FALSE; + } + return TRUE; +} + +/* UTF-8 bytes of a UTF-16 string, for WindowCollector.add; NULL with an OutOfMemoryError pending. */ +static jbyteArray utf8_bytes(JNIEnv *env, const WCHAR *text, int length) { + int size = length > 0 ? WideCharToMultiByte(CP_UTF8, 0, text, length, NULL, 0, NULL, NULL) : 0; + char *buffer = size > 0 ? (char *)malloc((size_t)size) : NULL; + if (buffer != NULL) WideCharToMultiByte(CP_UTF8, 0, text, length, buffer, size, NULL, NULL); + else size = 0; + jbyteArray bytes = (*env)->NewByteArray(env, size); + if (bytes != NULL && size > 0) (*env)->SetByteArrayRegion(env, bytes, 0, size, (const jbyte *)buffer); + free(buffer); + return bytes; +} + +/* The process's executable name without its extension ("chrome"); empty when it cannot be read. */ +static int process_name(DWORD pid, WCHAR *out, int capacity) { + out[0] = 0; + HANDLE process = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, FALSE, pid); + if (process == NULL) return 0; + WCHAR path[MAX_PATH]; + DWORD size = MAX_PATH; + BOOL ok = QueryFullProcessImageNameW(process, 0, path, &size); + CloseHandle(process); + if (!ok) return 0; + const WCHAR *name = path; + for (const WCHAR *p = path; *p; p++) { + if (*p == L'\\' || *p == L'/') name = p + 1; + } + lstrcpynW(out, name, capacity); + WCHAR *dot = NULL; + for (WCHAR *p = out; *p; p++) { + if (*p == L'.') dot = p; + } + if (dot != NULL && dot != out) *dot = 0; + return lstrlenW(out); +} + +JNIEXPORT jint JNICALL +Java_dev_nucleusframework_screencapture_internal_NativeScreenCapture_nativeListWindows(JNIEnv *env, jclass cls, + jobject sink, + jobjectArray message) { + (void)cls; + jclass sink_class = (*env)->GetObjectClass(env, sink); + jmethodID add = (*env)->GetMethodID(env, sink_class, "add", "(J[B[BJIIII)V"); + (*env)->DeleteLocalRef(env, sink_class); + if (add == NULL) { + nucleus_jni_clear_exception(env); + set_message(env, message, "WindowCollector.add not found"); + return STATUS_FAILED; + } + WindowList *list = (WindowList *)calloc(1, sizeof(WindowList)); + if (list == NULL) return STATUS_FAILED; + EnumWindows(collect_window, (LPARAM)list); /* top-level windows, front to back */ + + DPI_AWARENESS_CONTEXT old = enter_physical_pixels(); + int status = STATUS_OK; + for (int i = 0; i < list->count && status == STATUS_OK; i++) { + HWND hwnd = list->items[i]; + if (!is_listed_window(hwnd)) continue; + /* InternalGetWindowText never sends WM_GETTEXT: a hung window (ours included) cannot block it. */ + WCHAR title[512]; + int title_length = InternalGetWindowText(hwnd, title, 512); + if (title_length <= 0) continue; + RECT frame; + if (FAILED(DwmGetWindowAttribute(hwnd, DWMWA_EXTENDED_FRAME_BOUNDS, &frame, sizeof(frame))) && + !GetWindowRect(hwnd, &frame)) { + continue; + } + if (frame.right <= frame.left || frame.bottom <= frame.top) continue; + DWORD pid = 0; + GetWindowThreadProcessId(hwnd, &pid); + WCHAR app[MAX_PATH]; + int app_length = process_name(pid, app, MAX_PATH); + + jbyteArray jtitle = utf8_bytes(env, title, title_length); + jbyteArray japp = jtitle != NULL ? utf8_bytes(env, app, app_length) : NULL; + if (jtitle == NULL || japp == NULL) { + if (jtitle != NULL) (*env)->DeleteLocalRef(env, jtitle); + status = STATUS_FAILED; /* OutOfMemoryError pending: let it propagate */ + break; + } + (*env)->CallVoidMethod(env, sink, add, (jlong)(intptr_t)hwnd, jtitle, japp, (jlong)pid, (jint)frame.left, + (jint)frame.top, (jint)(frame.right - frame.left), (jint)(frame.bottom - frame.top)); + (*env)->DeleteLocalRef(env, jtitle); + (*env)->DeleteLocalRef(env, japp); + if (nucleus_jni_clear_exception(env)) { + set_message(env, message, "WindowCollector.add threw"); + status = STATUS_FAILED; + } + } + leave_physical_pixels(old); + free(list); + return status; +} + JNIEXPORT jint JNICALL Java_dev_nucleusframework_screencapture_internal_NativeScreenCapture_nativePermissionStatus(JNIEnv *env, jclass cls) { (void)env; diff --git a/screen-capture/src/main/resources/META-INF/native-image/dev.nucleusframework/nucleus.screen-capture/reachability-metadata.json b/screen-capture/src/main/resources/META-INF/native-image/dev.nucleusframework/nucleus.screen-capture/reachability-metadata.json index 34db02b15..d77455fa5 100644 --- a/screen-capture/src/main/resources/META-INF/native-image/dev.nucleusframework/nucleus.screen-capture/reachability-metadata.json +++ b/screen-capture/src/main/resources/META-INF/native-image/dev.nucleusframework/nucleus.screen-capture/reachability-metadata.json @@ -21,6 +21,25 @@ } ] }, + { + "type": "dev.nucleusframework.screencapture.internal.WindowCollector", + "jniAccessible": true, + "methods": [ + { + "name": "add", + "parameterTypes": [ + "long", + "byte[]", + "byte[]", + "long", + "int", + "int", + "int", + "int" + ] + } + ] + }, { "type": "dev.nucleusframework.screencapture.internal.NativeScreenCapture", "jniAccessible": true diff --git a/screen-capture/src/test/kotlin/dev/nucleusframework/screencapture/ScreenCaptureLiveTest.kt b/screen-capture/src/test/kotlin/dev/nucleusframework/screencapture/ScreenCaptureLiveTest.kt index 9baa6fa59..a34c6490d 100644 --- a/screen-capture/src/test/kotlin/dev/nucleusframework/screencapture/ScreenCaptureLiveTest.kt +++ b/screen-capture/src/test/kotlin/dev/nucleusframework/screencapture/ScreenCaptureLiveTest.kt @@ -52,6 +52,21 @@ class ScreenCaptureLiveTest { } } + @Test + fun `listed windows are distinct and capturable`() { + assumeCapturable() + val windows = ScreenCapture.windows() + assertEquals(windows.size, windows.map { it.id }.toSet().size, "duplicate ids: $windows") + // The first few only: a window may close between the listing and its capture. + val captured = + windows.take(5).count { window -> + runCatching { ScreenCapture.captureWindow(window.id) } + .onSuccess { assertTrue(it.width > 0 && it.height > 0, "$window -> $it") } + .isSuccess + } + assertTrue(windows.isEmpty() || captured > 0, "none of ${windows.take(5)} could be captured") + } + @Test fun `full capture has the display's pixel size`() { assumeCapturable() diff --git a/scripts/screen-capture-linux-e2e/harness/dev/nucleusframework/screencapture/internal/Harness.java b/scripts/screen-capture-linux-e2e/harness/dev/nucleusframework/screencapture/internal/Harness.java index fad51ff5f..221abe3a2 100644 --- a/scripts/screen-capture-linux-e2e/harness/dev/nucleusframework/screencapture/internal/Harness.java +++ b/scripts/screen-capture-linux-e2e/harness/dev/nucleusframework/screencapture/internal/Harness.java @@ -269,6 +269,12 @@ static void cursorCheck(DisplayCollector.D d) throws Exception { static void windowChecks(long deadWindow) { int[] res = new int[3]; String[] m = new String[1]; + // Without a window manager the list comes from XQueryTree: the test window must be in it, + // at its place, and the dead one must not. + WindowCollector wc = new WindowCollector(); + check(NativeScreenCapture.nativeListWindows(wc, m) == 0, "window list " + m[0]); + check(wc.list.stream().anyMatch(v -> v.id == wid && v.x == wx && v.y == wy && v.w == ww && v.h == wh), "window listed at its geometry: " + wc.list); + check(wc.list.stream().noneMatch(v -> v.id == deadWindow), "dead window not listed"); int[] px = NativeScreenCapture.nativeCaptureWindow(wid, false, res, m); check(px != null && res[0] == 0 && res[1] == ww && res[2] == wh, "window capture size " + res[1] + "x" + res[2] + " st " + res[0] + " " + m[0]); if (px != null) { @@ -383,6 +389,7 @@ static void noDisplay() { int[] res = new int[3]; check(NativeScreenCapture.nativeCaptureDisplay("screen", 0, 0, 0, 0, false, res, m) == null && res[0] == 1, "capture -> UNSUPPORTED " + res[0]); check(NativeScreenCapture.nativeCaptureWindow(1234, false, res, m) == null && res[0] == 1, "window -> UNSUPPORTED " + res[0]); + check(NativeScreenCapture.nativeListWindows(new WindowCollector(), m) == 1, "window list -> UNSUPPORTED: " + m[0]); } // X server killed while captures run: must not exit the JVM. diff --git a/scripts/screen-capture-linux-e2e/harness/dev/nucleusframework/screencapture/internal/NativeScreenCapture.java b/scripts/screen-capture-linux-e2e/harness/dev/nucleusframework/screencapture/internal/NativeScreenCapture.java index a909c9658..fbd3aa8cf 100644 --- a/scripts/screen-capture-linux-e2e/harness/dev/nucleusframework/screencapture/internal/NativeScreenCapture.java +++ b/scripts/screen-capture-linux-e2e/harness/dev/nucleusframework/screencapture/internal/NativeScreenCapture.java @@ -3,6 +3,7 @@ public final class NativeScreenCapture { public static native int nativeBackend(); public static native int nativeListDisplays(DisplayCollector sink, String[] message); + public static native int nativeListWindows(WindowCollector sink, String[] message); public static native int[] nativeCaptureDisplay(String id, int x, int y, int w, int h, boolean cursor, int[] result, String[] message); public static native int[] nativeCaptureWindow(long id, boolean cursor, int[] result, String[] message); public static native int nativePermissionStatus(); diff --git a/scripts/screen-capture-linux-e2e/harness/dev/nucleusframework/screencapture/internal/WindowCollector.java b/scripts/screen-capture-linux-e2e/harness/dev/nucleusframework/screencapture/internal/WindowCollector.java new file mode 100644 index 000000000..543f50ebe --- /dev/null +++ b/scripts/screen-capture-linux-e2e/harness/dev/nucleusframework/screencapture/internal/WindowCollector.java @@ -0,0 +1,18 @@ +package dev.nucleusframework.screencapture.internal; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +public final class WindowCollector { + public static final class W { + public long id, pid; public String title, app; public int x, y, w, h; + public String toString() { return Long.toHexString(id) + "[" + title + "/" + app + " pid=" + pid + " " + x + "," + y + " " + w + "x" + h + "]"; } + } + public final List<W> list = new ArrayList<>(); + public void add(long id, byte[] title, byte[] app, long pid, int x, int y, int w, int h) { + W v = new W(); v.id = id; v.title = new String(title, StandardCharsets.UTF_8); v.app = new String(app, StandardCharsets.UTF_8); + v.pid = pid; v.x = x; v.y = y; v.w = w; v.h = h; + list.add(v); + } +} diff --git a/scripts/screen-capture-macos-e2e.sh b/scripts/screen-capture-macos-e2e.sh index a8a7e2d59..41924be7b 100755 --- a/scripts/screen-capture-macos-e2e.sh +++ b/scripts/screen-capture-macos-e2e.sh @@ -1,7 +1,6 @@ #!/bin/bash # macOS E2E of screen-capture: runs examples/screen-capture-demo in self-test mode, the same # protocol as screen-capture-windows-e2e.ps1. On top of the demo's own checks this script: -# - hands the demo its CGWindowID (Tao does not expose it yet); # - parks the cursor at the centre of the main display (cursor compositing check); # - opens a window whose app never pumps its run loop again (a hung foreign window); # - samples the demo's file descriptors, threads, Mach ports and RSS around the torture phase. @@ -14,8 +13,8 @@ TIMEOUT=${2:-900} ROOT="$(cd "$(dirname "$0")/.." && pwd)" OUT=${OUT_DIR:-${TMPDIR:-/tmp}/screen-capture-e2e} mkdir -p "$OUT" -LOG="$OUT/selftest.log"; IDFILE="$OUT/window.id"; HUNGFILE="$OUT/hung.id" -rm -f "$LOG" "$IDFILE" "$HUNGFILE" +LOG="$OUT/selftest.log"; HUNGFILE="$OUT/hung.id" +rm -f "$LOG" "$HUNGFILE" HELPER="$OUT/helper" swiftc -O -o "$HELPER" "$ROOT/scripts/screen-capture-macos-e2e/helper.swift" || exit 1 @@ -33,16 +32,13 @@ $(top -l 1 -pid "$1" -stats ports | tail -1 | tr -dc '0-9') $(ps -o rss= -p "$1" SCREEN_CAPTURE_DEMO_SELFTEST=1 SCREEN_CAPTURE_DEMO_LOG="$LOG" SCREEN_CAPTURE_DEMO_OUT="$OUT" \ SCREEN_CAPTURE_DEMO_TORTURE="$TORTURE" SCREEN_CAPTURE_DEMO_HUNG_HWND="$HUNG_ID" \ -SCREEN_CAPTURE_DEMO_CURSOR_EXPECTED=1 SCREEN_CAPTURE_DEMO_WINDOW_ID_FILE="$IDFILE" \ +SCREEN_CAPTURE_DEMO_CURSOR_EXPECTED=1 \ "$ROOT/gradlew" -p "$ROOT" :examples:screen-capture-demo:run --console=plain -q >"$OUT/gradle.out" 2>&1 & GRADLE=$! deadline=$((SECONDS + TIMEOUT)); APP=0; BEFORE=""; AFTER="" while [ $SECONDS -lt $deadline ]; do sleep 0.5 - if [ ! -s "$IDFILE" ]; then - id=$("$HELPER" windowid 0 "Screen Capture Demo" 2>/dev/null) && echo "$id" >"$IDFILE" && echo "demo window: $id" - fi [ -f "$LOG" ] || { kill -0 $GRADLE 2>/dev/null && continue || break; } [ $APP = 0 ] && APP=$(sed -n 's/.*START pid=\([0-9]*\).*/\1/p' "$LOG" | head -1) && APP=${APP:-0} if [ $APP != 0 ] && [ -z "$BEFORE" ] && grep -q "PHASE torture-begin" "$LOG"; then sleep 1; BEFORE=$(sample $APP); fi diff --git a/scripts/screen-capture-macos-e2e/helper.swift b/scripts/screen-capture-macos-e2e/helper.swift index c64bcf2d9..8c99a2dd2 100644 --- a/scripts/screen-capture-macos-e2e/helper.swift +++ b/scripts/screen-capture-macos-e2e/helper.swift @@ -1,18 +1,10 @@ // Helper of screen-capture-macos-e2e.sh. -// helper windowid <pid> <title> CGWindowID of the on-screen window of <pid> (0: any) named <title> // helper park cursor to the centre of the main display // helper hung <file> opens a window, writes its CGWindowID to <file>, never pumps again import AppKit let args = CommandLine.arguments switch args.count > 1 ? args[1] : "" { -case "windowid": - let pid = Int32(args[2])! - let list = CGWindowListCopyWindowInfo([.optionOnScreenOnly], kCGNullWindowID) as? [[String: Any]] ?? [] - for w in list where (pid == 0 || (w[kCGWindowOwnerPID as String] as? Int32) == pid) && (w[kCGWindowName as String] as? String) == args[3] { - print(w[kCGWindowNumber as String] as! Int); exit(0) - } - exit(1) case "park": let b = CGDisplayBounds(CGMainDisplayID()) // A real motion event: a warp alone leaves a cursor hidden by setHiddenUntilMouseMoves hidden. From e63ad3b26ca86c54c76cbdad94144c925bab3fd1 Mon Sep 17 00:00:00 2001 From: Elie Gambache <elyahou.hadass@gmail.com> Date: Sun, 27 Sep 2026 07:32:03 +0300 Subject: [PATCH 4/4] fix(ci): refresh API dumps, classify TaoMouseButtonWireDriftTest nucleus-2.6's own Pre Merge Checks fail on the same four tasks: - The Kotlin compiler now emits a public no-arg constructor for classes whose parameters all have defaults and include value classes (Dp, Duration). Dumps of decorated-window-core, decorated-window-tao and scheduler refreshed; additions only, nothing removed. - TaoMouseButtonWireDriftTest (#728) reads events.rs from the repo, a wire guard like TaoScrollWireDriftTest: registered as JVM-only. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> --- decorated-window-core/api/decorated-window-core.api | 2 ++ decorated-window-tao/api/decorated-window-tao.api | 5 +++++ .../window/tao/TaoSceneTestBatteryDriftTest.kt | 2 ++ scheduler/api/scheduler.api | 2 ++ 4 files changed, 11 insertions(+) diff --git a/decorated-window-core/api/decorated-window-core.api b/decorated-window-core/api/decorated-window-core.api index bfe4762e7..8720bc181 100644 --- a/decorated-window-core/api/decorated-window-core.api +++ b/decorated-window-core/api/decorated-window-core.api @@ -710,6 +710,7 @@ public final class dev/nucleusframework/window/styling/DecoratedWindowColors { public final class dev/nucleusframework/window/styling/DecoratedWindowMetrics { public static final field $stable I + public fun <init> ()V public synthetic fun <init> (FILkotlin/jvm/internal/DefaultConstructorMarker;)V public synthetic fun <init> (FLkotlin/jvm/internal/DefaultConstructorMarker;)V public final fun component1-D9Ej5fM ()F @@ -771,6 +772,7 @@ public final class dev/nucleusframework/window/styling/TitleBarColors { public final class dev/nucleusframework/window/styling/TitleBarMetrics { public static final field $stable I + public fun <init> ()V public synthetic fun <init> (FFFJILkotlin/jvm/internal/DefaultConstructorMarker;)V public synthetic fun <init> (FFFJLkotlin/jvm/internal/DefaultConstructorMarker;)V public final fun component1-D9Ej5fM ()F diff --git a/decorated-window-tao/api/decorated-window-tao.api b/decorated-window-tao/api/decorated-window-tao.api index a70909b1e..36b6b948d 100644 --- a/decorated-window-tao/api/decorated-window-tao.api +++ b/decorated-window-tao/api/decorated-window-tao.api @@ -591,6 +591,7 @@ public final class dev/nucleusframework/window/tao/SatellitePlacement$Docked : d public final class dev/nucleusframework/window/tao/SatellitePlacement$Floating : dev/nucleusframework/window/tao/SatellitePlacement { public static final field $stable I public static final field Companion Ldev/nucleusframework/window/tao/SatellitePlacement$Floating$Companion; + public fun <init> ()V public synthetic fun <init> (Ldev/nucleusframework/window/tao/WindowPositioner;JLandroidx/compose/ui/unit/DpRect;ILkotlin/jvm/internal/DefaultConstructorMarker;)V public synthetic fun <init> (Ldev/nucleusframework/window/tao/WindowPositioner;JLandroidx/compose/ui/unit/DpRect;Lkotlin/jvm/internal/DefaultConstructorMarker;)V public final fun component1 ()Ldev/nucleusframework/window/tao/WindowPositioner; @@ -649,6 +650,7 @@ public final class dev/nucleusframework/window/tao/SatelliteWindowKt { public final class dev/nucleusframework/window/tao/SatelliteWindowState { public static final field $stable I + public fun <init> ()V public synthetic fun <init> (JLdev/nucleusframework/window/tao/WindowPositioner;Landroidx/compose/ui/unit/DpRect;ILkotlin/jvm/internal/DefaultConstructorMarker;)V public synthetic fun <init> (JLdev/nucleusframework/window/tao/WindowPositioner;Landroidx/compose/ui/unit/DpRect;Lkotlin/jvm/internal/DefaultConstructorMarker;)V public final fun getAnchorRect ()Landroidx/compose/ui/unit/DpRect; @@ -815,6 +817,7 @@ public final class dev/nucleusframework/window/tao/TabGroupSnapshot { public final class dev/nucleusframework/window/tao/TabHoverPreview { public static final field $stable I public static final field Companion Ldev/nucleusframework/window/tao/TabHoverPreview$Companion; + public fun <init> ()V public synthetic fun <init> (JJZLkotlin/jvm/functions/Function3;ILkotlin/jvm/internal/DefaultConstructorMarker;)V public synthetic fun <init> (JJZLkotlin/jvm/functions/Function3;Lkotlin/jvm/internal/DefaultConstructorMarker;)V public final fun getContent ()Lkotlin/jvm/functions/Function3; @@ -915,6 +918,7 @@ public final class dev/nucleusframework/window/tao/TabWindowsKt { public final class dev/nucleusframework/window/tao/TabWorkspace { public static final field $stable I public static final field Companion Ldev/nucleusframework/window/tao/TabWorkspace$Companion; + public fun <init> ()V public synthetic fun <init> (JZILkotlin/jvm/internal/DefaultConstructorMarker;)V public synthetic fun <init> (JZLkotlin/jvm/internal/DefaultConstructorMarker;)V public final fun beginDrag-0AR0LA0 (Ljava/lang/String;Ldev/nucleusframework/window/tao/TabDragOrigin;J)Ldev/nucleusframework/window/tao/TabDragSession; @@ -1512,6 +1516,7 @@ public final class dev/nucleusframework/window/tao/WindowExceptionHandlerFactory public final class dev/nucleusframework/window/tao/WindowPositioner { public static final field $stable I + public fun <init> ()V public synthetic fun <init> (Ldev/nucleusframework/window/tao/WindowAnchor;Ldev/nucleusframework/window/tao/WindowAnchor;JLdev/nucleusframework/window/tao/WindowConstraintAdjustment;ILkotlin/jvm/internal/DefaultConstructorMarker;)V public synthetic fun <init> (Ldev/nucleusframework/window/tao/WindowAnchor;Ldev/nucleusframework/window/tao/WindowAnchor;JLdev/nucleusframework/window/tao/WindowConstraintAdjustment;Lkotlin/jvm/internal/DefaultConstructorMarker;)V public final fun component1 ()Ldev/nucleusframework/window/tao/WindowAnchor; diff --git a/decorated-window-tao/src/test/kotlin/dev/nucleusframework/window/tao/TaoSceneTestBatteryDriftTest.kt b/decorated-window-tao/src/test/kotlin/dev/nucleusframework/window/tao/TaoSceneTestBatteryDriftTest.kt index 43a1926a1..290fa7b94 100644 --- a/decorated-window-tao/src/test/kotlin/dev/nucleusframework/window/tao/TaoSceneTestBatteryDriftTest.kt +++ b/decorated-window-tao/src/test/kotlin/dev/nucleusframework/window/tao/TaoSceneTestBatteryDriftTest.kt @@ -135,6 +135,8 @@ class TaoSceneTestBatteryDriftTest { TaoTransferableAccessGuardTest::class.java to "Compose interop ABI guard, not a scene behaviour", TaoScrollWireDriftTest::class.java to "reads popup_panel.m / events.rs from the repo; wire guard, not a scene behaviour", + TaoMouseButtonWireDriftTest::class.java to + "reads events.rs from the repo; wire guard, not a scene behaviour", dev.nucleusframework.window.tao.scene.TaoKeepScreenOnTest::class.java to "acquires real EnergyManager awake handles against the host OS", TaoSceneTestBatteryDriftTest::class.java to "meta-test for the battery itself", diff --git a/scheduler/api/scheduler.api b/scheduler/api/scheduler.api index 6d55a8638..a049918c6 100644 --- a/scheduler/api/scheduler.api +++ b/scheduler/api/scheduler.api @@ -204,6 +204,7 @@ public abstract class dev/nucleusframework/scheduler/RetryPolicy { public final class dev/nucleusframework/scheduler/RetryPolicy$ExponentialBackoff : dev/nucleusframework/scheduler/RetryPolicy { public static final field MAX_SHIFT I + public fun <init> ()V public synthetic fun <init> (JIILkotlin/jvm/internal/DefaultConstructorMarker;)V public synthetic fun <init> (JILkotlin/jvm/internal/DefaultConstructorMarker;)V public final fun component1-UwyO8pc ()J @@ -218,6 +219,7 @@ public final class dev/nucleusframework/scheduler/RetryPolicy$ExponentialBackoff } public final class dev/nucleusframework/scheduler/RetryPolicy$Linear : dev/nucleusframework/scheduler/RetryPolicy { + public fun <init> ()V public synthetic fun <init> (JIILkotlin/jvm/internal/DefaultConstructorMarker;)V public synthetic fun <init> (JILkotlin/jvm/internal/DefaultConstructorMarker;)V public final fun component1-UwyO8pc ()J