From 8833736b6fc8b88d97068fa5a4d4643867cfe6d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=80=E4=B8=AA=E6=88=90=E7=86=9F=E7=9A=84=E5=89=AA?= =?UTF-8?q?=E8=BE=91=E7=8C=BF?= <297668904+akahoz94@users.noreply.github.com> Date: Wed, 4 Nov 2026 08:58:47 +0800 Subject: [PATCH 1/3] feat(recording): show pressed keys on screen while a take is live Adds a keystroke overlay for the recorded display, keyviz-style: dark badges bottom-centre naming the keys pressed during a recording, so tutorials can show the keyboard without a separate tool. - The Windows cursor sampler already runs a global WH_MOUSE_LL hook and streams live JSONL; a WH_KEYBOARD_LL hook rides the same pattern. Keydowns queue in the hook (which must not block) and the sampling loop drains them as {"type":"key",...} lines; auto-repeat is filtered. A key hook failure warns and never fails the recording. - windowsNativeRecordingSession forwards "key" events to an optional onKeyEvent callback; handlers broadcast them to the overlay window, which lives only while a take is live and is bounded to the selected source's display. Content protection keeps the badges out of the video (same trade as the HUD on pre-22H2 Windows). - The renderer composes Ctrl/Alt/Shift/Win combos (keystrokeLabels.ts) and fades badges after ~1.3 s. Depends on the cursor-sampler helper running (cursor capture enabled). Tests: vitest keystrokeLabels.test.ts (5 cases, the vk mapping and combo order), CountdownOverlay tests still 4/4; tsc both configs clean; biome clean; native helper rebuilt, all 22 C++ tests pass. --- electron/electron-env.d.ts | 2 + electron/ipc/handlers.ts | 16 ++++ electron/keystrokeOverlay.ts | 62 +++++++++++++ .../native-bridge/cursor/recording/factory.ts | 3 + .../windowsNativeRecordingSession.ts | 6 ++ .../windowsNativeRecordingSession.types.ts | 11 +++ .../native/wgc-capture/src/cursor-sampler.cpp | 53 +++++++++++ electron/preload.ts | 5 + electron/windows.ts | 45 +++++++++ src/App.tsx | 3 + src/components/launch/KeystrokeOverlay.tsx | 70 ++++++++++++++ src/index.css | 27 ++++++ src/lib/keystrokeLabels.test.ts | 41 +++++++++ src/lib/keystrokeLabels.ts | 91 +++++++++++++++++++ src/main.tsx | 3 +- 15 files changed, 437 insertions(+), 1 deletion(-) create mode 100644 electron/keystrokeOverlay.ts create mode 100644 src/components/launch/KeystrokeOverlay.tsx create mode 100644 src/lib/keystrokeLabels.test.ts create mode 100644 src/lib/keystrokeLabels.ts diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index f0c93ff955..fe425da363 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -553,6 +553,8 @@ interface Window { ) => () => void; /** The overlay painted a frame without the digit; a Linux hide waits for this. */ notifyCountdownOverlayCleared: (runId: number) => void; + /** A live keydown while a take is running (`KeystrokeOverlay`). */ + onKeystrokeKey: (callback: (vk: number) => void) => () => void; setMicrophoneExpanded: (expanded: boolean) => void; setHasUnsavedChanges: (hasChanges: boolean) => void; onRequestSaveBeforeClose: (callback: () => Promise | boolean) => () => void; diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index e4bab0f5cb..88376254b6 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -82,6 +82,11 @@ import { isDiagnosticModeEnabled, mainLogFile } from "../diagnostics/main-log-fi import { createZip } from "../diagnostics/zip"; import { mainT } from "../i18n"; import { getInstallChannel } from "../install-channel"; +import { + broadcastKeystroke, + hideKeystrokeOverlay, + showKeystrokeOverlay, +} from "../keystrokeOverlay"; import { DEFAULT_RECORDINGS_DIR, getRecordingsDirInfo, @@ -1398,6 +1403,7 @@ async function startCursorRecording(recordingId?: number) { sourceId: getSelectedSourceId(), startTimeMs: typeof recordingId === "number" && Number.isFinite(recordingId) ? recordingId : undefined, + onKeyEvent: (event) => broadcastKeystroke(event.vk), }); try { @@ -2056,6 +2062,16 @@ export function registerIpcHandlers( if (!recording && !preparing) cameraPreview.setCaptureAllowed(true); if (!preparing) mcpAppRecorder.recordingChanged(recording, recordingId); onRecordingStateChange?.(recording, sourceName, preparing); + // The keystroke overlay lives only while a take is live, on the display being + // recorded. Windows only: the overlay's invisibility-to-capture relies on + // content protection, which `applyContentProtection` gates per platform. + if (process.platform === "win32") { + if (recording) { + void showKeystrokeOverlay(getSelectedSourceBounds()); + } else { + hideKeystrokeOverlay(); + } + } }; const appSettings = new AppSettingsStore(app.getPath("userData")); const broadcastSelectedSource = (source: SelectedSource | null) => { diff --git a/electron/keystrokeOverlay.ts b/electron/keystrokeOverlay.ts new file mode 100644 index 0000000000..11ee407102 --- /dev/null +++ b/electron/keystrokeOverlay.ts @@ -0,0 +1,62 @@ +import { BrowserWindow, type Rectangle } from "electron"; +import { createKeystrokeOverlayWindow } from "./windows"; + +/** + * Lifecycle of the keystroke overlay: one window, alive only while a take is live, + * bounded to the display being recorded. The window is destroyed on hide rather than + * cached — a hidden window would otherwise linger as the app's last window and block + * quit on Windows, the same way the countdown overlay's lingering is special-cased in + * `main.ts`. A take recreates it lazily, and the first badges may land during the + * ~0.2 s the renderer needs to load: acceptable for a hint overlay. + */ +let overlayWindow: BrowserWindow | null = null; + +async function ensureOverlayWindow(): Promise { + if (overlayWindow && !overlayWindow.isDestroyed()) { + return overlayWindow; + } + overlayWindow = createKeystrokeOverlayWindow(); + if (overlayWindow.isDestroyed()) { + overlayWindow = null; + return null; + } + // Wait for the first frame before showing, else Chromium flashes a black + // rectangle because it has not rendered any pixels yet. + if (overlayWindow.webContents.isLoading()) { + await new Promise((resolve) => { + overlayWindow?.once("ready-to-show", resolve); + }); + if (!overlayWindow || overlayWindow.isDestroyed()) { + return null; + } + } + return overlayWindow; +} + +export async function showKeystrokeOverlay(bounds: Rectangle): Promise { + const win = await ensureOverlayWindow(); + if (!win || win.isDestroyed()) { + return; + } + + win.setBounds(bounds); + if (!win.isVisible()) { + win.showInactive(); + } +} + +export function hideKeystrokeOverlay(): void { + const win = overlayWindow; + overlayWindow = null; + if (win && !win.isDestroyed()) { + win.destroy(); + } +} + +/** Forwarded live keydowns (`windowsNativeRecordingSession` → `handlers`). */ +export function broadcastKeystroke(vk: number): void { + const win = overlayWindow; + if (win && !win.isDestroyed() && win.isVisible()) { + win.webContents.send("keystroke-overlay-key", vk); + } +} diff --git a/electron/native-bridge/cursor/recording/factory.ts b/electron/native-bridge/cursor/recording/factory.ts index 4bafd38525..c7c8a1f6da 100644 --- a/electron/native-bridge/cursor/recording/factory.ts +++ b/electron/native-bridge/cursor/recording/factory.ts @@ -13,6 +13,8 @@ interface CreateCursorRecordingSessionOptions { sampleIntervalMs: number; sourceId?: string | null; startTimeMs?: number; + /** Live Windows keydowns for the keystroke overlay (win32 only). */ + onKeyEvent?: (event: { type: "key"; timestampMs: number; vk: number }) => void; } export function createCursorRecordingSession( @@ -25,6 +27,7 @@ export function createCursorRecordingSession( sampleIntervalMs: options.sampleIntervalMs, sourceId: options.sourceId, startTimeMs: options.startTimeMs, + onKeyEvent: options.onKeyEvent, }); } diff --git a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts index 87be87d68c..5ee689b2b4 100644 --- a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts +++ b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts @@ -186,6 +186,12 @@ export class WindowsNativeRecordingSession implements CursorRecordingSession { return; } + if (payload.type === "key") { + // The keystroke overlay's live feed — never touches the telemetry sidecar. + this.options.onKeyEvent?.(payload); + return; + } + if (payload.asset?.id && !this.assets.has(payload.asset.id)) { // payload.x/y are physical screen pixels; `screen` works in DIPs. const assetDisplay = screen.getDisplayNearestPoint( diff --git a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts index f3b69da0fd..55478ac4fe 100644 --- a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts +++ b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts @@ -26,6 +26,14 @@ export interface WindowsCursorReadyEvent { timestampMs: number; } +/** A live keydown from the sampler's WH_KEYBOARD_LL hook, for the keystroke overlay. */ +export interface WindowsCursorKeyEvent { + type: "key"; + timestampMs: number; + /** Win32 virtual-key code (`KBDLLHOOKSTRUCT.vkCode`). */ + vk: number; +} + export interface WindowsCursorErrorEvent { type: "error"; timestampMs: number; @@ -45,6 +53,7 @@ export interface WindowsCursorAssetPayload { export type WindowsCursorEvent = | WindowsCursorSampleEvent | WindowsCursorReadyEvent + | WindowsCursorKeyEvent | WindowsCursorErrorEvent; export interface WindowsNativeRecordingSessionOptions { @@ -53,4 +62,6 @@ export interface WindowsNativeRecordingSessionOptions { sampleIntervalMs: number; sourceId?: string | null; startTimeMs?: number; + /** Live keydowns, forwarded to the keystroke overlay. Absent = not consumed. */ + onKeyEvent?: (event: WindowsCursorKeyEvent) => void; } diff --git a/electron/native/wgc-capture/src/cursor-sampler.cpp b/electron/native/wgc-capture/src/cursor-sampler.cpp index cf4947c556..8f52d8ab20 100644 --- a/electron/native/wgc-capture/src/cursor-sampler.cpp +++ b/electron/native/wgc-capture/src/cursor-sampler.cpp @@ -28,6 +28,18 @@ static std::atomic g_leftUpCount{0}; static std::atomic g_stop{false}; static std::mutex g_stdoutMtx; +// ───────────────────────────────────────────────────────────────────────────── +// Global keyboard-hook state: keydowns the on-screen keystroke overlay shows. +// The hook only queues — an LL hook must not block, so the sampling loop drains +// the queue and writes the JSONL lines. ponytail: the repeat filter is one +// atomic, so alternating two held keys re-queues a repeat of the first one; +// a per-key down set is the upgrade if that ever matters. +// ───────────────────────────────────────────────────────────────────────────── +static HHOOK g_keyHook = nullptr; +static std::mutex g_keyMtx; +static std::vector g_keyQueue; +static std::atomic g_lastKeyDown{0}; + static LRESULT CALLBACK LowLevelMouseProc(int nCode, WPARAM wParam, LPARAM lParam) { if (nCode >= 0) { if (wParam == WM_LBUTTONDOWN) g_leftDownCount.fetch_add(1, std::memory_order_relaxed); @@ -36,6 +48,23 @@ static LRESULT CALLBACK LowLevelMouseProc(int nCode, WPARAM wParam, LPARAM lPara return CallNextHookEx(g_mouseHook, nCode, wParam, lParam); } +static LRESULT CALLBACK LowLevelKeyboardProc(int nCode, WPARAM wParam, LPARAM lParam) { + if (nCode >= 0) { + const KBDLLHOOKSTRUCT* info = reinterpret_cast(lParam); + if (info && (wParam == WM_KEYDOWN || wParam == WM_SYSKEYDOWN)) { + // The LL hook keeps firing while a key is held; the overlay wants one + // badge per press, so only a change of key passes the filter. + if (g_lastKeyDown.exchange(info->vkCode, std::memory_order_relaxed) != info->vkCode) { + std::lock_guard lock(g_keyMtx); + g_keyQueue.push_back(info->vkCode); + } + } else if (info && (wParam == WM_KEYUP || wParam == WM_SYSKEYUP)) { + g_lastKeyDown.store(0, std::memory_order_relaxed); + } + } + return CallNextHookEx(g_keyHook, nCode, wParam, lParam); +} + // ───────────────────────────────────────────────────────────────────────────── // Utilities // ───────────────────────────────────────────────────────────────────────────── @@ -322,6 +351,22 @@ static void runSamplingLoop(int intervalMs, HWND targetWindow, const CLSID& pngC const int downCount = g_leftDownCount.exchange(0, std::memory_order_relaxed); const int upCount = g_leftUpCount.exchange(0, std::memory_order_relaxed); + // Drain the keyboard hook's queue: one JSONL line per keydown, numeric + // fields only, so nothing needs escaping. Keyups carry no information the + // overlay shows, so they are never emitted. + std::vector keys; + { + std::lock_guard lock(g_keyMtx); + keys.swap(g_keyQueue); + } + for (const DWORD vk : keys) { + char keyBuf[96]; + std::snprintf(keyBuf, sizeof(keyBuf), + "{\"type\":\"key\",\"timestampMs\":%" PRId64 ",\"vk\":%lu}", + nowMs(), static_cast(vk)); + writeJsonLine(keyBuf); + } + CURSORINFO ci{}; ci.cbSize = sizeof(ci); if (!GetCursorInfo(&ci)) { @@ -474,6 +519,13 @@ int main(int argc, char* argv[]) { return 1; } + // The keyboard hook feeds the keystroke overlay only, so its failure costs + // nothing the recording needs — warn and keep going rather than aborting. + g_keyHook = SetWindowsHookEx(WH_KEYBOARD_LL, LowLevelKeyboardProc, GetModuleHandle(nullptr), 0); + if (!g_keyHook) { + std::cerr << "SetWindowsHookEx(WH_KEYBOARD_LL) failed; no keystroke overlay" << std::endl; + } + // Prime GetAsyncKeyState so the first poll doesn't return stale "since-last-call" bits GetAsyncKeyState(VK_LBUTTON); @@ -499,6 +551,7 @@ int main(int argc, char* argv[]) { g_stop.store(true, std::memory_order_relaxed); if (sampler.joinable()) sampler.join(); UnhookWindowsHookEx(g_mouseHook); + if (g_keyHook) UnhookWindowsHookEx(g_keyHook); Gdiplus::GdiplusShutdown(gdipToken); return 0; } diff --git a/electron/preload.ts b/electron/preload.ts index 3f5716036d..d90d8898f7 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -621,6 +621,11 @@ contextBridge.exposeInMainWorld("electronAPI", { notifyCountdownOverlayCleared: (runId: number) => { ipcRenderer.send("countdown-overlay-cleared", runId); }, + onKeystrokeKey: (callback: (vk: number) => void) => { + const listener = (_event: unknown, vk: number) => callback(vk); + ipcRenderer.on("keystroke-overlay-key", listener); + return () => ipcRenderer.removeListener("keystroke-overlay-key", listener); + }, onRequestSaveBeforeClose: (callback: () => Promise | boolean) => { const listener = async () => { try { diff --git a/electron/windows.ts b/electron/windows.ts index 386e256cce..6c9589f878 100644 --- a/electron/windows.ts +++ b/electron/windows.ts @@ -852,6 +852,51 @@ export function createCountdownOverlayWindow(): BrowserWindow { return win; } +/** + * Transparent click-through window that shows the keys pressed while a take is live + * (`keystrokeOverlay.ts` owns the lifecycle). Unlike the countdown overlay this one is + * visible DURING the capture, so content protection is what keeps it out of the video — + * same trade as the HUD: on Windows before 11 22H2 the protection has to go, and the + * badges would land in the recording (see applyContentProtection). + */ +export function createKeystrokeOverlayWindow(): BrowserWindow { + const win = new BrowserWindow({ + width: 480, + height: 200, + frame: false, + resizable: false, + alwaysOnTop: true, + skipTaskbar: true, + focusable: false, + transparent: true, + backgroundColor: "#00000000", + hasShadow: false, + show: false, + webPreferences: { + preload: path.join(__dirname, "preload.mjs"), + additionalArguments: [ASSET_BASE_URL_ARG], + nodeIntegration: false, + contextIsolation: true, + backgroundThrottling: false, + }, + }); + + win.setIgnoreMouseEvents(true); + applyContentProtection(win, "keystroke overlay", { clickThrough: true }); + keepAlwaysOnTop(win); + + if (VITE_DEV_SERVER_URL) { + win.loadURL(VITE_DEV_SERVER_URL + "?windowType=keystroke-overlay"); + } else { + win.loadFile(path.join(RENDERER_DIST, "index.html"), { + query: { windowType: "keystroke-overlay" }, + }); + } + + markSheetless(win); + return win; +} + /** * The macOS permissions window: first run, the app menu, and wherever a missing permission * would otherwise stop a recording. An ordinary opaque window on purpose -- it has to sit diff --git a/src/App.tsx b/src/App.tsx index c326d3b835..4b579ee59c 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -7,6 +7,7 @@ installBrowserShims(); import { CountdownOverlay } from "./components/launch/CountdownOverlay.tsx"; import { FloatingCameraPreview } from "./components/launch/FloatingCameraPreview"; +import { KeystrokeOverlay } from "./components/launch/KeystrokeOverlay.tsx"; import { LaunchWindow } from "./components/launch/LaunchWindow"; import { NotesWindow } from "./components/launch/NotesWindow.tsx"; import { SourceSelector } from "./components/launch/SourceSelector"; @@ -92,6 +93,8 @@ export default function App() { return ; case "countdown-overlay": return ; + case "keystroke-overlay": + return ; case "permissions": return ; case "cli-export": diff --git a/src/components/launch/KeystrokeOverlay.tsx b/src/components/launch/KeystrokeOverlay.tsx new file mode 100644 index 0000000000..5a8563b0cb --- /dev/null +++ b/src/components/launch/KeystrokeOverlay.tsx @@ -0,0 +1,70 @@ +import { useEffect, useRef, useState } from "react"; +import { badgeLabel, VK_CONTROL, VK_LWIN, VK_MENU, VK_RWIN, VK_SHIFT } from "@/lib/keystrokeLabels"; + +/** How long a badge stays up before it fades out (matched by the CSS animation). */ +export const BADGE_VISIBLE_MS = 1_300; + +interface KeystrokeBadge { + id: number; + label: string; +} + +/** Modifier virtual keys, as a set the badge labels compose from. */ +const MODIFIER_KEYS = new Set([VK_SHIFT, VK_CONTROL, VK_MENU, VK_LWIN, VK_RWIN]); + +/** + * The keystroke overlay: badges for the keys pressed while a take is live, bottom-centre + * of the display being recorded. Keydowns arrive live from the Windows sampler's + * WH_KEYBOARD_LL hook (`keystroke-overlay-key` via preload); nothing is shown when no + * take is running, because the window itself only exists then. + */ +export function KeystrokeOverlay() { + const [badges, setBadges] = useState([]); + const heldModifiers = useRef>(new Set()); + const nextBadgeId = useRef(1); + + useEffect(() => { + const timers: ReturnType[] = []; + + const unsubscribe = window.electronAPI.onKeystrokeKey((vk) => { + // Compose the label from the modifiers held so far; a non-modifier press then + // releases them: the badge pair Ctrl then C reads as one "Ctrl+C". + const isModifier = MODIFIER_KEYS.has(vk); + const label = badgeLabel(vk, heldModifiers.current); + if (!isModifier) { + heldModifiers.current.clear(); + } + if (!label) return; + + const id = nextBadgeId.current++; + setBadges((current) => [...current.slice(-4), { id, label }]); + timers.push( + setTimeout(() => { + setBadges((current) => current.filter((badge) => badge.id !== id)); + }, BADGE_VISIBLE_MS), + ); + }); + + return () => { + unsubscribe(); + for (const timer of timers) clearTimeout(timer); + heldModifiers.current.clear(); + }; + }, []); + + if (badges.length === 0) { + return null; + } + + return ( +
+
+ {badges.map((badge) => ( +
+ {badge.label} +
+ ))} +
+
+ ); +} diff --git a/src/index.css b/src/index.css index 2a92cfc263..daf62254ac 100644 --- a/src/index.css +++ b/src/index.css @@ -397,3 +397,30 @@ html [data-sonner-toaster] [data-sonner-toast][data-styled="true"] [data-close-b background: color-mix(in oklab, var(--fg) 10%, transparent); color: var(--fg); } +/* Keystroke overlay badges (`KeystrokeOverlay.tsx`): dark pills with the key combo, + bottom-centre of the recorded display. The fade lives on the element itself, one + animation per badge, so the stack never needs transition juggling. */ +@keyframes keystroke-badge-fade { + 0%, + 80% { + opacity: 1; + } + 100% { + opacity: 0; + } +} + +.keystroke-badge { + display: inline-flex; + align-items: center; + padding: 8px 16px; + border-radius: 10px; + background: rgba(0, 0, 0, 0.72); + border: 1px solid rgba(255, 255, 255, 0.18); + color: rgba(255, 255, 255, 0.92); + font-size: 16px; + font-weight: 600; + letter-spacing: 0.02em; + box-shadow: 0 6px 24px rgba(0, 0, 0, 0.45); + animation: keystroke-badge-fade 1.3s ease-out 1 both; +} diff --git a/src/lib/keystrokeLabels.test.ts b/src/lib/keystrokeLabels.test.ts new file mode 100644 index 0000000000..d570e00e06 --- /dev/null +++ b/src/lib/keystrokeLabels.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it } from "vitest"; +import { badgeLabel, baseKeyLabel, isModifierVk, VK_CONTROL, VK_SHIFT } from "./keystrokeLabels"; + +describe("keystrokeLabels", () => { + it("names letters, digits and function keys off the vk code ranges", () => { + expect(baseKeyLabel(0x41)).toBe("A"); + expect(baseKeyLabel(0x31)).toBe("1"); + expect(baseKeyLabel(0x70)).toBe("F1"); + expect(baseKeyLabel(0x87)).toBe("F24"); + expect(baseKeyLabel(0x64)).toBe("Num4"); + }); + + it("names the named and OEM keys the overlay is likely to see", () => { + expect(baseKeyLabel(0x0d)).toBe("Enter"); + expect(baseKeyLabel(0x1b)).toBe("Esc"); + expect(baseKeyLabel(0x20)).toBe("Space"); + expect(baseKeyLabel(0xba)).toBe(";"); + expect(baseKeyLabel(0xdb)).toBe("["); + }); + + it("returns null for keys the overlay hides, and names every modifier", () => { + expect(baseKeyLabel(0xa6)).toBeNull(); // media key, no label + expect(baseKeyLabel(0x10)).toBe("Shift"); + expect(baseKeyLabel(0x11)).toBe("Ctrl"); + expect(baseKeyLabel(0x12)).toBe("Alt"); + expect(baseKeyLabel(0x5b)).toBe("Win"); + }); + + it("composes held modifiers in Ctrl, Alt, Shift, Win order", () => { + expect(badgeLabel(0x43, new Set([VK_CONTROL, VK_SHIFT]))).toBe("Ctrl+Shift+C"); + expect(badgeLabel(0x2e, new Set([VK_CONTROL]))).toBe("Ctrl+Del"); + expect(badgeLabel(0x43, new Set())).toBe("C"); + }); + + it("shows modifiers alone and never double-prefixes one", () => { + expect(badgeLabel(0x10, new Set([VK_SHIFT]))).toBe("Shift"); + expect(badgeLabel(0x11, new Set())).toBe("Ctrl"); + expect(isModifierVk(0x11)).toBe(true); + expect(isModifierVk(0x41)).toBe(false); + }); +}); diff --git a/src/lib/keystrokeLabels.ts b/src/lib/keystrokeLabels.ts new file mode 100644 index 0000000000..794d303217 --- /dev/null +++ b/src/lib/keystrokeLabels.ts @@ -0,0 +1,91 @@ +// Win32 virtual-key codes → display labels for the keystroke overlay. +// +// The C++ sampler reports raw `vkCode`s (`cursor-sampler.cpp`); naming and modifier +// composition live here, in the renderer. Modifiers compose a combo prefix on the next +// non-modifier key, keyviz-style; a modifier press on its own shows as a badge too, so +// the user sees Ctrl happen even if nothing follows it. + +export const VK_SHIFT = 0x10; +export const VK_CONTROL = 0x11; +export const VK_MENU = 0x12; +export const VK_LWIN = 0x5b; +export const VK_RWIN = 0x5c; + +const MODIFIER_VKS: readonly number[] = [VK_SHIFT, VK_CONTROL, VK_MENU, VK_LWIN, VK_RWIN]; + +/** Combo prefix order, the one keyviz uses: Ctrl · Alt · Shift · Win. */ +const MODIFIER_ORDER: readonly [number, string][] = [ + [VK_CONTROL, "Ctrl"], + [VK_MENU, "Alt"], + [VK_SHIFT, "Shift"], + [VK_LWIN, "Win"], + [VK_RWIN, "Win"], +]; + +const NAMED_KEYS: Record = { + 0x08: "Backspace", + 0x09: "Tab", + 0x0d: "Enter", + 0x13: "Pause", + 0x14: "CapsLock", + 0x1b: "Esc", + 0x20: "Space", + 0x21: "PageUp", + 0x22: "PageDown", + 0x23: "End", + 0x24: "Home", + 0x25: "←", + 0x26: "↑", + 0x27: "→", + 0x28: "↓", + 0x2d: "Ins", + 0x2e: "Del", +}; + +const OEM_KEYS: Record = { + 0xba: ";", + 0xbb: "=", + 0xbc: ",", + 0xbd: "-", + 0xbe: ".", + 0xbf: "/", + 0xc0: "`", + 0xdb: "[", + 0xdc: "\\", + 0xdd: "]", + 0xde: "'", +}; + +/** Labels for one virtual key, without modifiers; `null` for keys the overlay hides. */ +export function baseKeyLabel(vk: number): string | null { + if (vk >= 0x30 && vk <= 0x39) return String.fromCharCode(vk); + if (vk >= 0x41 && vk <= 0x5a) return String.fromCharCode(vk); + if (vk >= 0x60 && vk <= 0x69) return `Num${vk - 0x60}`; + if (vk >= 0x70 && vk <= 0x87) return `F${vk - 0x70 + 1}`; + if (vk === VK_LWIN || vk === VK_RWIN) return "Win"; + if (vk === VK_SHIFT) return "Shift"; + if (vk === VK_CONTROL) return "Ctrl"; + if (vk === VK_MENU) return "Alt"; + return NAMED_KEYS[vk] ?? OEM_KEYS[vk] ?? null; +} + +/** `true` when the key is one of the composing modifiers the overlay tracks. */ +export function isModifierVk(vk: number): boolean { + return MODIFIER_VKS.includes(vk); +} + +/** + * The badge label for a keydown: the held modifiers as a `Ctrl+Alt+Delete`-style combo, + * or the bare key. `heldModifiers` carries the modifier keys still down, as tracked by + * the overlay from the same event stream. + */ +export function badgeLabel(vk: number, heldModifiers: ReadonlySet): string | null { + const base = baseKeyLabel(vk); + if (base === null) return null; + if (isModifierVk(vk)) return base; + + const prefix = MODIFIER_ORDER.filter(([modVk]) => heldModifiers.has(modVk)).map( + ([, label]) => label, + ); + return [...prefix, base].join("+"); +} diff --git a/src/main.tsx b/src/main.tsx index 939360e624..366eab1516 100644 --- a/src/main.tsx +++ b/src/main.tsx @@ -21,7 +21,8 @@ if ( showNotes || windowType === "hud-overlay" || windowType === "source-selector" || - windowType === "countdown-overlay" + windowType === "countdown-overlay" || + windowType === "keystroke-overlay" ) { document.body.style.background = "transparent"; document.documentElement.style.background = "transparent"; From d263732e02dcfd456d42e1b4942e7cce2b153b2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=80=E4=B8=AA=E6=88=90=E7=86=9F=E7=9A=84=E5=89=AA?= =?UTF-8?q?=E8=BE=91=E7=8C=BF?= <297668904+akahoz94@users.noreply.github.com> Date: Fri, 20 Nov 2026 05:59:57 +0800 Subject: [PATCH 2/3] fix(keystroke): show real modifier combos and paint the overlay before showing CodeRabbit review follow-ups for the keystroke-overlay PR: - normalise left/right Shift/Ctrl/Alt virtual keys to the generic modifier codes in the keyboard hook, and carry the modifier state captured at event time with each keydown (the sampling loop drains too late to infer it); - compose the badge from that event-time mask, with local modifier accumulation as a fallback, so Ctrl+C reads as one combo and a standalone modifier keeps its own badge; - make every overlay show wait on the same first-frame readiness signal, including callers that receive an already-created window; - position the overlay from the take's resolved capture bounds when the native request supplied them, falling back to the selected-source bounds. Adds a component test driving onKeystrokeKey for modifier combos. --- electron/electron-env.d.ts | 2 +- electron/ipc/handlers.ts | 17 ++++- electron/keystrokeOverlay.ts | 51 ++++++++++----- .../native-bridge/cursor/recording/factory.ts | 2 +- .../windowsNativeRecordingSession.types.ts | 4 +- .../native/wgc-capture/src/cursor-sampler.cpp | 58 ++++++++++++++--- electron/preload.ts | 4 +- .../launch/KeystrokeOverlay.test.tsx | 65 +++++++++++++++++++ src/components/launch/KeystrokeOverlay.tsx | 33 ++++++++-- 9 files changed, 197 insertions(+), 39 deletions(-) create mode 100644 src/components/launch/KeystrokeOverlay.test.tsx diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index fe425da363..923f5b884d 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -554,7 +554,7 @@ interface Window { /** The overlay painted a frame without the digit; a Linux hide waits for this. */ notifyCountdownOverlayCleared: (runId: number) => void; /** A live keydown while a take is running (`KeystrokeOverlay`). */ - onKeystrokeKey: (callback: (vk: number) => void) => () => void; + onKeystrokeKey: (callback: (vk: number, modifiers: number) => void) => () => void; setMicrophoneExpanded: (expanded: boolean) => void; setHasUnsavedChanges: (hasChanges: boolean) => void; onRequestSaveBeforeClose: (callback: () => Promise | boolean) => () => void; diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index 88376254b6..24a3e1ce67 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -1403,7 +1403,7 @@ async function startCursorRecording(recordingId?: number) { sourceId: getSelectedSourceId(), startTimeMs: typeof recordingId === "number" && Number.isFinite(recordingId) ? recordingId : undefined, - onKeyEvent: (event) => broadcastKeystroke(event.vk), + onKeyEvent: (event) => broadcastKeystroke(event.vk, event.modifiers), }); try { @@ -2043,6 +2043,13 @@ export function registerIpcHandlers( ) { /** The HUD is recording a take, the user's or an MCP client's. Set by `setAppRecording` only. */ let appRecording = false; + /** + * Resolved capture bounds of the current take, when the start path knows them + * explicitly (native Windows request can target a display other than the + * mutable selection). The keystroke overlay is positioned from these; the + * selection getter is only a fallback for paths without explicit bounds. + */ + let takeRecordingBounds: Rectangle | null = null; const recordingCompletion = new RecordingCompletion(); /** * Every start and stop of a take goes through here: the browser path's `set-recording-state` @@ -2067,9 +2074,10 @@ export function registerIpcHandlers( // content protection, which `applyContentProtection` gates per platform. if (process.platform === "win32") { if (recording) { - void showKeystrokeOverlay(getSelectedSourceBounds()); + void showKeystrokeOverlay(takeRecordingBounds ?? getSelectedSourceBounds()); } else { hideKeystrokeOverlay(); + takeRecordingBounds = null; } } }; @@ -3157,6 +3165,11 @@ export function registerIpcHandlers( null) : getSelectedDisplay(); const bounds = sourceDisplay?.bounds ?? getSelectedSourceBounds(); + // The overlay must sit on the display this take actually captures, + // which the request resolved above and may differ from the mutable + // selection (MCP clients can target another display). Consumed by + // setAppRecording when it shows the keystroke overlay. + takeRecordingBounds = bounds; // `bounds` is DIPs; the helper matches it against physical monitor rects // (getopenscreen/openscreen#346). Converted here, at the wire, and not in // `getSelectedSourceBounds` — the cursor session shares that getter and diff --git a/electron/keystrokeOverlay.ts b/electron/keystrokeOverlay.ts index 11ee407102..f84d970831 100644 --- a/electron/keystrokeOverlay.ts +++ b/electron/keystrokeOverlay.ts @@ -10,26 +10,44 @@ import { createKeystrokeOverlayWindow } from "./windows"; * ~0.2 s the renderer needs to load: acceptable for a hint overlay. */ let overlayWindow: BrowserWindow | null = null; +// The first-frame wait, stored alongside the window so EVERY caller awaits it — +// including callers that receive an already-created window still mid-load. Without +// this, a second recording start could showInactive an unpainted window (#1633). +let overlayReady: Promise | null = null; async function ensureOverlayWindow(): Promise { - if (overlayWindow && !overlayWindow.isDestroyed()) { - return overlayWindow; + if (!overlayWindow || overlayWindow.isDestroyed()) { + overlayWindow = createKeystrokeOverlayWindow(); + if (overlayWindow.isDestroyed()) { + overlayWindow = null; + return null; + } + // Wait for the first frame before showing, else Chromium flashes a black + // rectangle because it has not rendered any pixels yet. The resolve is + // idempotent and shared by all callers for this window's lifetime. + overlayReady = new Promise((resolve) => { + const win = overlayWindow; + if (!win || win.isDestroyed()) { + resolve(); + return; + } + if (win.webContents.isLoading()) { + win.once("ready-to-show", resolve); + // A window destroyed before its first frame must not leave waiters + // hanging; resolve on close too, the destroyed check below then bails. + win.once("closed", resolve); + } else { + resolve(); + } + }); } - overlayWindow = createKeystrokeOverlayWindow(); - if (overlayWindow.isDestroyed()) { + // Every caller, new window or reused, waits on the same readiness signal. + await overlayReady; + if (!overlayWindow || overlayWindow.isDestroyed()) { overlayWindow = null; + overlayReady = null; return null; } - // Wait for the first frame before showing, else Chromium flashes a black - // rectangle because it has not rendered any pixels yet. - if (overlayWindow.webContents.isLoading()) { - await new Promise((resolve) => { - overlayWindow?.once("ready-to-show", resolve); - }); - if (!overlayWindow || overlayWindow.isDestroyed()) { - return null; - } - } return overlayWindow; } @@ -48,15 +66,16 @@ export async function showKeystrokeOverlay(bounds: Rectangle): Promise { export function hideKeystrokeOverlay(): void { const win = overlayWindow; overlayWindow = null; + overlayReady = null; if (win && !win.isDestroyed()) { win.destroy(); } } /** Forwarded live keydowns (`windowsNativeRecordingSession` → `handlers`). */ -export function broadcastKeystroke(vk: number): void { +export function broadcastKeystroke(vk: number, modifiers: number): void { const win = overlayWindow; if (win && !win.isDestroyed() && win.isVisible()) { - win.webContents.send("keystroke-overlay-key", vk); + win.webContents.send("keystroke-overlay-key", vk, modifiers); } } diff --git a/electron/native-bridge/cursor/recording/factory.ts b/electron/native-bridge/cursor/recording/factory.ts index c7c8a1f6da..13314bf6d8 100644 --- a/electron/native-bridge/cursor/recording/factory.ts +++ b/electron/native-bridge/cursor/recording/factory.ts @@ -14,7 +14,7 @@ interface CreateCursorRecordingSessionOptions { sourceId?: string | null; startTimeMs?: number; /** Live Windows keydowns for the keystroke overlay (win32 only). */ - onKeyEvent?: (event: { type: "key"; timestampMs: number; vk: number }) => void; + onKeyEvent?: (event: { type: "key"; timestampMs: number; vk: number; modifiers: number }) => void; } export function createCursorRecordingSession( diff --git a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts index 55478ac4fe..364dbdf1b2 100644 --- a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts +++ b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts @@ -30,8 +30,10 @@ export interface WindowsCursorReadyEvent { export interface WindowsCursorKeyEvent { type: "key"; timestampMs: number; - /** Win32 virtual-key code (`KBDLLHOOKSTRUCT.vkCode`). */ + /** Win32 virtual-key code (`KBDLLHOOKSTRUCT.vkCode`), left/right modifiers normalised. */ vk: number; + /** Modifier bits held at event time: 1=Shift 2=Ctrl 4=Alt 8=Win. */ + modifiers: number; } export interface WindowsCursorErrorEvent { diff --git a/electron/native/wgc-capture/src/cursor-sampler.cpp b/electron/native/wgc-capture/src/cursor-sampler.cpp index 8f52d8ab20..fb87b5153c 100644 --- a/electron/native/wgc-capture/src/cursor-sampler.cpp +++ b/electron/native/wgc-capture/src/cursor-sampler.cpp @@ -37,9 +37,39 @@ static std::mutex g_stdoutMtx; // ───────────────────────────────────────────────────────────────────────────── static HHOOK g_keyHook = nullptr; static std::mutex g_keyMtx; -static std::vector g_keyQueue; +struct KeyPress { + DWORD vk; + DWORD modifiers; // Modifier mask captured at event time (see modifierMask). +}; +static std::vector g_keyQueue; static std::atomic g_lastKeyDown{0}; +// The generic modifier virtual keys are what the renderer's label layer knows. +// Left/right variants (VK_LSHIFT..VK_RMENU) are normalised here, at the hook, +// so both the repeat filter and every downstream consumer see the generic code. +static DWORD normalizeModifierVk(DWORD vk) { + switch (vk) { + case 0xA0: case 0xA1: return VK_SHIFT; // VK_LSHIFT / VK_RSHIFT + case 0xA2: case 0xA3: return VK_CONTROL; // VK_LCONTROL / VK_RCONTROL + case 0xA4: case 0xA5: return VK_MENU; // VK_LMENU / VK_RMENU + default: return vk; + } +} + +// Bit mask of modifiers currently held, read IN THE HOOK at event time. The +// sampling loop drains the queue later, when the physical state may have +// changed; inferring it there would mislabel a combo whose modifier was already +// released. Bit values match the renderer's MODIFIER mask in keystrokeLabels.ts. +static DWORD modifierMask() { + DWORD mask = 0; + if (GetAsyncKeyState(VK_SHIFT) & 0x8000) mask |= 1; // Shift + if (GetAsyncKeyState(VK_CONTROL) & 0x8000) mask |= 2; // Ctrl + if (GetAsyncKeyState(VK_MENU) & 0x8000) mask |= 4; // Alt + if (GetAsyncKeyState(VK_LWIN) & 0x8000) mask |= 8; // Win (either side) + if (GetAsyncKeyState(VK_RWIN) & 0x8000) mask |= 8; + return mask; +} + static LRESULT CALLBACK LowLevelMouseProc(int nCode, WPARAM wParam, LPARAM lParam) { if (nCode >= 0) { if (wParam == WM_LBUTTONDOWN) g_leftDownCount.fetch_add(1, std::memory_order_relaxed); @@ -52,11 +82,17 @@ static LRESULT CALLBACK LowLevelKeyboardProc(int nCode, WPARAM wParam, LPARAM lP if (nCode >= 0) { const KBDLLHOOKSTRUCT* info = reinterpret_cast(lParam); if (info && (wParam == WM_KEYDOWN || wParam == WM_SYSKEYDOWN)) { + const DWORD vk = normalizeModifierVk(info->vkCode); // The LL hook keeps firing while a key is held; the overlay wants one - // badge per press, so only a change of key passes the filter. - if (g_lastKeyDown.exchange(info->vkCode, std::memory_order_relaxed) != info->vkCode) { + // badge per press, so only a change of key passes the filter. The + // normalised code is used so L/R variants of a modifier do not count + // as alternating distinct keys. + if (g_lastKeyDown.exchange(vk, std::memory_order_relaxed) != vk) { + // Capture the modifier state here, at event time, not when the + // sampling loop drains the queue (#1633). + const DWORD mods = modifierMask(); std::lock_guard lock(g_keyMtx); - g_keyQueue.push_back(info->vkCode); + g_keyQueue.push_back({vk, mods}); } } else if (info && (wParam == WM_KEYUP || wParam == WM_SYSKEYUP)) { g_lastKeyDown.store(0, std::memory_order_relaxed); @@ -353,17 +389,19 @@ static void runSamplingLoop(int intervalMs, HWND targetWindow, const CLSID& pngC // Drain the keyboard hook's queue: one JSONL line per keydown, numeric // fields only, so nothing needs escaping. Keyups carry no information the - // overlay shows, so they are never emitted. - std::vector keys; + // overlay shows, so they are never emitted. `modifiers` was captured in + // the hook at event time so a released modifier cannot relabel the combo. + std::vector keys; { std::lock_guard lock(g_keyMtx); keys.swap(g_keyQueue); } - for (const DWORD vk : keys) { - char keyBuf[96]; + for (const KeyPress& key : keys) { + char keyBuf[128]; std::snprintf(keyBuf, sizeof(keyBuf), - "{\"type\":\"key\",\"timestampMs\":%" PRId64 ",\"vk\":%lu}", - nowMs(), static_cast(vk)); + "{\"type\":\"key\",\"timestampMs\":%" PRId64 ",\"vk\":%lu,\"modifiers\":%lu}", + nowMs(), static_cast(key.vk), + static_cast(key.modifiers)); writeJsonLine(keyBuf); } diff --git a/electron/preload.ts b/electron/preload.ts index d90d8898f7..7f23b6919b 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -621,8 +621,8 @@ contextBridge.exposeInMainWorld("electronAPI", { notifyCountdownOverlayCleared: (runId: number) => { ipcRenderer.send("countdown-overlay-cleared", runId); }, - onKeystrokeKey: (callback: (vk: number) => void) => { - const listener = (_event: unknown, vk: number) => callback(vk); + onKeystrokeKey: (callback: (vk: number, modifiers: number) => void) => { + const listener = (_event: unknown, vk: number, modifiers: number) => callback(vk, modifiers); ipcRenderer.on("keystroke-overlay-key", listener); return () => ipcRenderer.removeListener("keystroke-overlay-key", listener); }, diff --git a/src/components/launch/KeystrokeOverlay.test.tsx b/src/components/launch/KeystrokeOverlay.test.tsx new file mode 100644 index 0000000000..ee1d8ca017 --- /dev/null +++ b/src/components/launch/KeystrokeOverlay.test.tsx @@ -0,0 +1,65 @@ +// @vitest-environment jsdom +import { act, render } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { KeystrokeOverlay } from "./KeystrokeOverlay"; + +type KeyListener = (vk: number, modifiers: number) => void; + +describe("KeystrokeOverlay", () => { + let pushKey: KeyListener = () => undefined; + + beforeEach(() => { + window.electronAPI = { + ...window.electronAPI, + onKeystrokeKey: (listener: KeyListener) => { + pushKey = listener; + return () => undefined; + }, + } as typeof window.electronAPI; + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it("shows a modifier followed by a regular key as one combined badge", () => { + const { container } = render(); + + // The sampler sends the modifier press, then the letter with the modifier + // bit held at event time. The overlay must compose "Ctrl+C", not two badges. + act(() => pushKey(0x11, 2)); // Ctrl down (modifier bit for itself) + act(() => pushKey(0x43, 2)); // C down while Ctrl held + + expect(container.textContent).toContain("Ctrl+C"); + }); + + it("composes from the event-time modifier mask alone", () => { + const { container } = render(); + + // No modifier keydown was ever delivered, but C arrives with Ctrl in its + // mask (e.g. the first key seen mid-take). The mask alone must still yield + // a combo rather than a bare "C". + act(() => pushKey(0x43, 2)); + expect(container.textContent).toContain("Ctrl+C"); + }); + + it("leaves an unmodified key as a bare badge", () => { + const { container } = render(); + act(() => pushKey(0x43, 0)); + expect(container.textContent).toBe("C"); + }); + + it("keeps a standalone modifier visible and clears the combo after the next key", () => { + const { container } = render(); + act(() => pushKey(0x11, 2)); // Ctrl alone + expect(container.textContent).toContain("Ctrl"); + + act(() => pushKey(0x43, 2)); // Ctrl+C + expect(container.textContent).toContain("Ctrl+C"); + + // A later key with no modifier must not inherit the released Ctrl. + act(() => pushKey(0x44, 0)); // D, no modifiers + expect(container.textContent).toContain("D"); + expect(container.textContent).not.toContain("Ctrl+D"); + }); +}); diff --git a/src/components/launch/KeystrokeOverlay.tsx b/src/components/launch/KeystrokeOverlay.tsx index 5a8563b0cb..68e0186e1d 100644 --- a/src/components/launch/KeystrokeOverlay.tsx +++ b/src/components/launch/KeystrokeOverlay.tsx @@ -12,6 +12,14 @@ interface KeystrokeBadge { /** Modifier virtual keys, as a set the badge labels compose from. */ const MODIFIER_KEYS = new Set([VK_SHIFT, VK_CONTROL, VK_MENU, VK_LWIN, VK_RWIN]); +/** Modifier bit mask from the sampler: 1=Shift 2=Ctrl 4=Alt 8=Win. */ +const MASK_TO_VK: readonly [number, number][] = [ + [1, VK_SHIFT], + [2, VK_CONTROL], + [4, VK_MENU], + [8, VK_LWIN], +]; + /** * The keystroke overlay: badges for the keys pressed while a take is live, bottom-centre * of the display being recorded. Keydowns arrive live from the Windows sampler's @@ -26,21 +34,34 @@ export function KeystrokeOverlay() { useEffect(() => { const timers: ReturnType[] = []; - const unsubscribe = window.electronAPI.onKeystrokeKey((vk) => { - // Compose the label from the modifiers held so far; a non-modifier press then - // releases them: the badge pair Ctrl then C reads as one "Ctrl+C". + const unsubscribe = window.electronAPI.onKeystrokeKey((vk, modifiers = 0) => { + // The sampler captured the modifier state at the key's event time. Expand + // it into the same VK set the labels compose from; the local accumulation + // is a fallback for streams that omit modifiers (older helpers). Both are + // unioned so a modifier press followed by a regular key reads as one combo. + const current = new Set(heldModifiers.current); + for (const [bit, modVk] of MASK_TO_VK) { + if (modifiers & bit) current.add(modVk); + } const isModifier = MODIFIER_KEYS.has(vk); - const label = badgeLabel(vk, heldModifiers.current); + if (isModifier) { + // A standalone modifier still shows its own badge, and stays held for + // the next non-modifier key (rather than being cleared). + heldModifiers.current.add(vk); + current.add(vk); + } + // Compose from the current set BEFORE a non-modifier releases it. + const label = badgeLabel(vk, current); if (!isModifier) { heldModifiers.current.clear(); } if (!label) return; const id = nextBadgeId.current++; - setBadges((current) => [...current.slice(-4), { id, label }]); + setBadges((currentBadges) => [...currentBadges.slice(-4), { id, label }]); timers.push( setTimeout(() => { - setBadges((current) => current.filter((badge) => badge.id !== id)); + setBadges((currentBadges) => currentBadges.filter((badge) => badge.id !== id)); }, BADGE_VISIBLE_MS), ); }); From f7bac5e79c57da89326ab6befcf4c1453b902e76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=80=E4=B8=AA=E6=88=90=E7=86=9F=E7=9A=84=E5=89=AA?= =?UTF-8?q?=E8=BE=91=E7=8C=BF?= <297668904+akahoz94@users.noreply.github.com> Date: Fri, 20 Nov 2026 20:01:29 +0800 Subject: [PATCH 3/3] fix(keystroke): capture keys in every cursor mode and harden overlay lifecycle CodeRabbit follow-up on the keystroke-overlay PR: - give keyboard capture its own session for Windows takes that have no cursor telemetry session (system-cursor mode). The cursor session previously carried the low-level keyboard hook, so system-cursor takes showed an overlay that never displayed a key. The key-only session is tied to the take lifecycle and never writes cursor telemetry. - treat an explicit modifier mask (including 0) as authoritative; only streams that omit it fall back to locally accumulated modifiers, so Ctrl down/up then C reads as a bare C. - capture the overlay window before the first-frame await and verify it is still current, so a stop+start during the wait cannot show a replacement window with stale bounds. - publish the resolved take bounds only after the final start reservation guard, so a rejected start cannot move an already-live take's overlay. Adds component tests for the explicit-mask and no-mask paths. --- electron/electron-env.d.ts | 2 +- electron/ipc/handlers.ts | 62 +++++++++++++++++-- electron/keystrokeOverlay.ts | 16 +++-- .../launch/KeystrokeOverlay.test.tsx | 22 ++++++- src/components/launch/KeystrokeOverlay.tsx | 24 ++++--- 5 files changed, 105 insertions(+), 21 deletions(-) diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index 923f5b884d..c29f9e8711 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -554,7 +554,7 @@ interface Window { /** The overlay painted a frame without the digit; a Linux hide waits for this. */ notifyCountdownOverlayCleared: (runId: number) => void; /** A live keydown while a take is running (`KeystrokeOverlay`). */ - onKeystrokeKey: (callback: (vk: number, modifiers: number) => void) => () => void; + onKeystrokeKey: (callback: (vk: number, modifiers?: number) => void) => () => void; setMicrophoneExpanded: (expanded: boolean) => void; setHasUnsavedChanges: (hasChanges: boolean) => void; onRequestSaveBeforeClose: (callback: () => Promise | boolean) => () => void; diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index 24a3e1ce67..a90ec41c19 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -802,6 +802,14 @@ const CURSOR_SAMPLE_INTERVAL_MS = 33; const MAX_CURSOR_SAMPLES = 60 * 60 * 30; // 1 hour @ 30Hz let cursorRecordingSession: CursorRecordingSession | null = null; +/** + * Keyboard-only capture for takes that do not run a cursor telemetry session + * (Windows system-cursor mode). The keystroke overlay needs the native helper's + * low-level keyboard hook even when no editable cursor overlay is drawn; reusing + * the telemetry session would tie key capture to `editable-overlay` and leave + * system-cursor takes with an overlay that never shows a key (#1633 review). + */ +let keystrokeKeySession: CursorRecordingSession | null = null; let pendingCursorRecordingData: CursorRecordingData | null = null; let nativeWindowsCaptureProcess: ChildProcessWithoutNullStreams | null = null; let nativeWindowsCaptureOutput = ""; @@ -1429,6 +1437,42 @@ async function stopCursorRecording() { } } +/** + * Start a keyboard-only native session for a Windows take that has no cursor + * telemetry session. The helper's low-level keyboard hook is what drives the + * keystroke overlay; it is independent of the selected cursor mode. No-op when + * a cursor session already exists (it carries the same hook) or off Windows. + */ +async function startKeystrokeKeyCapture(): Promise { + if (process.platform !== "win32" || cursorRecordingSession || keystrokeKeySession) return; + const session = createCursorRecordingSession({ + getDisplayBounds: getSelectedSourceBounds, + maxSamples: MAX_CURSOR_SAMPLES, + platform: "win32", + sampleIntervalMs: CURSOR_SAMPLE_INTERVAL_MS, + sourceId: getSelectedSourceId(), + onKeyEvent: (event) => broadcastKeystroke(event.vk, event.modifiers), + }); + try { + await session.start(); + keystrokeKeySession = session; + } catch (error) { + console.error("Failed to start keystroke key capture:", error); + } +} + +/** Stop the keyboard-only session; its cursor samples are never written out. */ +async function stopKeystrokeKeyCapture(): Promise { + const session = keystrokeKeySession; + keystrokeKeySession = null; + if (!session) return; + try { + await session.stop(); + } catch (error) { + console.error("Failed to stop keystroke key capture:", error); + } +} + async function writePendingCursorTelemetry(videoPath: string) { const telemetryPath = `${videoPath}.cursor.json`; if (pendingCursorRecordingData && pendingCursorRecordingData.samples.length > 0) { @@ -2075,9 +2119,16 @@ export function registerIpcHandlers( if (process.platform === "win32") { if (recording) { void showKeystrokeOverlay(takeRecordingBounds ?? getSelectedSourceBounds()); + // The keystroke overlay needs the native keyboard hook in every cursor + // mode. In editable-overlay mode the cursor session already carries it; + // otherwise (system cursor) no session is running, so start a + // keyboard-only one. Guarded by `!preparing` so it attaches to the take + // proper, after the editable-overlay session has had a chance to start. + if (!preparing) void startKeystrokeKeyCapture(); } else { hideKeystrokeOverlay(); takeRecordingBounds = null; + void stopKeystrokeKeyCapture(); } } }; @@ -3165,11 +3216,6 @@ export function registerIpcHandlers( null) : getSelectedDisplay(); const bounds = sourceDisplay?.bounds ?? getSelectedSourceBounds(); - // The overlay must sit on the display this take actually captures, - // which the request resolved above and may differ from the mutable - // selection (MCP clients can target another display). Consumed by - // setAppRecording when it shows the keystroke overlay. - takeRecordingBounds = bounds; // `bounds` is DIPs; the helper matches it against physical monitor rects // (getopenscreen/openscreen#346). Converted here, at the wire, and not in // `getSelectedSourceBounds` — the cursor session shares that getter and @@ -3272,6 +3318,12 @@ export function registerIpcHandlers( } // Reserve startup before cursor/helper awaits without reporting a started take. startReserved = true; + // Publish the overlay's display only now, after the final reservation + // guard: a rejected start must not move a take that is already live, and + // there is no await between here and the overlay show below (#1633). + // The request resolved this display; it may differ from the mutable + // selection when an MCP client targets another screen. + takeRecordingBounds = bounds; setAppRecording(true, selectedSource?.name ?? "Screen", undefined, true); await fs.mkdir(RECORDINGS_DIR, { recursive: true }); nativeWindowsCaptureOutput = ""; diff --git a/electron/keystrokeOverlay.ts b/electron/keystrokeOverlay.ts index f84d970831..41e7fc5e97 100644 --- a/electron/keystrokeOverlay.ts +++ b/electron/keystrokeOverlay.ts @@ -41,14 +41,22 @@ async function ensureOverlayWindow(): Promise { } }); } + // Capture the window we are about to wait on. A stop + start during the await + // can replace `overlayWindow`; this continuation must not then pick up the new + // window and show it with the old bounds before its own first frame (#1633). + const awaited = overlayWindow; // Every caller, new window or reused, waits on the same readiness signal. await overlayReady; - if (!overlayWindow || overlayWindow.isDestroyed()) { - overlayWindow = null; - overlayReady = null; + if (!awaited || awaited.isDestroyed() || overlayWindow !== awaited) { + // Only clear the globals if the window we waited on is still the current one; + // otherwise a replacement (with its own readiness wait) owns them. + if (overlayWindow === awaited) { + overlayWindow = null; + overlayReady = null; + } return null; } - return overlayWindow; + return awaited; } export async function showKeystrokeOverlay(bounds: Rectangle): Promise { diff --git a/src/components/launch/KeystrokeOverlay.test.tsx b/src/components/launch/KeystrokeOverlay.test.tsx index ee1d8ca017..99a2a684bd 100644 --- a/src/components/launch/KeystrokeOverlay.test.tsx +++ b/src/components/launch/KeystrokeOverlay.test.tsx @@ -3,7 +3,7 @@ import { act, render } from "@testing-library/react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { KeystrokeOverlay } from "./KeystrokeOverlay"; -type KeyListener = (vk: number, modifiers: number) => void; +type KeyListener = (vk: number, modifiers?: number) => void; describe("KeystrokeOverlay", () => { let pushKey: KeyListener = () => undefined; @@ -49,6 +49,26 @@ describe("KeystrokeOverlay", () => { expect(container.textContent).toBe("C"); }); + it("an explicit zero mask overrides a locally accumulated modifier", () => { + const { container } = render(); + + // A stream that omits the mask (older helper) still composes via local + // accumulation. But once an explicit mask arrives it is authoritative: + // Ctrl down with no mask, then C with mask 0 must read as a bare "C". + act(() => pushKey(0x11)); // Ctrl down, mask omitted + act(() => pushKey(0x43, 0)); // C down, explicit no-modifier mask + + expect(container.textContent).toContain("C"); + expect(container.textContent).not.toContain("Ctrl+C"); + }); + + it("an omitted mask falls back to locally accumulated modifiers", () => { + const { container } = render(); + act(() => pushKey(0x11)); // Ctrl down, mask omitted + act(() => pushKey(0x43)); // C down, mask omitted + expect(container.textContent).toContain("Ctrl+C"); + }); + it("keeps a standalone modifier visible and clears the combo after the next key", () => { const { container } = render(); act(() => pushKey(0x11, 2)); // Ctrl alone diff --git a/src/components/launch/KeystrokeOverlay.tsx b/src/components/launch/KeystrokeOverlay.tsx index 68e0186e1d..7feb372405 100644 --- a/src/components/launch/KeystrokeOverlay.tsx +++ b/src/components/launch/KeystrokeOverlay.tsx @@ -34,19 +34,23 @@ export function KeystrokeOverlay() { useEffect(() => { const timers: ReturnType[] = []; - const unsubscribe = window.electronAPI.onKeystrokeKey((vk, modifiers = 0) => { - // The sampler captured the modifier state at the key's event time. Expand - // it into the same VK set the labels compose from; the local accumulation - // is a fallback for streams that omit modifiers (older helpers). Both are - // unioned so a modifier press followed by a regular key reads as one combo. - const current = new Set(heldModifiers.current); - for (const [bit, modVk] of MASK_TO_VK) { - if (modifiers & bit) current.add(modVk); + const unsubscribe = window.electronAPI.onKeystrokeKey((vk, modifiers) => { + // The sampler captures the modifier state at the key's event time and sends + // it with every keydown. An explicit mask -- including 0 -- is authoritative: + // Ctrl down, Ctrl up, then C arrives with mask 0 and must read as "C", not a + // stale "Ctrl+C". Only a stream that omits the mask (an older helper) falls + // back to locally accumulated modifier presses. + const current = new Set(); + if (modifiers === undefined) { + for (const modVk of heldModifiers.current) current.add(modVk); + } else { + for (const [bit, modVk] of MASK_TO_VK) { + if (modifiers & bit) current.add(modVk); + } } const isModifier = MODIFIER_KEYS.has(vk); if (isModifier) { - // A standalone modifier still shows its own badge, and stays held for - // the next non-modifier key (rather than being cleared). + // Track standalone modifier presses for the no-mask fallback only. heldModifiers.current.add(vk); current.add(vk); }