diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index f0c93ff955..c29f9e8711 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -553,6 +553,8 @@ interface Window { ) => () => void; /** The overlay painted a frame without the digit; a Linux hide waits for this. */ notifyCountdownOverlayCleared: (runId: number) => void; + /** A live keydown while a take is running (`KeystrokeOverlay`). */ + onKeystrokeKey: (callback: (vk: number, modifiers?: number) => void) => () => void; setMicrophoneExpanded: (expanded: boolean) => void; setHasUnsavedChanges: (hasChanges: boolean) => void; onRequestSaveBeforeClose: (callback: () => Promise | boolean) => () => void; diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index e4bab0f5cb..a90ec41c19 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -82,6 +82,11 @@ import { isDiagnosticModeEnabled, mainLogFile } from "../diagnostics/main-log-fi import { createZip } from "../diagnostics/zip"; import { mainT } from "../i18n"; import { getInstallChannel } from "../install-channel"; +import { + broadcastKeystroke, + hideKeystrokeOverlay, + showKeystrokeOverlay, +} from "../keystrokeOverlay"; import { DEFAULT_RECORDINGS_DIR, getRecordingsDirInfo, @@ -797,6 +802,14 @@ const CURSOR_SAMPLE_INTERVAL_MS = 33; const MAX_CURSOR_SAMPLES = 60 * 60 * 30; // 1 hour @ 30Hz let cursorRecordingSession: CursorRecordingSession | null = null; +/** + * Keyboard-only capture for takes that do not run a cursor telemetry session + * (Windows system-cursor mode). The keystroke overlay needs the native helper's + * low-level keyboard hook even when no editable cursor overlay is drawn; reusing + * the telemetry session would tie key capture to `editable-overlay` and leave + * system-cursor takes with an overlay that never shows a key (#1633 review). + */ +let keystrokeKeySession: CursorRecordingSession | null = null; let pendingCursorRecordingData: CursorRecordingData | null = null; let nativeWindowsCaptureProcess: ChildProcessWithoutNullStreams | null = null; let nativeWindowsCaptureOutput = ""; @@ -1398,6 +1411,7 @@ async function startCursorRecording(recordingId?: number) { sourceId: getSelectedSourceId(), startTimeMs: typeof recordingId === "number" && Number.isFinite(recordingId) ? recordingId : undefined, + onKeyEvent: (event) => broadcastKeystroke(event.vk, event.modifiers), }); try { @@ -1423,6 +1437,42 @@ async function stopCursorRecording() { } } +/** + * Start a keyboard-only native session for a Windows take that has no cursor + * telemetry session. The helper's low-level keyboard hook is what drives the + * keystroke overlay; it is independent of the selected cursor mode. No-op when + * a cursor session already exists (it carries the same hook) or off Windows. + */ +async function startKeystrokeKeyCapture(): Promise { + if (process.platform !== "win32" || cursorRecordingSession || keystrokeKeySession) return; + const session = createCursorRecordingSession({ + getDisplayBounds: getSelectedSourceBounds, + maxSamples: MAX_CURSOR_SAMPLES, + platform: "win32", + sampleIntervalMs: CURSOR_SAMPLE_INTERVAL_MS, + sourceId: getSelectedSourceId(), + onKeyEvent: (event) => broadcastKeystroke(event.vk, event.modifiers), + }); + try { + await session.start(); + keystrokeKeySession = session; + } catch (error) { + console.error("Failed to start keystroke key capture:", error); + } +} + +/** Stop the keyboard-only session; its cursor samples are never written out. */ +async function stopKeystrokeKeyCapture(): Promise { + const session = keystrokeKeySession; + keystrokeKeySession = null; + if (!session) return; + try { + await session.stop(); + } catch (error) { + console.error("Failed to stop keystroke key capture:", error); + } +} + async function writePendingCursorTelemetry(videoPath: string) { const telemetryPath = `${videoPath}.cursor.json`; if (pendingCursorRecordingData && pendingCursorRecordingData.samples.length > 0) { @@ -2037,6 +2087,13 @@ export function registerIpcHandlers( ) { /** The HUD is recording a take, the user's or an MCP client's. Set by `setAppRecording` only. */ let appRecording = false; + /** + * Resolved capture bounds of the current take, when the start path knows them + * explicitly (native Windows request can target a display other than the + * mutable selection). The keystroke overlay is positioned from these; the + * selection getter is only a fallback for paths without explicit bounds. + */ + let takeRecordingBounds: Rectangle | null = null; const recordingCompletion = new RecordingCompletion(); /** * Every start and stop of a take goes through here: the browser path's `set-recording-state` @@ -2056,6 +2113,24 @@ export function registerIpcHandlers( if (!recording && !preparing) cameraPreview.setCaptureAllowed(true); if (!preparing) mcpAppRecorder.recordingChanged(recording, recordingId); onRecordingStateChange?.(recording, sourceName, preparing); + // The keystroke overlay lives only while a take is live, on the display being + // recorded. Windows only: the overlay's invisibility-to-capture relies on + // content protection, which `applyContentProtection` gates per platform. + if (process.platform === "win32") { + if (recording) { + void showKeystrokeOverlay(takeRecordingBounds ?? getSelectedSourceBounds()); + // The keystroke overlay needs the native keyboard hook in every cursor + // mode. In editable-overlay mode the cursor session already carries it; + // otherwise (system cursor) no session is running, so start a + // keyboard-only one. Guarded by `!preparing` so it attaches to the take + // proper, after the editable-overlay session has had a chance to start. + if (!preparing) void startKeystrokeKeyCapture(); + } else { + hideKeystrokeOverlay(); + takeRecordingBounds = null; + void stopKeystrokeKeyCapture(); + } + } }; const appSettings = new AppSettingsStore(app.getPath("userData")); const broadcastSelectedSource = (source: SelectedSource | null) => { @@ -3243,6 +3318,12 @@ export function registerIpcHandlers( } // Reserve startup before cursor/helper awaits without reporting a started take. startReserved = true; + // Publish the overlay's display only now, after the final reservation + // guard: a rejected start must not move a take that is already live, and + // there is no await between here and the overlay show below (#1633). + // The request resolved this display; it may differ from the mutable + // selection when an MCP client targets another screen. + takeRecordingBounds = bounds; setAppRecording(true, selectedSource?.name ?? "Screen", undefined, true); await fs.mkdir(RECORDINGS_DIR, { recursive: true }); nativeWindowsCaptureOutput = ""; diff --git a/electron/keystrokeOverlay.ts b/electron/keystrokeOverlay.ts new file mode 100644 index 0000000000..41e7fc5e97 --- /dev/null +++ b/electron/keystrokeOverlay.ts @@ -0,0 +1,89 @@ +import { BrowserWindow, type Rectangle } from "electron"; +import { createKeystrokeOverlayWindow } from "./windows"; + +/** + * Lifecycle of the keystroke overlay: one window, alive only while a take is live, + * bounded to the display being recorded. The window is destroyed on hide rather than + * cached — a hidden window would otherwise linger as the app's last window and block + * quit on Windows, the same way the countdown overlay's lingering is special-cased in + * `main.ts`. A take recreates it lazily, and the first badges may land during the + * ~0.2 s the renderer needs to load: acceptable for a hint overlay. + */ +let overlayWindow: BrowserWindow | null = null; +// The first-frame wait, stored alongside the window so EVERY caller awaits it — +// including callers that receive an already-created window still mid-load. Without +// this, a second recording start could showInactive an unpainted window (#1633). +let overlayReady: Promise | null = null; + +async function ensureOverlayWindow(): Promise { + if (!overlayWindow || overlayWindow.isDestroyed()) { + overlayWindow = createKeystrokeOverlayWindow(); + if (overlayWindow.isDestroyed()) { + overlayWindow = null; + return null; + } + // Wait for the first frame before showing, else Chromium flashes a black + // rectangle because it has not rendered any pixels yet. The resolve is + // idempotent and shared by all callers for this window's lifetime. + overlayReady = new Promise((resolve) => { + const win = overlayWindow; + if (!win || win.isDestroyed()) { + resolve(); + return; + } + if (win.webContents.isLoading()) { + win.once("ready-to-show", resolve); + // A window destroyed before its first frame must not leave waiters + // hanging; resolve on close too, the destroyed check below then bails. + win.once("closed", resolve); + } else { + resolve(); + } + }); + } + // Capture the window we are about to wait on. A stop + start during the await + // can replace `overlayWindow`; this continuation must not then pick up the new + // window and show it with the old bounds before its own first frame (#1633). + const awaited = overlayWindow; + // Every caller, new window or reused, waits on the same readiness signal. + await overlayReady; + if (!awaited || awaited.isDestroyed() || overlayWindow !== awaited) { + // Only clear the globals if the window we waited on is still the current one; + // otherwise a replacement (with its own readiness wait) owns them. + if (overlayWindow === awaited) { + overlayWindow = null; + overlayReady = null; + } + return null; + } + return awaited; +} + +export async function showKeystrokeOverlay(bounds: Rectangle): Promise { + const win = await ensureOverlayWindow(); + if (!win || win.isDestroyed()) { + return; + } + + win.setBounds(bounds); + if (!win.isVisible()) { + win.showInactive(); + } +} + +export function hideKeystrokeOverlay(): void { + const win = overlayWindow; + overlayWindow = null; + overlayReady = null; + if (win && !win.isDestroyed()) { + win.destroy(); + } +} + +/** Forwarded live keydowns (`windowsNativeRecordingSession` → `handlers`). */ +export function broadcastKeystroke(vk: number, modifiers: number): void { + const win = overlayWindow; + if (win && !win.isDestroyed() && win.isVisible()) { + win.webContents.send("keystroke-overlay-key", vk, modifiers); + } +} diff --git a/electron/native-bridge/cursor/recording/factory.ts b/electron/native-bridge/cursor/recording/factory.ts index 4bafd38525..13314bf6d8 100644 --- a/electron/native-bridge/cursor/recording/factory.ts +++ b/electron/native-bridge/cursor/recording/factory.ts @@ -13,6 +13,8 @@ interface CreateCursorRecordingSessionOptions { sampleIntervalMs: number; sourceId?: string | null; startTimeMs?: number; + /** Live Windows keydowns for the keystroke overlay (win32 only). */ + onKeyEvent?: (event: { type: "key"; timestampMs: number; vk: number; modifiers: number }) => void; } export function createCursorRecordingSession( @@ -25,6 +27,7 @@ export function createCursorRecordingSession( sampleIntervalMs: options.sampleIntervalMs, sourceId: options.sourceId, startTimeMs: options.startTimeMs, + onKeyEvent: options.onKeyEvent, }); } diff --git a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts index 87be87d68c..5ee689b2b4 100644 --- a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts +++ b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts @@ -186,6 +186,12 @@ export class WindowsNativeRecordingSession implements CursorRecordingSession { return; } + if (payload.type === "key") { + // The keystroke overlay's live feed — never touches the telemetry sidecar. + this.options.onKeyEvent?.(payload); + return; + } + if (payload.asset?.id && !this.assets.has(payload.asset.id)) { // payload.x/y are physical screen pixels; `screen` works in DIPs. const assetDisplay = screen.getDisplayNearestPoint( diff --git a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts index f3b69da0fd..364dbdf1b2 100644 --- a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts +++ b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.types.ts @@ -26,6 +26,16 @@ export interface WindowsCursorReadyEvent { timestampMs: number; } +/** A live keydown from the sampler's WH_KEYBOARD_LL hook, for the keystroke overlay. */ +export interface WindowsCursorKeyEvent { + type: "key"; + timestampMs: number; + /** Win32 virtual-key code (`KBDLLHOOKSTRUCT.vkCode`), left/right modifiers normalised. */ + vk: number; + /** Modifier bits held at event time: 1=Shift 2=Ctrl 4=Alt 8=Win. */ + modifiers: number; +} + export interface WindowsCursorErrorEvent { type: "error"; timestampMs: number; @@ -45,6 +55,7 @@ export interface WindowsCursorAssetPayload { export type WindowsCursorEvent = | WindowsCursorSampleEvent | WindowsCursorReadyEvent + | WindowsCursorKeyEvent | WindowsCursorErrorEvent; export interface WindowsNativeRecordingSessionOptions { @@ -53,4 +64,6 @@ export interface WindowsNativeRecordingSessionOptions { sampleIntervalMs: number; sourceId?: string | null; startTimeMs?: number; + /** Live keydowns, forwarded to the keystroke overlay. Absent = not consumed. */ + onKeyEvent?: (event: WindowsCursorKeyEvent) => void; } diff --git a/electron/native/wgc-capture/src/cursor-sampler.cpp b/electron/native/wgc-capture/src/cursor-sampler.cpp index cf4947c556..fb87b5153c 100644 --- a/electron/native/wgc-capture/src/cursor-sampler.cpp +++ b/electron/native/wgc-capture/src/cursor-sampler.cpp @@ -28,6 +28,48 @@ static std::atomic g_leftUpCount{0}; static std::atomic g_stop{false}; static std::mutex g_stdoutMtx; +// ───────────────────────────────────────────────────────────────────────────── +// Global keyboard-hook state: keydowns the on-screen keystroke overlay shows. +// The hook only queues — an LL hook must not block, so the sampling loop drains +// the queue and writes the JSONL lines. ponytail: the repeat filter is one +// atomic, so alternating two held keys re-queues a repeat of the first one; +// a per-key down set is the upgrade if that ever matters. +// ───────────────────────────────────────────────────────────────────────────── +static HHOOK g_keyHook = nullptr; +static std::mutex g_keyMtx; +struct KeyPress { + DWORD vk; + DWORD modifiers; // Modifier mask captured at event time (see modifierMask). +}; +static std::vector g_keyQueue; +static std::atomic g_lastKeyDown{0}; + +// The generic modifier virtual keys are what the renderer's label layer knows. +// Left/right variants (VK_LSHIFT..VK_RMENU) are normalised here, at the hook, +// so both the repeat filter and every downstream consumer see the generic code. +static DWORD normalizeModifierVk(DWORD vk) { + switch (vk) { + case 0xA0: case 0xA1: return VK_SHIFT; // VK_LSHIFT / VK_RSHIFT + case 0xA2: case 0xA3: return VK_CONTROL; // VK_LCONTROL / VK_RCONTROL + case 0xA4: case 0xA5: return VK_MENU; // VK_LMENU / VK_RMENU + default: return vk; + } +} + +// Bit mask of modifiers currently held, read IN THE HOOK at event time. The +// sampling loop drains the queue later, when the physical state may have +// changed; inferring it there would mislabel a combo whose modifier was already +// released. Bit values match the renderer's MODIFIER mask in keystrokeLabels.ts. +static DWORD modifierMask() { + DWORD mask = 0; + if (GetAsyncKeyState(VK_SHIFT) & 0x8000) mask |= 1; // Shift + if (GetAsyncKeyState(VK_CONTROL) & 0x8000) mask |= 2; // Ctrl + if (GetAsyncKeyState(VK_MENU) & 0x8000) mask |= 4; // Alt + if (GetAsyncKeyState(VK_LWIN) & 0x8000) mask |= 8; // Win (either side) + if (GetAsyncKeyState(VK_RWIN) & 0x8000) mask |= 8; + return mask; +} + static LRESULT CALLBACK LowLevelMouseProc(int nCode, WPARAM wParam, LPARAM lParam) { if (nCode >= 0) { if (wParam == WM_LBUTTONDOWN) g_leftDownCount.fetch_add(1, std::memory_order_relaxed); @@ -36,6 +78,29 @@ static LRESULT CALLBACK LowLevelMouseProc(int nCode, WPARAM wParam, LPARAM lPara return CallNextHookEx(g_mouseHook, nCode, wParam, lParam); } +static LRESULT CALLBACK LowLevelKeyboardProc(int nCode, WPARAM wParam, LPARAM lParam) { + if (nCode >= 0) { + const KBDLLHOOKSTRUCT* info = reinterpret_cast(lParam); + if (info && (wParam == WM_KEYDOWN || wParam == WM_SYSKEYDOWN)) { + const DWORD vk = normalizeModifierVk(info->vkCode); + // The LL hook keeps firing while a key is held; the overlay wants one + // badge per press, so only a change of key passes the filter. The + // normalised code is used so L/R variants of a modifier do not count + // as alternating distinct keys. + if (g_lastKeyDown.exchange(vk, std::memory_order_relaxed) != vk) { + // Capture the modifier state here, at event time, not when the + // sampling loop drains the queue (#1633). + const DWORD mods = modifierMask(); + std::lock_guard lock(g_keyMtx); + g_keyQueue.push_back({vk, mods}); + } + } else if (info && (wParam == WM_KEYUP || wParam == WM_SYSKEYUP)) { + g_lastKeyDown.store(0, std::memory_order_relaxed); + } + } + return CallNextHookEx(g_keyHook, nCode, wParam, lParam); +} + // ───────────────────────────────────────────────────────────────────────────── // Utilities // ───────────────────────────────────────────────────────────────────────────── @@ -322,6 +387,24 @@ static void runSamplingLoop(int intervalMs, HWND targetWindow, const CLSID& pngC const int downCount = g_leftDownCount.exchange(0, std::memory_order_relaxed); const int upCount = g_leftUpCount.exchange(0, std::memory_order_relaxed); + // Drain the keyboard hook's queue: one JSONL line per keydown, numeric + // fields only, so nothing needs escaping. Keyups carry no information the + // overlay shows, so they are never emitted. `modifiers` was captured in + // the hook at event time so a released modifier cannot relabel the combo. + std::vector keys; + { + std::lock_guard lock(g_keyMtx); + keys.swap(g_keyQueue); + } + for (const KeyPress& key : keys) { + char keyBuf[128]; + std::snprintf(keyBuf, sizeof(keyBuf), + "{\"type\":\"key\",\"timestampMs\":%" PRId64 ",\"vk\":%lu,\"modifiers\":%lu}", + nowMs(), static_cast(key.vk), + static_cast(key.modifiers)); + writeJsonLine(keyBuf); + } + CURSORINFO ci{}; ci.cbSize = sizeof(ci); if (!GetCursorInfo(&ci)) { @@ -474,6 +557,13 @@ int main(int argc, char* argv[]) { return 1; } + // The keyboard hook feeds the keystroke overlay only, so its failure costs + // nothing the recording needs — warn and keep going rather than aborting. + g_keyHook = SetWindowsHookEx(WH_KEYBOARD_LL, LowLevelKeyboardProc, GetModuleHandle(nullptr), 0); + if (!g_keyHook) { + std::cerr << "SetWindowsHookEx(WH_KEYBOARD_LL) failed; no keystroke overlay" << std::endl; + } + // Prime GetAsyncKeyState so the first poll doesn't return stale "since-last-call" bits GetAsyncKeyState(VK_LBUTTON); @@ -499,6 +589,7 @@ int main(int argc, char* argv[]) { g_stop.store(true, std::memory_order_relaxed); if (sampler.joinable()) sampler.join(); UnhookWindowsHookEx(g_mouseHook); + if (g_keyHook) UnhookWindowsHookEx(g_keyHook); Gdiplus::GdiplusShutdown(gdipToken); return 0; } diff --git a/electron/preload.ts b/electron/preload.ts index 3f5716036d..7f23b6919b 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -621,6 +621,11 @@ contextBridge.exposeInMainWorld("electronAPI", { notifyCountdownOverlayCleared: (runId: number) => { ipcRenderer.send("countdown-overlay-cleared", runId); }, + onKeystrokeKey: (callback: (vk: number, modifiers: number) => void) => { + const listener = (_event: unknown, vk: number, modifiers: number) => callback(vk, modifiers); + ipcRenderer.on("keystroke-overlay-key", listener); + return () => ipcRenderer.removeListener("keystroke-overlay-key", listener); + }, onRequestSaveBeforeClose: (callback: () => Promise | boolean) => { const listener = async () => { try { diff --git a/electron/windows.ts b/electron/windows.ts index 386e256cce..6c9589f878 100644 --- a/electron/windows.ts +++ b/electron/windows.ts @@ -852,6 +852,51 @@ export function createCountdownOverlayWindow(): BrowserWindow { return win; } +/** + * Transparent click-through window that shows the keys pressed while a take is live + * (`keystrokeOverlay.ts` owns the lifecycle). Unlike the countdown overlay this one is + * visible DURING the capture, so content protection is what keeps it out of the video — + * same trade as the HUD: on Windows before 11 22H2 the protection has to go, and the + * badges would land in the recording (see applyContentProtection). + */ +export function createKeystrokeOverlayWindow(): BrowserWindow { + const win = new BrowserWindow({ + width: 480, + height: 200, + frame: false, + resizable: false, + alwaysOnTop: true, + skipTaskbar: true, + focusable: false, + transparent: true, + backgroundColor: "#00000000", + hasShadow: false, + show: false, + webPreferences: { + preload: path.join(__dirname, "preload.mjs"), + additionalArguments: [ASSET_BASE_URL_ARG], + nodeIntegration: false, + contextIsolation: true, + backgroundThrottling: false, + }, + }); + + win.setIgnoreMouseEvents(true); + applyContentProtection(win, "keystroke overlay", { clickThrough: true }); + keepAlwaysOnTop(win); + + if (VITE_DEV_SERVER_URL) { + win.loadURL(VITE_DEV_SERVER_URL + "?windowType=keystroke-overlay"); + } else { + win.loadFile(path.join(RENDERER_DIST, "index.html"), { + query: { windowType: "keystroke-overlay" }, + }); + } + + markSheetless(win); + return win; +} + /** * The macOS permissions window: first run, the app menu, and wherever a missing permission * would otherwise stop a recording. An ordinary opaque window on purpose -- it has to sit diff --git a/src/App.tsx b/src/App.tsx index c326d3b835..4b579ee59c 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -7,6 +7,7 @@ installBrowserShims(); import { CountdownOverlay } from "./components/launch/CountdownOverlay.tsx"; import { FloatingCameraPreview } from "./components/launch/FloatingCameraPreview"; +import { KeystrokeOverlay } from "./components/launch/KeystrokeOverlay.tsx"; import { LaunchWindow } from "./components/launch/LaunchWindow"; import { NotesWindow } from "./components/launch/NotesWindow.tsx"; import { SourceSelector } from "./components/launch/SourceSelector"; @@ -92,6 +93,8 @@ export default function App() { return ; case "countdown-overlay": return ; + case "keystroke-overlay": + return ; case "permissions": return ; case "cli-export": diff --git a/src/components/launch/KeystrokeOverlay.test.tsx b/src/components/launch/KeystrokeOverlay.test.tsx new file mode 100644 index 0000000000..99a2a684bd --- /dev/null +++ b/src/components/launch/KeystrokeOverlay.test.tsx @@ -0,0 +1,85 @@ +// @vitest-environment jsdom +import { act, render } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { KeystrokeOverlay } from "./KeystrokeOverlay"; + +type KeyListener = (vk: number, modifiers?: number) => void; + +describe("KeystrokeOverlay", () => { + let pushKey: KeyListener = () => undefined; + + beforeEach(() => { + window.electronAPI = { + ...window.electronAPI, + onKeystrokeKey: (listener: KeyListener) => { + pushKey = listener; + return () => undefined; + }, + } as typeof window.electronAPI; + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it("shows a modifier followed by a regular key as one combined badge", () => { + const { container } = render(); + + // The sampler sends the modifier press, then the letter with the modifier + // bit held at event time. The overlay must compose "Ctrl+C", not two badges. + act(() => pushKey(0x11, 2)); // Ctrl down (modifier bit for itself) + act(() => pushKey(0x43, 2)); // C down while Ctrl held + + expect(container.textContent).toContain("Ctrl+C"); + }); + + it("composes from the event-time modifier mask alone", () => { + const { container } = render(); + + // No modifier keydown was ever delivered, but C arrives with Ctrl in its + // mask (e.g. the first key seen mid-take). The mask alone must still yield + // a combo rather than a bare "C". + act(() => pushKey(0x43, 2)); + expect(container.textContent).toContain("Ctrl+C"); + }); + + it("leaves an unmodified key as a bare badge", () => { + const { container } = render(); + act(() => pushKey(0x43, 0)); + expect(container.textContent).toBe("C"); + }); + + it("an explicit zero mask overrides a locally accumulated modifier", () => { + const { container } = render(); + + // A stream that omits the mask (older helper) still composes via local + // accumulation. But once an explicit mask arrives it is authoritative: + // Ctrl down with no mask, then C with mask 0 must read as a bare "C". + act(() => pushKey(0x11)); // Ctrl down, mask omitted + act(() => pushKey(0x43, 0)); // C down, explicit no-modifier mask + + expect(container.textContent).toContain("C"); + expect(container.textContent).not.toContain("Ctrl+C"); + }); + + it("an omitted mask falls back to locally accumulated modifiers", () => { + const { container } = render(); + act(() => pushKey(0x11)); // Ctrl down, mask omitted + act(() => pushKey(0x43)); // C down, mask omitted + expect(container.textContent).toContain("Ctrl+C"); + }); + + it("keeps a standalone modifier visible and clears the combo after the next key", () => { + const { container } = render(); + act(() => pushKey(0x11, 2)); // Ctrl alone + expect(container.textContent).toContain("Ctrl"); + + act(() => pushKey(0x43, 2)); // Ctrl+C + expect(container.textContent).toContain("Ctrl+C"); + + // A later key with no modifier must not inherit the released Ctrl. + act(() => pushKey(0x44, 0)); // D, no modifiers + expect(container.textContent).toContain("D"); + expect(container.textContent).not.toContain("Ctrl+D"); + }); +}); diff --git a/src/components/launch/KeystrokeOverlay.tsx b/src/components/launch/KeystrokeOverlay.tsx new file mode 100644 index 0000000000..7feb372405 --- /dev/null +++ b/src/components/launch/KeystrokeOverlay.tsx @@ -0,0 +1,95 @@ +import { useEffect, useRef, useState } from "react"; +import { badgeLabel, VK_CONTROL, VK_LWIN, VK_MENU, VK_RWIN, VK_SHIFT } from "@/lib/keystrokeLabels"; + +/** How long a badge stays up before it fades out (matched by the CSS animation). */ +export const BADGE_VISIBLE_MS = 1_300; + +interface KeystrokeBadge { + id: number; + label: string; +} + +/** Modifier virtual keys, as a set the badge labels compose from. */ +const MODIFIER_KEYS = new Set([VK_SHIFT, VK_CONTROL, VK_MENU, VK_LWIN, VK_RWIN]); + +/** Modifier bit mask from the sampler: 1=Shift 2=Ctrl 4=Alt 8=Win. */ +const MASK_TO_VK: readonly [number, number][] = [ + [1, VK_SHIFT], + [2, VK_CONTROL], + [4, VK_MENU], + [8, VK_LWIN], +]; + +/** + * The keystroke overlay: badges for the keys pressed while a take is live, bottom-centre + * of the display being recorded. Keydowns arrive live from the Windows sampler's + * WH_KEYBOARD_LL hook (`keystroke-overlay-key` via preload); nothing is shown when no + * take is running, because the window itself only exists then. + */ +export function KeystrokeOverlay() { + const [badges, setBadges] = useState([]); + const heldModifiers = useRef>(new Set()); + const nextBadgeId = useRef(1); + + useEffect(() => { + const timers: ReturnType[] = []; + + const unsubscribe = window.electronAPI.onKeystrokeKey((vk, modifiers) => { + // The sampler captures the modifier state at the key's event time and sends + // it with every keydown. An explicit mask -- including 0 -- is authoritative: + // Ctrl down, Ctrl up, then C arrives with mask 0 and must read as "C", not a + // stale "Ctrl+C". Only a stream that omits the mask (an older helper) falls + // back to locally accumulated modifier presses. + const current = new Set(); + if (modifiers === undefined) { + for (const modVk of heldModifiers.current) current.add(modVk); + } else { + for (const [bit, modVk] of MASK_TO_VK) { + if (modifiers & bit) current.add(modVk); + } + } + const isModifier = MODIFIER_KEYS.has(vk); + if (isModifier) { + // Track standalone modifier presses for the no-mask fallback only. + heldModifiers.current.add(vk); + current.add(vk); + } + // Compose from the current set BEFORE a non-modifier releases it. + const label = badgeLabel(vk, current); + if (!isModifier) { + heldModifiers.current.clear(); + } + if (!label) return; + + const id = nextBadgeId.current++; + setBadges((currentBadges) => [...currentBadges.slice(-4), { id, label }]); + timers.push( + setTimeout(() => { + setBadges((currentBadges) => currentBadges.filter((badge) => badge.id !== id)); + }, BADGE_VISIBLE_MS), + ); + }); + + return () => { + unsubscribe(); + for (const timer of timers) clearTimeout(timer); + heldModifiers.current.clear(); + }; + }, []); + + if (badges.length === 0) { + return null; + } + + return ( +
+
+ {badges.map((badge) => ( +
+ {badge.label} +
+ ))} +
+
+ ); +} diff --git a/src/index.css b/src/index.css index 2a92cfc263..daf62254ac 100644 --- a/src/index.css +++ b/src/index.css @@ -397,3 +397,30 @@ html [data-sonner-toaster] [data-sonner-toast][data-styled="true"] [data-close-b background: color-mix(in oklab, var(--fg) 10%, transparent); color: var(--fg); } +/* Keystroke overlay badges (`KeystrokeOverlay.tsx`): dark pills with the key combo, + bottom-centre of the recorded display. The fade lives on the element itself, one + animation per badge, so the stack never needs transition juggling. */ +@keyframes keystroke-badge-fade { + 0%, + 80% { + opacity: 1; + } + 100% { + opacity: 0; + } +} + +.keystroke-badge { + display: inline-flex; + align-items: center; + padding: 8px 16px; + border-radius: 10px; + background: rgba(0, 0, 0, 0.72); + border: 1px solid rgba(255, 255, 255, 0.18); + color: rgba(255, 255, 255, 0.92); + font-size: 16px; + font-weight: 600; + letter-spacing: 0.02em; + box-shadow: 0 6px 24px rgba(0, 0, 0, 0.45); + animation: keystroke-badge-fade 1.3s ease-out 1 both; +} diff --git a/src/lib/keystrokeLabels.test.ts b/src/lib/keystrokeLabels.test.ts new file mode 100644 index 0000000000..d570e00e06 --- /dev/null +++ b/src/lib/keystrokeLabels.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it } from "vitest"; +import { badgeLabel, baseKeyLabel, isModifierVk, VK_CONTROL, VK_SHIFT } from "./keystrokeLabels"; + +describe("keystrokeLabels", () => { + it("names letters, digits and function keys off the vk code ranges", () => { + expect(baseKeyLabel(0x41)).toBe("A"); + expect(baseKeyLabel(0x31)).toBe("1"); + expect(baseKeyLabel(0x70)).toBe("F1"); + expect(baseKeyLabel(0x87)).toBe("F24"); + expect(baseKeyLabel(0x64)).toBe("Num4"); + }); + + it("names the named and OEM keys the overlay is likely to see", () => { + expect(baseKeyLabel(0x0d)).toBe("Enter"); + expect(baseKeyLabel(0x1b)).toBe("Esc"); + expect(baseKeyLabel(0x20)).toBe("Space"); + expect(baseKeyLabel(0xba)).toBe(";"); + expect(baseKeyLabel(0xdb)).toBe("["); + }); + + it("returns null for keys the overlay hides, and names every modifier", () => { + expect(baseKeyLabel(0xa6)).toBeNull(); // media key, no label + expect(baseKeyLabel(0x10)).toBe("Shift"); + expect(baseKeyLabel(0x11)).toBe("Ctrl"); + expect(baseKeyLabel(0x12)).toBe("Alt"); + expect(baseKeyLabel(0x5b)).toBe("Win"); + }); + + it("composes held modifiers in Ctrl, Alt, Shift, Win order", () => { + expect(badgeLabel(0x43, new Set([VK_CONTROL, VK_SHIFT]))).toBe("Ctrl+Shift+C"); + expect(badgeLabel(0x2e, new Set([VK_CONTROL]))).toBe("Ctrl+Del"); + expect(badgeLabel(0x43, new Set())).toBe("C"); + }); + + it("shows modifiers alone and never double-prefixes one", () => { + expect(badgeLabel(0x10, new Set([VK_SHIFT]))).toBe("Shift"); + expect(badgeLabel(0x11, new Set())).toBe("Ctrl"); + expect(isModifierVk(0x11)).toBe(true); + expect(isModifierVk(0x41)).toBe(false); + }); +}); diff --git a/src/lib/keystrokeLabels.ts b/src/lib/keystrokeLabels.ts new file mode 100644 index 0000000000..794d303217 --- /dev/null +++ b/src/lib/keystrokeLabels.ts @@ -0,0 +1,91 @@ +// Win32 virtual-key codes → display labels for the keystroke overlay. +// +// The C++ sampler reports raw `vkCode`s (`cursor-sampler.cpp`); naming and modifier +// composition live here, in the renderer. Modifiers compose a combo prefix on the next +// non-modifier key, keyviz-style; a modifier press on its own shows as a badge too, so +// the user sees Ctrl happen even if nothing follows it. + +export const VK_SHIFT = 0x10; +export const VK_CONTROL = 0x11; +export const VK_MENU = 0x12; +export const VK_LWIN = 0x5b; +export const VK_RWIN = 0x5c; + +const MODIFIER_VKS: readonly number[] = [VK_SHIFT, VK_CONTROL, VK_MENU, VK_LWIN, VK_RWIN]; + +/** Combo prefix order, the one keyviz uses: Ctrl · Alt · Shift · Win. */ +const MODIFIER_ORDER: readonly [number, string][] = [ + [VK_CONTROL, "Ctrl"], + [VK_MENU, "Alt"], + [VK_SHIFT, "Shift"], + [VK_LWIN, "Win"], + [VK_RWIN, "Win"], +]; + +const NAMED_KEYS: Record = { + 0x08: "Backspace", + 0x09: "Tab", + 0x0d: "Enter", + 0x13: "Pause", + 0x14: "CapsLock", + 0x1b: "Esc", + 0x20: "Space", + 0x21: "PageUp", + 0x22: "PageDown", + 0x23: "End", + 0x24: "Home", + 0x25: "←", + 0x26: "↑", + 0x27: "→", + 0x28: "↓", + 0x2d: "Ins", + 0x2e: "Del", +}; + +const OEM_KEYS: Record = { + 0xba: ";", + 0xbb: "=", + 0xbc: ",", + 0xbd: "-", + 0xbe: ".", + 0xbf: "/", + 0xc0: "`", + 0xdb: "[", + 0xdc: "\\", + 0xdd: "]", + 0xde: "'", +}; + +/** Labels for one virtual key, without modifiers; `null` for keys the overlay hides. */ +export function baseKeyLabel(vk: number): string | null { + if (vk >= 0x30 && vk <= 0x39) return String.fromCharCode(vk); + if (vk >= 0x41 && vk <= 0x5a) return String.fromCharCode(vk); + if (vk >= 0x60 && vk <= 0x69) return `Num${vk - 0x60}`; + if (vk >= 0x70 && vk <= 0x87) return `F${vk - 0x70 + 1}`; + if (vk === VK_LWIN || vk === VK_RWIN) return "Win"; + if (vk === VK_SHIFT) return "Shift"; + if (vk === VK_CONTROL) return "Ctrl"; + if (vk === VK_MENU) return "Alt"; + return NAMED_KEYS[vk] ?? OEM_KEYS[vk] ?? null; +} + +/** `true` when the key is one of the composing modifiers the overlay tracks. */ +export function isModifierVk(vk: number): boolean { + return MODIFIER_VKS.includes(vk); +} + +/** + * The badge label for a keydown: the held modifiers as a `Ctrl+Alt+Delete`-style combo, + * or the bare key. `heldModifiers` carries the modifier keys still down, as tracked by + * the overlay from the same event stream. + */ +export function badgeLabel(vk: number, heldModifiers: ReadonlySet): string | null { + const base = baseKeyLabel(vk); + if (base === null) return null; + if (isModifierVk(vk)) return base; + + const prefix = MODIFIER_ORDER.filter(([modVk]) => heldModifiers.has(modVk)).map( + ([, label]) => label, + ); + return [...prefix, base].join("+"); +} diff --git a/src/main.tsx b/src/main.tsx index 939360e624..366eab1516 100644 --- a/src/main.tsx +++ b/src/main.tsx @@ -21,7 +21,8 @@ if ( showNotes || windowType === "hud-overlay" || windowType === "source-selector" || - windowType === "countdown-overlay" + windowType === "countdown-overlay" || + windowType === "keystroke-overlay" ) { document.body.style.background = "transparent"; document.documentElement.style.background = "transparent";