diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 387bed2b96..dd0f477a66 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -52,10 +52,12 @@ jobs: uses: ./.github/actions/setup # STT is the bundled whisper-stt-server (whisper.cpp with native DTW token - # timestamps); no VAD model is fetched here. The binary is built by - # build-whisper-stt.yml and staged below — without that step the installer - # ships without speech-to-text. See - # technical-documentation/architecture/transcription-and-captions.md. + # timestamps); no model is fetched here. The binary is built by + # build-whisper-stt.yml and staged below, from the run built from this + # commit's helper sources — without that step the installer ships without + # speech-to-text. See + # technical-documentation/architecture/transcription-and-captions.md, and + # technical-documentation/engineering/release-and-secrets.md for the run. - name: Stage whisper-stt binaries shell: bash env: diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index e8a87f80fa..8565f14e91 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -76,6 +76,17 @@ jobs: - name: Check media budget working-directory: website run: npm run check:media + # The demo loops are served from R2 (media.getopenscreen.com), not git, so + # nothing above would notice a loop a page names that was never uploaded, + # or a cut folder holding the wrong files. media/loops.json records what + # scripts/media/publish-loops.mjs uploaded; this holds the code to it and + # asks the CDN for every recorded file, paced under the zone's rate limit. + - name: Check demo loops + working-directory: website + run: npm run check:loops -- --live + - name: Check responsive backgrounds + working-directory: website + run: npm run check:backgrounds -- --live # The recreation's data is generated from the app's own source and a # vendored project document. It is committed, so it can go stale in either # direction — a locale key moves, a timeline constant changes — and the @@ -137,4 +148,4 @@ jobs: id-token: write steps: - id: deployment - uses: actions/deploy-pages@cd2ce8fcbc39b97be8ca5fce6e763baed58fa128 # v5.0.0 \ No newline at end of file + uses: actions/deploy-pages@cd2ce8fcbc39b97be8ca5fce6e763baed58fa128 # v5.0.0 diff --git a/.gitignore b/.gitignore index 24ad425ebc..8b2682b6ec 100644 --- a/.gitignore +++ b/.gitignore @@ -95,6 +95,9 @@ result-* /tools/stt-eval/whispercpp-dtw-poc/fixtures/*.wav /tools/stt-eval/whispercpp-dtw-poc/results/ +# Word-timing harness (tools/stt-eval/word-timing): TTS corpus and results. +/tools/stt-eval/word-timing/data/ + opencode.json opencode.json diff --git a/README.md b/README.md index 872f991443..23cb6784de 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@

A free, open-source desktop app for recording your screen and turning the result into polished product demos and walkthroughs.

- Editing a recording in OpenScreen: wallpaper and video effects, an AI-assisted cut driven from the chat, then export + Editing a recording in OpenScreen: one click restyles the whole video, deleting a sentence in the transcript cuts it from the video, and the editor's agent adds a zoom on every click

diff --git a/THIRD-PARTY-NOTICES.md b/THIRD-PARTY-NOTICES.md index dcbc432cff..b2611d75b6 100644 --- a/THIRD-PARTY-NOTICES.md +++ b/THIRD-PARTY-NOTICES.md @@ -1,12 +1,14 @@ # Third-party notices OpenScreen is MIT licensed (see [LICENSE](LICENSE)). The installers additionally -bundle the pre-built native components below. This file ships inside the +bundle the third-party components below. This file ships inside the application resources and satisfies the attribution and source-offer obligations that come with them. npm dependencies are not listed here: they are resolved from `package.json` and -distributed by their own registries, not redistributed inside our binaries. +distributed by their own registries, not redistributed inside our binaries. The +exception is code bundled into the renderer whose licence asks for its notice in +the distributed form (js-aruco2 below): the bundler strips the source headers. --- @@ -38,6 +40,23 @@ distributed by their own registries, not redistributed inside our binaries. replace them with your own build of the same FFmpeg version by overwriting the DLLs in `resources/electron/native/bin/win32-*/`. +## web-demuxer — WebAssembly demuxer + +- **Component**: `wasm/web-demuxer.wasm`, inside the application bundle. +- **Used by**: the editor's streaming decoder, audio waveforms and caption audio + extraction, which demux recordings in the renderer with it. +- **License**: MIT for web-demuxer itself — Copyright (c) 2024 ForeverSc, under the + same permission notice as OpenScreen's own [LICENSE](LICENSE). The module also + contains FFmpeg code compiled to WebAssembly, under the **GNU Lesser General + Public License v2.1 or later**, as web-demuxer's README states. +- **Upstream**: the npm package `web-demuxer` 4.0.0, + . The shipped file is byte-identical + to that release's `dist/wasm-files/web-demuxer.wasm`. +- **Corresponding source**: the web-demuxer repository at the 4.0.0 release, whose + `lib/` directory holds the FFmpeg-derived sources and their build scripts. +- **Replacing it**: the module is a separate file loaded at runtime, so it can be + replaced with your own build of the same version. + ## whisper.cpp and ggml - **Components**: `whisper-stt-server` and its ggml backend sidecars, under @@ -147,6 +166,132 @@ distributed by their own registries, not redistributed inside our binaries. - IBM Plex Mono 2.3 — Copyright © 2017 IBM Corp. `ofl/ibmplexmono/` of at `23e54b51dd`. +## Recordly — helper code + +- **Components**: parts of the zoom, motion-smoothing and custom-cursor code, and a + few lines elsewhere, the native capture helpers included. +- **License**: MIT — Copyright (c) 2026 webadderall, under the same permission + notice as OpenScreen's own [LICENSE](LICENSE). Published by Recordly under MIT, + before its relicensing in March 2026. + +## js-aruco2 — ArUco marker detection + +- **Components**: `src/aruco.js` and `src/cv.js` of the npm package `js-aruco2` + 2.0.0, compiled into the renderer bundle (`dist/assets/*.js`). +- **Used by**: the camera calibration dialog, which finds the four printed corner + markers in a camera still. +- **License**: MIT — Copyright (c) 2020 Damiano Falcioni, Copyright (c) 2011 Juan + Mellado, under the same permission notice as OpenScreen's own + [LICENSE](LICENSE). +- **Upstream**: , release 2.0.0. + +## OpenCV — ArUco 4×4 dictionary codes + +- **Component**: the marker codes in js-aruco2's + `src/dictionaries/aruco_4x4_1000.js`, compiled into the renderer bundle. They + are taken from OpenCV's `predefined_dictionaries.hpp`; the marker sheet the + editor prints draws markers 0–3 of this dictionary. +- **License**: BSD 3-Clause — Copyright (C) 2013, OpenCV Foundation, all rights + reserved. Third party copyrights are property of their respective owners. + + > Redistribution and use in source and binary forms, with or without + > modification, are permitted provided that the following conditions are met: + > + > - Redistributions of source code must retain the above copyright notice, + > this list of conditions and the following disclaimer. + > - Redistributions in binary form must reproduce the above copyright notice, + > this list of conditions and the following disclaimer in the documentation + > and/or other materials provided with the distribution. + > - Neither the names of the copyright holders nor the names of the + > contributors may be used to endorse or promote products derived from this + > software without specific prior written permission. + > + > This software is provided by the copyright holders and contributors "as is" + > and any express or implied warranties, including, but not limited to, the + > implied warranties of merchantability and fitness for a particular purpose + > are disclaimed. In no event shall copyright holders or contributors be liable + > for any direct, indirect, incidental, special, exemplary, or consequential + > damages (including, but not limited to, procurement of substitute goods or + > services; loss of use, data, or profits; or business interruption) however + > caused and on any theory of liability, whether in contract, strict liability, + > or tort (including negligence or otherwise) arising in any way out of the use + > of this software, even if advised of the possibility of such damage. + +## Rust crates — compiled into the compositor addon + +- **Component**: `compositor_view.node`, under + `resources/electron/native/bin/-/`. +- **What is listed**: every third-party crate in the addon's dependency graph in + `crates/Cargo.lock`, across all platforms. Build-time crates (procedural macros, + build scripts) are included even though none of their code ships. +- **Licenses**: as each crate publishes them on crates.io. Where a crate offers a + choice, OpenScreen uses it under MIT or Apache-2.0. Each crate's licence text is + in its source package at the version listed. +- **MIT OR Apache-2.0** (155): android_system_properties 0.1.5, anyhow 1.0.103, + arrayvec 0.7.8, ash 0.38.0+1.3.281, autocfg 1.5.1, bit-set 0.8.0, bit-vec 0.8.0, + bitflags 1.3.2, bitflags 2.13.1, bumpalo 3.20.3, cc 1.2.67, cexpr 0.6.0, cfg-if + 1.0.4, core-foundation 0.9.4, core-foundation-sys 0.8.7, core-graphics-types + 0.1.3, cosmic-text 0.19.0, crc32fast 1.5.0, ctor 0.2.9, document-features + 0.2.12, either 1.16.0, equivalent 1.0.2, fdeflate 0.3.7, find-msvc-tools 0.1.9, + flate2 1.1.9, font-types 0.11.3, font-types 0.12.2, foreign-types 0.5.0, + foreign-types-macros 0.2.4, foreign-types-shared 0.3.1, futures-core 0.3.33, + futures-task 0.3.33, futures-util 0.3.33, glob 0.3.3, gpu-alloc 0.6.2, + gpu-alloc-types 0.3.1, gpu-allocator 0.27.0, gpu-descriptor 0.3.2, + gpu-descriptor-types 0.2.0, hashbrown 0.15.5, hashbrown 0.17.1, heck 0.5.0, + image 0.25.10, indexmap 2.14.0, itertools 0.13.0, itoa 1.0.18, jni-sys 0.3.1, + jni-sys 0.4.1, jni-sys-macros 0.4.1, js-sys 0.3.103, khronos-egl 6.0.0, libc + 0.2.186, linebender_resource_handle 0.1.1, litrs 1.0.0, lock_api 0.4.14, log + 0.4.33, matrixmultiply 0.3.11, memmap2 0.9.11, metal 0.29.0, metal 0.31.0, + minimal-lexical 0.2.1, naga 24.0.0, ndarray 0.16.1, ndarray 0.17.2, ndk-sys + 0.5.0+25.2.9519653, num-complex 0.4.6, num-integer 0.1.47, num-traits 0.2.19, + once_cell 1.21.4, ort 2.0.0-rc.13, ort-sys 2.0.0-rc.13, parking_lot 0.12.5, + parking_lot_core 0.9.12, paste 1.0.15, pin-project-lite 0.2.17, pkg-config + 0.3.33, png 0.18.1, pollster 0.4.0, portable-atomic 1.15.0, portable-atomic-util + 0.2.7, presser 0.3.1, prettyplease 0.2.37, proc-macro2 1.0.106, profiling + 1.0.18, quote 1.0.46, range-alloc 0.1.5, rangemap 1.7.1, rawpointer 0.2.1, + read-fonts 0.37.0, read-fonts 0.41.0, regex 1.13.1, regex-automata 0.4.16, + regex-syntax 0.8.11, renderdoc-sys 1.1.0, roxmltree 0.20.0, rustc-hash 1.1.0, + rustc-hash 2.1.3, rustversion 1.0.23, scopeguard 1.2.0, semver 1.0.28, serde + 1.0.228, serde_core 1.0.228, serde_derive 1.0.228, serde_json 1.0.150, shlex + 1.3.0, shlex 2.0.1, skrifa 0.40.0, skrifa 0.44.0, smallvec 1.15.2, smol_str + 0.3.6, static_assertions 1.1.0, swash 0.2.10, syn 2.0.119, syn 3.0.3, sys-locale + 0.3.2, thiserror 1.0.69, thiserror 2.0.19, thiserror-impl 1.0.69, thiserror-impl + 2.0.19, ttf-parser 0.25.1, unicode-bidi 0.3.18, unicode-script 0.5.8, + unicode-segmentation 1.13.3, unicode-width 0.1.14, unicode-xid 0.2.6, + version_check 0.9.5, wasm-bindgen 0.2.126, wasm-bindgen-futures 0.4.76, + wasm-bindgen-macro 0.2.126, wasm-bindgen-macro-support 0.2.126, + wasm-bindgen-shared 0.2.126, web-sys 0.3.103, wgpu 24.0.5, wgpu-core 24.0.5, + wgpu-hal 24.0.4, wgpu-types 24.0.0, windows 0.58.0, windows_aarch64_gnullvm + 0.52.6, windows_aarch64_msvc 0.52.6, windows_i686_gnu 0.52.6, + windows_i686_gnullvm 0.52.6, windows_i686_msvc 0.52.6, windows_x86_64_gnu + 0.52.6, windows_x86_64_gnullvm 0.52.6, windows_x86_64_msvc 0.52.6, windows-core + 0.58.0, windows-implement 0.58.0, windows-interface 0.58.0, windows-link 0.2.1, + windows-result 0.2.0, windows-strings 0.1.0, windows-sys 0.61.2, windows-targets + 0.52.6, yazi 0.2.1, zeno 0.3.3. +- **MIT** (26): block 0.1.6, cfg_aliases 0.2.2, convert_case 0.6.0, core_maths + 0.1.1, fontconfig-parser 0.5.8, fontdb 0.23.0, harfrust 0.5.2, libm 0.2.16, + malloc_buf 0.0.6, napi 2.16.17, napi-build 2.3.2, napi-derive 2.16.13, + napi-derive-backend 1.0.75, napi-sys 2.4.0, nom 7.1.3, objc 0.2.7, ordered-float + 4.6.0, redox_syscall 0.5.18, simd-adler32 0.3.10, slab 0.4.12, strum 0.26.3, + strum_macros 0.26.4, tracing 0.1.44, tracing-core 0.1.36, xml-rs 0.8.28, zmij + 1.0.23. +- **MIT OR Apache-2.0 OR Zlib** (9): bytemuck 1.25.1, bytemuck_derive 1.11.0, + glow 0.16.0, miniz_oxide 0.8.9, raw-window-handle 0.6.2, tinyvec 1.12.0, + tinyvec_macros 0.1.1, zune-core 0.5.1, zune-jpeg 0.5.15. +- **Apache-2.0** (7): clang-sys 1.8.1, codespan-reporting 0.11.1, gl_generator + 0.14.0, glutin_wgl_sys 0.6.1, khronos_api 3.1.0, spirv 0.3.0+sdk-1.3.268.0, + unicode-linebreak 0.1.5. +- **Unlicense OR MIT** (5): aho-corasick 1.1.4, byteorder-lite 0.1.0, memchr + 2.8.3, termcolor 1.4.1, winapi-util 0.1.11. +- **Zlib** (2): foldhash 0.1.5, slotmap 1.1.1. +- **ISC** (2): libloading 0.8.9, libloading 0.9.0. +- **BSD-3-Clause OR Apache-2.0** (2): moxcms 0.8.1, pxfm 0.1.30. +- **0BSD OR MIT OR Apache-2.0** (1): adler2 2.0.1. +- **BSD-3-Clause** (1): bindgen 0.70.1. +- **CC0-1.0** (1): hexf-parse 0.2.1. +- **Apache-2.0 OR GPL-2.0-only** (1): self_cell 1.3.0. +- **(MIT OR Apache-2.0) AND Unicode-3.0** (1): unicode-ident 1.0.24. + ## OpenScreen native helpers `wgc-capture` (Windows Graphics Capture), the ScreenCaptureKit helper (macOS), diff --git a/crates/Cargo.toml b/crates/Cargo.toml index 3caa7edfd9..a65f9aaef2 100644 --- a/crates/Cargo.toml +++ b/crates/Cargo.toml @@ -68,6 +68,7 @@ features = [ "Win32_Graphics_Direct2D_Common", # D2D1_COLOR_F / PIXEL_FORMAT "Win32_Graphics_DirectWrite", # mise en page + rastérisation du texte "Win32_Graphics_Gdi", # brosses/police GUI de la preview (poc-d3d/src/app.rs) + "Win32_Security", # IDXGIResource1::CreateSharedHandle (shared_frames.rs) "Win32_System_Performance", # QueryPerformanceCounter / Frequency (§10) "Win32_System_LibraryLoader", # GetModuleHandleW (hInstance) "Win32_UI_WindowsAndMessaging", # fenêtres (harnais live + GUI du POC) diff --git a/crates/compositor-view-napi/src/lib.rs b/crates/compositor-view-napi/src/lib.rs index 8701c8bfef..5a1ded0e11 100644 --- a/crates/compositor-view-napi/src/lib.rs +++ b/crates/compositor-view-napi/src/lib.rs @@ -11,10 +11,11 @@ use napi::{Env, JsFunction, Task}; use napi_derive::napi; use openscreen_compositor::compositor::{live_params_from_scene, Compositor}; use openscreen_compositor::d3d::{Backend, Gpu}; +use openscreen_compositor::frame_geometry::FootageQuad; use openscreen_compositor::gif_export::{GifExportParams, GifStats}; use openscreen_compositor::gif_export_control::{GifExportCancelled, GifExportControl}; use openscreen_compositor::live::{LiveView, PausedPreviews}; -use openscreen_compositor::scene::Scene; +use openscreen_compositor::scene::{Scene, SceneClipCamera}; use openscreen_compositor::{config, pipeline}; use std::collections::HashMap; use std::path::PathBuf; @@ -197,6 +198,16 @@ pub struct FramePacket { /// R,G,B,A tightly-packed, `width * height * 4` octets — ce que `putImageData` / /// `ImageData` attendent côté JS (canvas 2D, format natif RGBA8). pub data: Buffer, + /// Le métrage dans cette image : ses coins TL, TR, BR, BL (x, y en fractions de l'image, + /// huit nombres), là où l'éditeur pose le gimbal d'un flou. Absent avant toute composition. + pub footage: Option>, + /// Le métrage passe de ses coins à l'image par leur homographie (caméra réelle), pas par + /// leur interpolation bilinéaire. + pub footage_projective: bool, + /// Clip actif de la scène quand la frame a été composée (`FramePosition`). + pub clip_index: u32, + /// Temps source de la frame écran composée, en secondes. + pub source_time_sec: f64, } /// Renvoie la dernière frame readback du thread de rendu SI elle est plus récente que @@ -236,17 +247,95 @@ pub fn read_frame(id: i32, since_gen: f64) -> Result> { v.latest_frame_since(since_gen.max(0.0) as u64) } }; - Ok(slot.map(|(gen, w, h, pixels)| { + Ok(slot.map(|(gen, w, h, pixels, footage, position)| { debug_assert_eq!(pixels.len(), (w as usize) * (h as usize) * 4); FramePacket { gen: gen as f64, width: w, height: h, data: Buffer::from(pixels), + footage: footage_corners(footage), + footage_projective: footage.is_some_and(|q| q.projective), + clip_index: position.clip_index, + source_time_sec: position.source_time_sec, } })) } +/// Les coins du métrage (TL, TR, BR, BL) aplatis en huit nombres, tels que JS les lit. +fn footage_corners(footage: Option) -> Option> { + footage.map(|q| q.corners.iter().flatten().map(|&v| v as f64).collect()) +} + +/// Une frame de preview posée dans une texture partagée : de quoi l'importer côté GPU +/// (`sharedTexture.importSharedTexture`) au lieu d'en recevoir les pixels. +#[napi(object)] +pub struct SharedFramePacket { + pub gen: f64, + /// Case de l'anneau qui porte la frame, à rendre avec `gen` par `release_shared_frame` + /// quand Chromium l'a relâchée. + pub slot: u32, + /// Handle NT de la texture, 8 octets little-endian : la forme qu'attend Electron + /// (`SharedTextureHandle.ntHandle`). Valable dans ce processus seulement. + pub handle: Buffer, + pub width: u32, + pub height: u32, + pub footage: Option>, + pub footage_projective: bool, + pub clip_index: u32, + pub source_time_sec: f64, +} + +/// Livrer les frames de la vue par textures partagées plutôt que par `read_frame`. Rend +/// `false` quand la machine ne le peut pas (hors Windows, backend logiciel) : la vue reste +/// alors au readback. +#[napi] +pub fn set_shared_frames(id: i32, enabled: bool) -> bool { + registry() + .lock() + .unwrap() + .get(&id) + .is_some_and(|v| v.set_shared_frames(enabled)) +} + +/// Le pendant de `read_frame` pour une vue en textures partagées : la dernière frame posée +/// dans l'anneau, si elle est plus récente que `since_gen`. Sa case reste tenue jusqu'à +/// `release_shared_frame`. `Ok(None)` aussi quand la vue relit en RAM. +#[napi] +pub fn read_shared_frame(id: i32, since_gen: f64) -> Result> { + let frame = match registry().lock().unwrap().get(&id) { + None => return Ok(None), + Some(v) => { + // Même relais que `read_frame` : sans lui, un thread de rendu mort ne se verrait + // que par un canvas noir. + if let Some(fatal) = v.fatal_error() { + return Err(Error::from_reason(fatal)); + } + v.take_shared_frame(since_gen.max(0.0) as u64) + } + }; + Ok(frame.map(|f| SharedFramePacket { + gen: f.gen as f64, + slot: f.slot, + handle: f.handle.to_le_bytes().to_vec().into(), + width: f.width, + height: f.height, + footage: footage_corners(f.footage), + footage_projective: f.footage.is_some_and(|q| q.projective), + clip_index: f.position.clip_index, + source_time_sec: f.position.source_time_sec, + })) +} + +/// Chromium a relâché la frame `gen` de la case `slot` (`allReferencesReleased` côté +/// Electron) : le thread de rendu peut y réécrire. Sans effet sur une vue détruite. +#[napi] +pub fn release_shared_frame(id: i32, slot: u32, gen: f64) { + if let Some(v) = registry().lock().unwrap().get(&id) { + v.release_shared_frame(slot, gen.max(0.0) as u64); + } +} + /// Param live (inspector). Le type de valeur route vers le bon setter : /// bool = switch (webcamMirror…), number = slider (shadow/roundness/motionBlur/backgroundBlur), /// string = sélection (backgroundColor "#rrggbb"). @@ -279,6 +368,9 @@ pub fn present_time(id: i32, seconds: f64) { /// Remplace les sources du clip actif sans recréer la vue ni son thread de rendu. L'identité /// timeline et le playhead source sont atomiques avec le switch : deux clips partageant les /// mêmes fichiers restent distincts, et les deux décodeurs ouvrent directement la bonne frame. +/// +/// `additional_cameras`: cameras 2-4 of the clip (index k-1 = camera k, an empty `path` = no +/// camera in that slot). Absent = none; the view decodes only those its layout regions show. #[napi] pub fn set_active_clip( id: i32, @@ -287,18 +379,33 @@ pub fn set_active_clip( webcam_offset_sec: f64, clip_index: u32, source_time_sec: f64, + additional_cameras: Option>, ) { if let Some(v) = registry().lock().unwrap().get(&id) { + let cameras = additional_cameras + .unwrap_or_default() + .into_iter() + .map(|c| SceneClipCamera { path: c.path, offset_sec: c.offset_sec }) + .collect(); v.set_active_clip( &screen_path, &webcam_path, webcam_offset_sec, + cameras, clip_index as usize, source_time_sec, ); } } +/// An additional camera of a clip (= TS `CompositorClipCamera`). +#[napi(object)] +pub struct ClipCameraInput { + pub path: String, + /// Camera source time = screen source time - this. + pub offset_sec: f64, +} + /// Installe la scène de l'app (JSON `SceneDescription`) sur la vue : layout preset piloté par /// l'app au lieu de la fixture. JSON invalide → ignoré côté natif. #[napi] @@ -415,6 +522,26 @@ pub struct ClipInput { pub webcam_offset_sec: f64, /// `false` évite une ouverture ffmpeg vouée à échouer et réserve du silence à ce clip. pub has_audio: bool, + /// Cameras 2-4 of the clip (index k-1 = camera k, an empty `path` = none). Absent = none. + pub additional_cameras: Option>, +} + +/// `ClipInput` → `pipeline::ClipSource`, shared by the MP4 and the GIF export. +fn clip_source(c: ClipInput) -> pipeline::ClipSource { + pipeline::ClipSource { + screen: c.screen_path, + webcam: c.webcam_path, + source_start_sec: c.source_start_sec, + source_end_sec: c.source_end_sec, + webcam_offset_sec: c.webcam_offset_sec, + has_audio: c.has_audio, + additional_cameras: c + .additional_cameras + .unwrap_or_default() + .into_iter() + .map(|cam| pipeline::ClipCamera { path: cam.path, offset_sec: cam.offset_sec }) + .collect(), + } } /// The export's pixel size. The SHAPE belongs to the scene: `scene.output` is what the live @@ -612,17 +739,7 @@ pub fn export_multi( params: Option, on_progress: Option, ) -> Result> { - let clips = clips - .into_iter() - .map(|c| pipeline::ClipSource { - screen: c.screen_path, - webcam: c.webcam_path, - source_start_sec: c.source_start_sec, - source_end_sec: c.source_end_sec, - webcam_offset_sec: c.webcam_offset_sec, - has_audio: c.has_audio, - }) - .collect(); + let clips = clips.into_iter().map(clip_source).collect(); Ok(AsyncTask::new(ExportMultiTask { out_path, clips, @@ -793,17 +910,7 @@ pub fn export_gif( // Deliberately the same argument shape as `export_multi`: the caller builds // one clip list and one scene, and picks the container. Cursor comes from // the scene like every other effect — there is no GIF-specific input left. - let clips = clips - .into_iter() - .map(|c| pipeline::ClipSource { - screen: c.screen_path, - webcam: c.webcam_path, - source_start_sec: c.source_start_sec, - source_end_sec: c.source_end_sec, - webcam_offset_sec: c.webcam_offset_sec, - has_audio: c.has_audio, - }) - .collect(); + let clips = clips.into_iter().map(clip_source).collect(); let gif_params = params .map(|p| GifExportParams { width: p.width, diff --git a/crates/compositor/build.rs b/crates/compositor/build.rs index 66f465153f..c55d8ee381 100644 --- a/crates/compositor/build.rs +++ b/crates/compositor/build.rs @@ -191,7 +191,7 @@ fn main() { fn compile_hlsl(out: &Path) { use windows::core::PCSTR; use windows::Win32::Graphics::Direct3D::Fxc::{D3DCompile, D3DCOMPILE_OPTIMIZATION_LEVEL3}; - use windows::Win32::Graphics::Direct3D::ID3DBlob; + use windows::Win32::Graphics::Direct3D::{ID3DBlob, D3D_SHADER_MACRO}; fn blob_bytes(blob: &ID3DBlob) -> Vec { unsafe { @@ -203,17 +203,25 @@ fn compile_hlsl(out: &Path) { const SRC: &str = "src/shaders.hlsl"; println!("cargo:rerun-if-changed={SRC}"); let hlsl = std::fs::read(SRC).expect(SRC); - // Tous les points d'entrée que `Compositor::new_inner` instancie, avec leur profil. - for (entry, profile) in [ - ("vs_main", "vs_5_0"), - ("ps_main", "ps_5_0"), - ("vs_fs", "vs_5_0"), - ("ps_y", "ps_5_0"), - ("ps_uv", "ps_5_0"), - ("ps_blur", "ps_5_0"), - ("ps_tex", "ps_5_0"), - ("ps_kawase_down", "ps_5_0"), - ("ps_kawase_up", "ps_5_0"), + // `ps_main` avec les modèles 3D : la variante que seuls ces calques utilisent (le haut de + // `shaders.hlsl`). La liste se termine par une entrée nulle. + let models = [ + D3D_SHADER_MACRO { Name: PCSTR(b"LAYER_MODELS\0".as_ptr()), Definition: PCSTR(b"1\0".as_ptr()) }, + D3D_SHADER_MACRO { Name: PCSTR::null(), Definition: PCSTR::null() }, + ]; + // Tous les points d'entrée que `Compositor::new_inner` instancie, avec leur profil, le nom du + // `.cso` qu'en lit `shader!` et leurs macros. + for (entry, profile, cso, defines) in [ + ("vs_main", "vs_5_0", "vs_main", None), + ("ps_main", "ps_5_0", "ps_main", None), + ("ps_main", "ps_5_0", "ps_main_models", Some(models.as_ptr())), + ("vs_fs", "vs_5_0", "vs_fs", None), + ("ps_y", "ps_5_0", "ps_y", None), + ("ps_uv", "ps_5_0", "ps_uv", None), + ("ps_blur", "ps_5_0", "ps_blur", None), + ("ps_tex", "ps_5_0", "ps_tex", None), + ("ps_kawase_down", "ps_5_0", "ps_kawase_down", None), + ("ps_kawase_up", "ps_5_0", "ps_kawase_up", None), ] { let (entry_z, profile_z) = (format!("{entry}\0"), format!("{profile}\0")); let (mut code, mut errors): (Option, Option) = (None, None); @@ -222,7 +230,7 @@ fn compile_hlsl(out: &Path) { hlsl.as_ptr().cast(), hlsl.len(), PCSTR(b"shaders.hlsl\0".as_ptr()), - None, + defines, None, PCSTR(entry_z.as_ptr()), PCSTR(profile_z.as_ptr()), @@ -234,9 +242,9 @@ fn compile_hlsl(out: &Path) { }; if let Err(e) = compiled { let log = errors.as_ref().map(blob_bytes).unwrap_or_default(); - panic!("{SRC} : {entry} ne compile pas ({e})\n{}", String::from_utf8_lossy(&log)); + panic!("{SRC} : {cso} ne compile pas ({e})\n{}", String::from_utf8_lossy(&log)); } - std::fs::write(out.join(format!("{entry}.cso")), blob_bytes(&code.unwrap())).expect(entry); + std::fs::write(out.join(format!("{cso}.cso")), blob_bytes(&code.unwrap())).expect(cso); } } diff --git a/crates/compositor/examples/live_free_run_bench_macos.rs b/crates/compositor/examples/live_free_run_bench_macos.rs new file mode 100644 index 0000000000..8b3ec5c3ba --- /dev/null +++ b/crates/compositor/examples/live_free_run_bench_macos.rs @@ -0,0 +1,125 @@ +//! Bench de la lecture libre de la preview (macOS) : rejoue la boucle de `render_thread` sur +//! un seul clip, à une vitesse donnée, et mesure le retard que la vue prend sur l'horloge. +//! +//! Même arithmétique que `render_thread` : l'accumulateur avance de `dt × vitesse`, `step()` +//! compose chaque frame due jusqu'à `max_steps` par tick (au-delà le reliquat est abandonné), +//! puis relit le RT UNE fois par tick, comme la publication macOS (pas de textures partagées). +//! `dt` est borné à 0,1 s comme dans `render_thread` : un tick plus long perd de l'horloge. +//! +//! Utilisation : +//! `cargo run --release --example live_free_run_bench_macos -- [vitesse=4] [secondes=8] [largeur=1920] [hauteur=1080]` + +#[cfg(target_os = "macos")] +fn main() -> anyhow::Result<()> { + use openscreen_compositor::compositor::Compositor; + use openscreen_compositor::config; + use openscreen_compositor::d3d::Gpu; + use openscreen_compositor::frame_geometry::live_params_from_scene; + use openscreen_compositor::live::Player; + use openscreen_compositor::scene::Scene; + use std::time::{Duration, Instant}; + + let mut args = std::env::args().skip(1); + let path = args.next().unwrap_or_else(|| { + eprintln!("usage: live_free_run_bench_macos [vitesse=4] [secondes=8] [largeur=1920] [hauteur=1080]"); + std::process::exit(2); + }); + let speed: f64 = args.next().and_then(|s| s.parse().ok()).unwrap_or(4.0); + let seconds: f64 = args.next().and_then(|s| s.parse().ok()).unwrap_or(8.0); + let width: u32 = args.next().and_then(|s| s.parse().ok()).unwrap_or(1920); + let height: u32 = args.next().and_then(|s| s.parse().ok()).unwrap_or(1080); + + let source = path.replace('\\', "/"); + let scene_json = format!( + r##"{{ + "clips": [{{"screenPath":"{source}","webcamPath":"","sourceStartSec":0,"sourceEndSec":100000,"webcamOffsetSec":0,"hasAudio":false}}], + "layout": {{"preset":"no-webcam","webcamSize":1.0,"webcamShape":"rounded","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false}}, + "effects": {{"padding":0.1,"blur":false,"shadow":0.5,"roundnessFrac":0.02,"motionBlur":0.0}}, + "background": {{"kind":"color","color":"#303030"}}, + "zoomRegions": [], + "speedRegions": [], + "cursor": {{"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0,"clipToBounds":false,"theme":"default"}}, + "cropByClip": [null], + "output": {{"width":{width},"height":{height},"fps":30}} + }}"## + ); + + let gpu = Gpu::create(false)?; + let mut cfg = config::all().pop().expect("au moins une config"); + cfg.zoom = false; + cfg.layout_anim = false; + let comp = Compositor::new_sized(&gpu, width, height)?; + let scene = Scene::from_json(&scene_json)?; + comp.set_live_params(live_params_from_scene(&scene)); + comp.set_scene(Some(scene.clone())); + comp.clear_cursor(); + + unsafe { + let mut player = Player::open(&path, "", &gpu)?; + player.set_programme_clock(Some(&scene), 0); + anyhow::ensure!(player.present_frame(&comp, &cfg, 0.0)?, "aucune frame à 0 s"); + // Temps source réellement joué, cumulé frame par frame : un clip plus court que la mesure + // reboucle à 0 (EOF), et la seule position finale compterait alors une seule passe. + let mut played = 0.0f64; + + let mut acc = 0.0f64; + let mut last = Instant::now(); + let t0 = last; + let (mut ticks, mut adopted, mut capped, mut published) = (0u64, 0u64, 0u64, 0u64); + // Ce que l'horloge de l'app a parcouru, en temps source : la cible que la vue devrait tenir. + let mut clock_source = 0.0f64; + while t0.elapsed().as_secs_f64() < seconds { + let now = Instant::now(); + let dt = (now - last).as_secs_f64().min(0.1); + last = now; + ticks += 1; + acc += dt * speed; + clock_source += (now - t0).as_secs_f64() * speed - clock_source; + let max_steps = ((3.0 * speed.max(1.0)).ceil() as i32).min(64); + let mut n = 0; + let mut stepped = false; + loop { + let before = player.screen_time_sec(); + if !player.step(&comp, &cfg, before + acc)? { + break; + } + stepped = true; + adopted += 1; + let after = player.screen_time_sec(); + played += if after >= before { after - before } else { after }; + acc = if after >= before { (acc - (after - before)).max(0.0) } else { 0.0 }; + n += 1; + if n >= max_steps { + capped += 1; + acc = 0.0; + break; + } + } + if stepped { + comp.readback_direct()?; + published += 1; + } else { + std::thread::sleep(Duration::from_millis(4)); + } + } + let wall = t0.elapsed().as_secs_f64(); + println!( + "vitesse {speed}x, {width}x{height}, {wall:.2} s : horloge {:.2} s source, vue {played:.2} s source, retard {:.2} s source ({:.2} s de lecture)", + clock_source, + clock_source - played, + (clock_source - played) / speed + ); + println!( + "ticks {ticks} ({:.1}/s), frames adoptées {adopted} ({:.1}/s), plafond atteint {capped} fois, publiées {published} ({:.1}/s)", + ticks as f64 / wall, + adopted as f64 / wall, + published as f64 / wall + ); + } + Ok(()) +} + +#[cfg(not(target_os = "macos"))] +fn main() { + eprintln!("live_free_run_bench_macos : macOS uniquement"); +} diff --git a/crates/compositor/src/camera_layers.rs b/crates/compositor/src/camera_layers.rs new file mode 100644 index 0000000000..ed5447b974 --- /dev/null +++ b/crates/compositor/src/camera_layers.rs @@ -0,0 +1,476 @@ +//! Which camera layers a frame draws, and where, inside camera layout regions. +//! +//! Pure planning: `plan_frame` calls `camera_layers_at` and the backends draw the result. A +//! region's layers glide in from whatever was on screen before it (the default camera-1 PiP, +//! or the layers of a region that ends exactly where it starts) and glide back out to the +//! default, on the Full Camera envelope (`regions::camera_fullscreen_region_phase`): the same +//! window lengths, the same `ease_out_screen_studio` curve, measured on the screen clock. A +//! camera-1 Full Camera region that meets a layout region at a seam is a neighbour like any +//! other: camera 0 frame-filling, gliding straight into or out of the layout region's layers. + +use crate::frame_geometry::webcam_shape_code; +use crate::regions::{ + ease_out_screen_studio, ScreenClock, FULLSCREEN_LEAD_OUT_WINDOW_S, REGION_SEAM_S, + TRANSITION_WINDOW_S, +}; +use crate::scene::{SceneCameraFullscreenRegion, SceneCameraLayer, SceneCameraLayoutRegion}; + +/// Two regions closer than this (source seconds) hand over directly, without the default. +const ADJACENT_S: f64 = REGION_SEAM_S; +/// Layers fainter than this are not drawn at all. +const MIN_OPACITY: f32 = 1e-3; +/// Cameras beyond camera 0 a compositor takes frames for (`set_extra_camera_frames`). +pub const MAX_EXTRA_CAMERAS: usize = 3; + +/// One camera to draw this frame. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct CameraLayerPlan { + pub camera: usize, + /// x, y, w, h in output fractions. + pub dst: [f32; 4], + /// Corner radius as a fraction of min(dst w, h) in pixels. + pub radius_frac: f32, + /// `webcam_shape_code` vocabulary. + pub shape: u32, + /// 0..1. + pub opacity: f32, + pub fills_frame: bool, +} + +fn plan_of(layer: &SceneCameraLayer) -> CameraLayerPlan { + let r = layer.rect; + CameraLayerPlan { + camera: layer.camera, + dst: [r.x, r.y, r.width, r.height], + radius_frac: layer.radius_frac, + shape: webcam_shape_code(&layer.shape), + opacity: 1.0, + fills_frame: layer.fills_frame, + } +} + +fn lerp(a: f32, b: f32, k: f32) -> f32 { + a + (b - a) * k +} + +/// `a` turning into `b` at `k` (0 = all `a`, 1 = all `b`). A camera in both moves; a camera in +/// only one of them fades. Draw order: frame-filling layers first, then the rest, each group in +/// `b`'s order followed by what only `a` still shows. +fn blend(a: &[CameraLayerPlan], b: &[CameraLayerPlan], k: f32) -> Vec { + let k = k.clamp(0.0, 1.0); + let mut out = Vec::with_capacity(a.len() + b.len()); + for to in b { + out.push(match a.iter().find(|from| from.camera == to.camera) { + Some(from) => { + // Discrete properties switch half-way through the move. + let flags = if k >= 0.5 { to } else { from }; + CameraLayerPlan { + camera: to.camera, + dst: std::array::from_fn(|i| lerp(from.dst[i], to.dst[i], k)), + radius_frac: lerp(from.radius_frac, to.radius_frac, k), + shape: flags.shape, + opacity: 1.0, + fills_frame: flags.fills_frame, + } + } + None => CameraLayerPlan { opacity: k, ..*to }, + }); + } + for from in a { + if !b.iter().any(|to| to.camera == from.camera) { + out.push(CameraLayerPlan { opacity: 1.0 - k, ..*from }); + } + } + out.retain(|layer| layer.opacity > MIN_OPACITY); + // Stable: each group keeps the order built above. + out.sort_by_key(|layer| !layer.fills_frame); + out +} + +/// Camera 0 as a Full Camera region shows it: the whole frame, square corners. Same discrete +/// shape as the default layer, so the hand-over to the default inside the region is seamless. +fn full_camera_layer(default: &CameraLayerPlan) -> CameraLayerPlan { + CameraLayerPlan { + camera: 0, + dst: [0.0, 0.0, 1.0, 1.0], + radius_frac: 0.0, + shape: default.shape, + opacity: 1.0, + fills_frame: true, + } +} + +/// The layers to draw at source time `t`, in draw order. +/// +/// `default_cam0`: camera 0's layer as today's plan places it (None = camera 0 not drawn). It +/// is what shows outside every region, and what a region glides from and back to. +/// +/// `fullscreen`: the camera-1 Full Camera regions. One that meets a layout region at a seam +/// is that region's neighbour, with camera 0 frame-filling: the layout region glides in from +/// it, or holds to its end and the Full Camera region's lead-in glides on from its layers +/// (the Full Camera phase holds 1 on that side, `regions::camera_fullscreen_region_phase`). +/// +/// Time base: copied from `camera_fullscreen_region_phase` — the region is found on SOURCE +/// time (its bounds compared to `t` as `f32`), then the windows are measured on the SCREEN +/// clock (`clock.at`), so a speed region does not stretch or squash the glide. +pub fn camera_layers_at( + regions: &[SceneCameraLayoutRegion], + fullscreen: &[SceneCameraFullscreenRegion], + t: f32, + clock: &ScreenClock, + default_cam0: Option, +) -> Vec { + let default: Vec = default_cam0.into_iter().collect(); + let full_cam0: Vec = default_cam0.iter().map(full_camera_layer).collect(); + // An empty (or reversed) region would match `t` at a single instant and draw its layers + // at full strength for that sample; it is neither drawn nor anyone's neighbour. + let non_empty = || regions.iter().enumerate().filter(|(_, r)| r.end_sec > r.start_sec); + let full_cameras = || fullscreen.iter().filter(|r| r.end_sec > r.start_sec); + let layers_of = |r: &SceneCameraLayoutRegion| r.layers.iter().map(plan_of).collect::>(); + let Some(index) = non_empty() + .find(|(_, r)| r.start_sec as f32 <= t && t <= r.end_sec as f32) + .map(|(i, _)| i) + else { + return full_camera_lead_in(regions, fullscreen, t, clock, &full_cam0).unwrap_or(default); + }; + let region = ®ions[index]; + let others = || non_empty().filter(move |(i, _)| *i != index).map(|(_, r)| r); + let prev = others() + .find(|r| (r.end_sec - region.start_sec).abs() <= ADJACENT_S) + .map(layers_of) + .or_else(|| { + full_cameras() + .any(|f| (f.end_sec - region.start_sec).abs() <= ADJACENT_S) + .then(|| full_cam0.clone()) + }); + let has_next = others().any(|r| (r.start_sec - region.end_sec).abs() <= ADJACENT_S) + || full_cameras().any(|f| (f.start_sec - region.end_sec).abs() <= ADJACENT_S); + let current = layers_of(region); + + let (start, end, t) = ( + clock.at(region.start_sec as f32), + clock.at(region.end_sec as f32), + clock.at(t), + ); + let half = (end - start) * 0.5; + let win_in = TRANSITION_WINDOW_S.min(half); + let win_out = FULLSCREEN_LEAD_OUT_WINDOW_S.min(half); + if t - start < win_in { + let from = prev.as_deref().unwrap_or(&default); + blend(from, ¤t, ease_out_screen_studio((t - start) / win_in)) + } else if !has_next && end - t < win_out { + // With a region right after this one, the hand-over is that region's lead-in instead. + blend(¤t, &default, 1.0 - ease_out_screen_studio((end - t) / win_out)) + } else { + blend(&[], ¤t, 1.0) + } +} + +/// Inside a Full Camera region that starts where a layout region ends, during its lead-in: +/// the layout region's layers gliding into camera 0 frame-filling (`full_cam0`), on the same +/// window and curve as a layout region's lead-in. None anywhere else. +fn full_camera_lead_in( + regions: &[SceneCameraLayoutRegion], + fullscreen: &[SceneCameraFullscreenRegion], + t: f32, + clock: &ScreenClock, + full_cam0: &[CameraLayerPlan], +) -> Option> { + let full = fullscreen.iter().find(|f| { + f.end_sec > f.start_sec && f.start_sec as f32 <= t && t <= f.end_sec as f32 + })?; + let before = regions.iter().find(|r| { + r.end_sec > r.start_sec && (r.end_sec - full.start_sec).abs() <= ADJACENT_S + })?; + let (start, end) = (clock.at(full.start_sec as f32), clock.at(full.end_sec as f32)); + let t = clock.at(t); + let win_in = TRANSITION_WINDOW_S.min((end - start) * 0.5); + if t - start >= win_in { + return None; + } + let from: Vec = before.layers.iter().map(plan_of).collect(); + Some(blend(&from, full_cam0, ease_out_screen_studio((t - start) / win_in))) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::scene::SceneRect; + + const PIP: [f32; 4] = [0.75, 0.7, 0.22, 0.22]; + const CORNER: [f32; 4] = [0.05, 0.05, 0.2, 0.2]; + const FULL: [f32; 4] = [0.0, 0.0, 1.0, 1.0]; + + fn default_pip() -> Option { + Some(CameraLayerPlan { + camera: 0, + dst: PIP, + radius_frac: 0.5, + shape: 1, + opacity: 1.0, + fills_frame: false, + }) + } + + fn region(start: f64, end: f64, layers: &[(usize, [f32; 4], bool)]) -> SceneCameraLayoutRegion { + SceneCameraLayoutRegion { + clip_index: Some(0), + start_sec: start, + end_sec: end, + layers: layers + .iter() + .map(|&(camera, r, fills_frame)| SceneCameraLayer { + camera, + rect: SceneRect { x: r[0], y: r[1], width: r[2], height: r[3] }, + radius_frac: if fills_frame { 0.0 } else { 0.3 }, + shape: if fills_frame { "rectangle" } else { "rounded" }.to_string(), + fills_frame, + }) + .collect(), + } + } + + fn at(regions: &[SceneCameraLayoutRegion], t: f32) -> Vec { + camera_layers_at(regions, &[], t, &ScreenClock::default(), default_pip()) + } + + fn layer(layers: &[CameraLayerPlan], camera: usize) -> Option { + layers.iter().copied().find(|l| l.camera == camera) + } + + fn strictly_between(v: f32, a: f32, b: f32) -> bool { + v > a.min(b) && v < a.max(b) + } + + #[test] + fn outside_every_region_only_the_default_camera_is_drawn() { + let regions = [region(2.0, 6.0, &[(1, FULL, true)])]; + for t in [0.0, 1.9, 6.1, 9.0] { + assert_eq!(at(®ions, t), vec![default_pip().unwrap()], "t = {t}"); + } + assert!(camera_layers_at(®ions, &[], 1.0, &ScreenClock::default(), None).is_empty()); + } + + #[test] + fn inside_a_region_after_the_lead_in_its_layers_are_drawn_fully() { + let regions = [region(2.0, 6.0, &[(1, FULL, true), (0, CORNER, false)])]; + let layers = at(®ions, 4.0); + assert_eq!(layers.len(), 2); + assert_eq!(layers[0].camera, 1); + assert_eq!(layers[0].dst, FULL); + assert_eq!(layers[0].opacity, 1.0); + assert!(layers[0].fills_frame); + assert_eq!(layers[1].camera, 0); + assert_eq!(layers[1].dst, CORNER); + assert_eq!(layers[1].opacity, 1.0); + assert_eq!(layers[1].shape, webcam_shape_code("rounded")); + assert_eq!(layers[1].radius_frac, 0.3); + } + + #[test] + fn entering_a_region_glides_camera_0_and_fades_in_a_new_camera() { + let regions = [region(2.0, 6.0, &[(1, FULL, true), (0, CORNER, false)])]; + let layers = at(®ions, 2.0 + TRANSITION_WINDOW_S / 2.0); + let cam0 = layer(&layers, 0).expect("camera 0"); + for i in 0..4 { + assert!(strictly_between(cam0.dst[i], PIP[i], CORNER[i]), "dst[{i}] = {}", cam0.dst[i]); + } + assert_eq!(cam0.opacity, 1.0); + let cam1 = layer(&layers, 1).expect("camera 1"); + assert!(strictly_between(cam1.opacity, 0.0, 1.0), "opacity {}", cam1.opacity); + assert_eq!(cam1.dst, FULL); + // On the region's start the glide has not begun. + assert_eq!(at(®ions, 2.0), vec![default_pip().unwrap()]); + } + + #[test] + fn leaving_a_region_returns_to_the_default() { + let regions = [region(2.0, 6.0, &[(1, FULL, true), (0, CORNER, false)])]; + let win_out = FULLSCREEN_LEAD_OUT_WINDOW_S.min(2.0); + let layers = at(®ions, 6.0 - win_out / 2.0); + let cam0 = layer(&layers, 0).expect("camera 0"); + for i in 0..4 { + assert!(strictly_between(cam0.dst[i], PIP[i], CORNER[i]), "dst[{i}] = {}", cam0.dst[i]); + } + let cam1 = layer(&layers, 1).expect("camera 1"); + assert!(strictly_between(cam1.opacity, 0.0, 1.0), "opacity {}", cam1.opacity); + // On the region's end it is back to the default, exactly. + let end = at(®ions, 6.0); + assert_eq!(end.len(), 1); + assert_eq!(end[0].camera, 0); + for i in 0..4 { + assert!((end[0].dst[i] - PIP[i]).abs() < 1e-6); + } + } + + #[test] + fn adjacent_regions_glide_directly() { + let regions = [region(2.0, 5.0, &[(1, FULL, true)]), region(5.0, 8.0, &[(2, FULL, true)])]; + // The end of A keeps A: no lead-out to the default PiP. + let before = at(®ions, 5.0 - 0.01); + assert_eq!(before.len(), 1); + assert_eq!(before[0].camera, 1); + assert_eq!(before[0].opacity, 1.0); + // B's lead-in hands camera 1 over to camera 2. + let during = at(®ions, 5.0 + TRANSITION_WINDOW_S / 2.0); + let cam1 = layer(&during, 1).expect("camera 1 fading out"); + let cam2 = layer(&during, 2).expect("camera 2 fading in"); + assert!(strictly_between(cam1.opacity, 0.0, 1.0)); + assert!(strictly_between(cam2.opacity, 0.0, 1.0)); + // Neither region has camera 0, so the default PiP never shows around the seam. + let mut t = 4.8f32; + while t <= 5.2 { + assert!(layer(&at(®ions, t), 0).is_none(), "camera 0 at t = {t}"); + t += 0.01; + } + } + + fn full_camera(start: f64, end: f64) -> SceneCameraFullscreenRegion { + SceneCameraFullscreenRegion { + clip_index: Some(0), + start_sec: start, + end_sec: end, + rotation: 0, + mirror: None, + full_frame: false, + } + } + + /// The plan with Full Camera regions, camera 0's default placed as `plan_frame` places it: + /// grown from the PiP to the frame by the Full Camera progress. + fn at_with_full( + regions: &[SceneCameraLayoutRegion], + fulls: &[SceneCameraFullscreenRegion], + t: f32, + ) -> Vec { + let clock = ScreenClock::default(); + let p = crate::regions::camera_fullscreen_progress_at(fulls, t, &clock, regions); + let default = CameraLayerPlan { + dst: std::array::from_fn(|i| lerp(PIP[i], FULL[i], p)), + fills_frame: p >= 1.0, + ..default_pip().unwrap() + }; + camera_layers_at(regions, fulls, t, &clock, Some(default)) + } + + /// Camera 0 on the straight path between two rects: never the default PiP's geometry. + fn on_the_path(dst: [f32; 4], a: [f32; 4], b: [f32; 4]) -> bool { + (0..4).all(|i| dst[i] >= a[i].min(b[i]) - 1e-5 && dst[i] <= a[i].max(b[i]) + 1e-5) + } + + #[test] + fn full_camera_then_layout_region_glides_directly() { + let regions = [region(5.0, 8.0, &[(1, FULL, true), (0, CORNER, false)])]; + let fulls = [full_camera(2.0, 5.0)]; + // The end of the Full Camera region keeps camera 0 on the frame: no shrink. + let before = at_with_full(®ions, &fulls, 5.0 - 0.01); + assert_eq!(before.len(), 1); + assert_eq!(before[0].camera, 0); + assert_eq!(before[0].dst, FULL); + assert!(before[0].fills_frame); + // The layout region's lead-in moves camera 0 from the frame to its corner. + let during = at_with_full(®ions, &fulls, 5.0 + TRANSITION_WINDOW_S / 2.0); + let cam0 = layer(&during, 0).expect("camera 0"); + for i in 2..4 { + assert!(strictly_between(cam0.dst[i], FULL[i], CORNER[i]), "dst[{i}] = {}", cam0.dst[i]); + } + let cam1 = layer(&during, 1).expect("camera 1 fading in"); + assert!(strictly_between(cam1.opacity, 0.0, 1.0)); + // The default PiP never shows around the seam. + let mut t = 3.5f32; + while t <= 5.0 + TRANSITION_WINDOW_S + 0.1 { + let cam0 = layer(&at_with_full(®ions, &fulls, t), 0).expect("camera 0"); + assert!(on_the_path(cam0.dst, FULL, CORNER), "t = {t}: {:?}", cam0.dst); + assert_eq!(cam0.opacity, 1.0, "t = {t}"); + t += 0.005; + } + } + + #[test] + fn layout_region_then_full_camera_glides_directly() { + let regions = [region(2.0, 5.0, &[(1, FULL, true), (0, CORNER, false)])]; + let fulls = [full_camera(5.0, 8.0)]; + // The end of the layout region keeps its layers: no lead-out to the default PiP. + let before = at_with_full(®ions, &fulls, 5.0 - 0.01); + assert_eq!(layer(&before, 0).expect("camera 0").dst, CORNER); + assert_eq!(layer(&before, 1).expect("camera 1").opacity, 1.0); + // The Full Camera lead-in takes camera 0 from its corner to the frame; camera 1 fades. + let during = at_with_full(®ions, &fulls, 5.0 + TRANSITION_WINDOW_S / 2.0); + let cam0 = layer(&during, 0).expect("camera 0"); + for i in 2..4 { + assert!(strictly_between(cam0.dst[i], CORNER[i], FULL[i]), "dst[{i}] = {}", cam0.dst[i]); + } + let cam1 = layer(&during, 1).expect("camera 1 fading out"); + assert!(strictly_between(cam1.opacity, 0.0, 1.0)); + // After the lead-in camera 0 is the default again, which the held phase keeps full. + let after = at_with_full(®ions, &fulls, 5.0 + TRANSITION_WINDOW_S + 0.01); + assert_eq!(after.len(), 1); + assert_eq!(after[0].dst, FULL); + let mut t = 3.5f32; + while t <= 7.0 { + let cam0 = layer(&at_with_full(®ions, &fulls, t), 0).expect("camera 0"); + assert!(on_the_path(cam0.dst, CORNER, FULL), "t = {t}: {:?}", cam0.dst); + assert_eq!(cam0.opacity, 1.0, "t = {t}"); + t += 0.005; + } + } + + #[test] + fn a_full_camera_region_off_the_seam_is_no_neighbour() { + // 10 ms apart: the layout region glides from the default as before. + let regions = [region(5.01, 8.0, &[(1, FULL, true), (0, CORNER, false)])]; + let fulls = [full_camera(2.0, 5.0)]; + assert_eq!( + at_with_full(®ions, &fulls, 5.01 + TRANSITION_WINDOW_S / 2.0), + at(®ions, 5.01 + TRANSITION_WINDOW_S / 2.0) + ); + let shrinking = layer(&at_with_full(®ions, &fulls, 4.9), 0).expect("camera 0"); + assert!(shrinking.dst[2] < 1.0, "the Full Camera region leads out: {:?}", shrinking.dst); + } + + #[test] + fn a_short_region_halves_its_windows() { + let regions = [region(2.0, 3.0, &[(1, FULL, true)])]; + // Both windows are 0.5 s: the middle is the only frame at full strength. + let mid = at(®ions, 2.5); + assert_eq!(mid.len(), 1); + assert_eq!(mid[0].camera, 1); + assert_eq!(mid[0].opacity, 1.0); + for t in [2.25, 2.75] { + let cam1 = layer(&at(®ions, t), 1).expect("camera 1"); + assert!(strictly_between(cam1.opacity, 0.0, 1.0), "t = {t}"); + } + } + + #[test] + fn an_empty_region_draws_nothing_of_its_own() { + // start == end (and a reversed region) would otherwise match `t` at its one instant + // and draw its layers at full strength for that sample. + for regions in [ + [region(3.0, 3.0, &[(1, FULL, true)])], + [region(3.0, 2.0, &[(1, FULL, true)])], + ] { + for t in [2.0, 2.5, 3.0] { + let layers = at(®ions, t); + assert!(layer(&layers, 1).is_none(), "t = {t}"); + assert_eq!(layers.len(), 1, "t = {t}"); + assert_eq!(layers[0].camera, 0, "t = {t}"); + } + } + // Nor does it count as a neighbour: the region before it still leads out. + let regions = [region(2.0, 6.0, &[(1, FULL, true)]), region(6.0, 6.0, &[(1, FULL, true)])]; + let cam1 = layer(&at(®ions, 5.9), 1).expect("camera 1"); + assert!(strictly_between(cam1.opacity, 0.0, 1.0)); + } + + #[test] + fn fills_frame_layers_are_drawn_first() { + let regions = [region(2.0, 6.0, &[(0, CORNER, false), (1, FULL, true)])]; + let order: Vec = at(®ions, 4.0).iter().map(|l| l.camera).collect(); + assert_eq!(order, vec![1, 0]); + // During the lead-in too. + let order: Vec = + at(®ions, 2.0 + TRANSITION_WINDOW_S / 2.0).iter().map(|l| l.camera).collect(); + assert_eq!(order, vec![1, 0]); + } +} diff --git a/crates/compositor/src/compositor_linux.rs b/crates/compositor/src/compositor_linux.rs index 198e6e637c..e969c08c50 100644 --- a/crates/compositor/src/compositor_linux.rs +++ b/crates/compositor/src/compositor_linux.rs @@ -52,6 +52,12 @@ use crate::scene::{Scene, SceneBackground, WallpaperMotion}; const LAYER_WGSL: &str = include_str!("vk_shaders/layer.wgsl"); const BLUR_WGSL: &str = include_str!("vk_shaders/blur.wgsl"); +/// `layer.wgsl` avec ou sans ses modeles 3D (modes 15 a 17) : la constante que le shader teste +/// pour les compiler, prefixee au source (pourquoi deux pipelines : le haut de `layer.wgsl`). +fn layer_source(models: bool) -> String { + format!("const LAYER_MODELS: bool = {models};\n{LAYER_WGSL}") +} + /// Budget du cache de textures image (`img_cache`), en octets. /// /// Doit tenir le JEU ACTIF d'une frame -- au pire un wallpaper d'ecran ET un @@ -61,16 +67,24 @@ const BLUR_WGSL: &str = include_str!("vk_shaders/blur.wgsl"); /// en parcourant les 18 wallpapers livres) en laissant le jeu actif resident. const IMG_CACHE_BUDGET_BYTES: u64 = 512 * 1024 * 1024; -/// Taille du buffer uniforme d'un calque : `LayerCB` entier (176 octets), le `struct Layer` de +/// Taille du buffer uniforme d'un calque : `LayerCB` entier (256 octets), le `struct Layer` de /// `layer.wgsl`. `blur.wgsl` n'en lit que les 128 premiers. const LAYER_BYTES: u64 = std::mem::size_of::() as u64; /// `&LayerCB` -> ses octets. `LayerCB` est `#[repr(C, align(16))]`, son layout EST le buffer -/// uniforme WGSL (dix vec4 et un vec2 + 2 f32 = 176 octets). +/// uniforme WGSL (seize vec4 = 256 octets). fn layer_bytes(cb: &LayerCB) -> &[u8] { unsafe { std::slice::from_raw_parts(cb as *const LayerCB as *const u8, LAYER_BYTES as usize) } } +/// Le maillage de Prism Glow tel que le lit le WGSL (`PrismMesh`) : les triangles, les +/// polygones, un point par vec4 (le pas d'un tableau uniform), puis les boîtes. +fn prism_mesh_bytes() -> Vec { + use crate::prism_mesh::{BOXES, POLYS, TRIS}; + let points = POLYS.iter().map(|&[x, y]| [x, y, 0.0, 0.0]); + TRIS.iter().copied().chain(points).chain(BOXES).flatten().flat_map(f32::to_ne_bytes).collect() +} + /// Un calque de fond deja lie, en attente de son `draw`. `_buf`/`_tex`/`_view` /// ne sont jamais relus : ils gardent en vie ce que le bind group reference /// jusqu'au submit. Ce backend encode toute la frame avant de la soumettre, la @@ -81,10 +95,25 @@ fn layer_bytes(cb: &LayerCB) -> &[u8] { /// ET le fond de la bulle webcam sont desormais construits par les memes /// methodes. struct BgDraw { - _buf: wgpu::Buffer, + layer: LayerBind, _tex: Option, _view: Option, +} + +/// Un calque pret a dessiner : son uniforme (qui doit vivre jusqu'au draw), son bind group, et +/// s'il est l'un des modeles 3D, que seul `pipeline_models` compile (`LayerCB::needs_models`). +/// `draw_layer` en tire le pipeline : c'est le mode du calque qui decide, pas le site d'appel. +struct LayerBind { + _buf: wgpu::Buffer, bind: wgpu::BindGroup, + models: bool, +} + +/// Le fond tel que la passe 1 (et son flou) l'a laisse dans le RT, et tout ce qui l'a decide +/// (`key`, cf. `compose_frame`). +struct BgCache { + key: crate::frame_geometry::BackgroundKey, + tex: wgpu::Texture, } /// Une copie RT -> staging DEJA SOUMISE, dont le mapping est arme mais pas @@ -262,16 +291,23 @@ pub struct Compositor { // Pipeline de calque (VS + FS `layer.wgsl`), sampler lineaire, bind group // layout (uniform + 2 textures + sampler). Immuables apres `new_sized`. + // `pipeline` n'a pas les modeles 3D (modes 15 a 17) : ils passent par `pipeline_models`. pipeline: wgpu::RenderPipeline, /// Meme shader et meme layout que `pipeline`, blend ADDITIF pondere par la /// constante de blend. Sert a sommer les copies de la trainee du curseur /// dans `accum` ; cf. `blend_add` cote Windows. pipeline_add: wgpu::RenderPipeline, + /// `pipeline` et `pipeline_add` avec les modeles 3D : le curseur modelise, l'impact de son + /// clic, l'appareil et son ombre, et rien d'autre (cf. `layer_source`, `draw_layer`). + pipeline_models: wgpu::RenderPipeline, + pipeline_add_models: wgpu::RenderPipeline, /// Copie plein ecran d'`accum` vers le RT en « over » premultiplie /// (`blur.wgsl` : `vs_fullscreen` + `fs_copy`). Utilise le layout du blur. pipeline_copy: wgpu::RenderPipeline, bind_group_layout: wgpu::BindGroupLayout, sampler: wgpu::Sampler, + /// Le maillage de Prism Glow (`prism_mesh.rs`) tel que le lit le WGSL (`PrismMesh`, binding 7). + prism_mesh: wgpu::Buffer, // Chaine de blur Kawase du fond (`blur.wgsl`) : layout dedie (uniform + 1 // tex + sampler), 2 pipelines (down/up), 3 textures de pyramide (1/2, 1/4, @@ -313,10 +349,15 @@ pub struct Compositor { readback_yuv: RefCell, // Etat pilote par live.rs (interior mutability : les methodes sont `&self`). + /// The frames of cameras 1..=3 (`set_extra_camera_frames`), as addresses (0 = none) so + /// the compositor's auto traits stay what they were without the raw pointers. + extra_camera_frames: std::cell::Cell<[usize; crate::camera_layers::MAX_EXTRA_CAMERAS]>, live_params: RefCell, scene: RefCell>, cursor: RefCell>, cursor_time: RefCell>, + /// Le métrage dans la dernière image composée, que l'éditeur lit avec elle (`live.rs`). + footage: std::cell::Cell>, timeline_time: RefCell>, /// Temps programme (secondes de sortie) -- cf. `FrameGeometryInput::programme_time`. programme_time: RefCell>, @@ -341,6 +382,9 @@ pub struct Compositor { /// touche depuis appartient au jeu actif et ne peut pas etre evince -- voir /// `cached_image`. img_frame_start: std::cell::Cell, + /// Le dernier fond fixe compose. Un fond qui ne bouge pas est le meme a chaque frame : on le + /// recopie au lieu de le redessiner (passe 1 de `compose_frame`). + bg_cache: RefCell>, /// Champs de distance des sprites de curseur (mode 15), R16F, par chemin, avec leur forme. /// Pas d'eviction : seuls les sprites du theme en cours y passent (~2,6 Mo pour les seize). sdf_cache: RefCell>, @@ -410,10 +454,14 @@ impl Compositor { feature_level: gpu.feature_level, }; - let module = gpu.device.create_shader_module(wgpu::ShaderModuleDescriptor { - label: Some("layer.wgsl"), - source: wgpu::ShaderSource::Wgsl(LAYER_WGSL.into()), - }); + let mk_module = |label: &str, models: bool| { + gpu.device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some(label), + source: wgpu::ShaderSource::Wgsl(layer_source(models).into()), + }) + }; + let module = mk_module("layer.wgsl", false); + let module_models = mk_module("layer.wgsl (modeles)", true); let sampler = gpu.device.create_sampler(&wgpu::SamplerDescriptor { label: Some("layer"), address_mode_u: wgpu::AddressMode::ClampToEdge, @@ -474,26 +522,44 @@ impl Compositor { // Pyramide de profondeur de champ (modes 8 et 18), `dummy` ailleurs : le // mode 18 lit a la fois le metrage, son rendu isole (binding 4) et elle. tex_entry(6), + // Le maillage de Prism Glow (mode 15). Un uniform et non des tables dans le + // shader : lavapipe les recopiait a chaque pixel de chaque calque, et une + // frame sans curseur se rendait 3,4 fois plus lentement. + wgpu::BindGroupLayoutEntry { + binding: 7, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: None, + }, + count: None, + }, ], }); + let prism_mesh = gpu.device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("prism-mesh"), + contents: &prism_mesh_bytes(), + usage: wgpu::BufferUsages::UNIFORM, + }); let pipeline_layout = gpu.device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { label: Some("layer"), bind_group_layouts: &[&bind_group_layout], push_constant_ranges: &[], }); - // Deux pipelines pour le MEME shader de calque : seul le blend change. - let mk_layer = |label: &str, blend: wgpu::BlendState| { + // Deux pipelines par source de calque : seul le blend change. + let mk_layer = |label: &str, module: &wgpu::ShaderModule, blend: wgpu::BlendState| { gpu.device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { label: Some(label), layout: Some(&pipeline_layout), vertex: wgpu::VertexState { - module: &module, + module, entry_point: Some("vs_main"), compilation_options: wgpu::PipelineCompilationOptions::default(), buffers: &[], }, fragment: Some(wgpu::FragmentState { - module: &module, + module, entry_point: Some("fs_main"), compilation_options: wgpu::PipelineCompilationOptions::default(), targets: &[Some(wgpu::ColorTargetState { @@ -512,22 +578,23 @@ impl Compositor { cache: None, }) }; - let pipeline = mk_layer("layer", wgpu::BlendState::PREMULTIPLIED_ALPHA_BLENDING); + let over = wgpu::BlendState::PREMULTIPLIED_ALPHA_BLENDING; + let pipeline = mk_layer("layer", &module, over); + let pipeline_models = mk_layer("layer-models", &module_models, over); // SOMME pondere : `src * constante + dst`. La constante (posee par pass // via `set_blend_constant`) vaut 1/taps, donc N copies d'un curseur // parfaitement immobile redonnent exactement ce curseur. Transcription // du `blend_add` D3D11 (BLEND_FACTOR / ONE / OP_ADD sur couleur ET // alpha) ; l'alpha doit suivre la couleur, sinon la somme n'est plus // premultipliee et la composition finale delave la trainee. - let add = wgpu::BlendComponent { + let sum = wgpu::BlendComponent { src_factor: wgpu::BlendFactor::Constant, dst_factor: wgpu::BlendFactor::One, operation: wgpu::BlendOperation::Add, }; - let pipeline_add = mk_layer( - "layer-add", - wgpu::BlendState { color: add, alpha: add }, - ); + let add = wgpu::BlendState { color: sum, alpha: sum }; + let pipeline_add = mk_layer("layer-add", &module, add); + let pipeline_add_models = mk_layer("layer-add-models", &module_models, add); // --- Chaine de blur Kawase du fond (`blur.wgsl`) --- let blur_module = gpu.device.create_shader_module(wgpu::ShaderModuleDescriptor { @@ -660,9 +727,12 @@ impl Compositor { render_h: h, pipeline, pipeline_add, + pipeline_models, + pipeline_add_models, pipeline_copy, bind_group_layout, sampler, + prism_mesh, blur_bgl, blur_down, blur_up, @@ -678,10 +748,12 @@ impl Compositor { readback, yuv: RefCell::new(None), readback_yuv, + extra_camera_frames: std::cell::Cell::new([0; crate::camera_layers::MAX_EXTRA_CAMERAS]), live_params: RefCell::new(LiveParams::default()), scene: RefCell::new(None), cursor: RefCell::new(None), cursor_time: RefCell::new(None), + footage: std::cell::Cell::new(None), timeline_time: RefCell::new(None), programme_time: RefCell::new(None), text_raster: crate::text::TextRasterizer::new().ok(), @@ -689,6 +761,7 @@ impl Compositor { sdf_cache: RefCell::new(std::collections::HashMap::new()), img_tick: std::cell::Cell::new(0), img_frame_start: std::cell::Cell::new(0), + bg_cache: RefCell::new(None), ann_copy, ann_copy_view, ann_copy_mips, @@ -749,6 +822,7 @@ impl Compositor { ann_copy, ann_copy_view, ann_copy_mips, + bg_cache: RefCell::new(None), ..self }) } @@ -817,7 +891,8 @@ impl Compositor { view_formats: &[], }) }; - let rt = mk("rt", wgpu::TextureUsages::COPY_SRC); + // COPY_DST : le fond fixe y est recopie (`bg_cache`). + let rt = mk("rt", wgpu::TextureUsages::COPY_SRC | wgpu::TextureUsages::COPY_DST); let accum = mk("accum", wgpu::TextureUsages::empty()); let rt_view = rt.create_view(&wgpu::TextureViewDescriptor::default()); let accum_view = accum.create_view(&wgpu::TextureViewDescriptor::default()); @@ -1095,6 +1170,9 @@ impl Compositor { pub fn set_scene(&self, s: Option) { *self.scene.borrow_mut() = s; + // A new scene can drop the regions that drew an extra camera: forget its frames so a + // stale pointer is never drawn before the next `set_extra_camera_frames`. + self.extra_camera_frames.set([0; crate::camera_layers::MAX_EXTRA_CAMERAS]); } pub fn set_cursor(&self, track: crate::cursor::CursorTrack) { @@ -1129,8 +1207,29 @@ impl Compositor { } /// Pas de cache de SRV cote wgpu (les `TextureView`s sont recreees a chaque - /// draw depuis le carrier) -- no-op conserve pour la symetrie d'API. - pub fn clear_srv_cache(&self) {} + /// draw depuis le carrier). Only the extra cameras' frame slots are reset. + pub fn clear_srv_cache(&self) { + // The extra cameras' frames belong to the decoders just closed: forget them too, so + // a stale pointer is never read before the next `set_extra_camera_frames`. + self.extra_camera_frames.set([0; crate::camera_layers::MAX_EXTRA_CAMERAS]); + } + + /// Frames for cameras 1..=3 (index 0 = scene camera 1). A null or missing entry means that + /// camera is not drawn this frame. The compositor keeps the pointers and reads them in every + /// `compose_frame` until the next call, so they must stay valid until then. + pub unsafe fn set_extra_camera_frames(&self, frames: &[*const AVFrame]) { + let mut slots = [0usize; crate::camera_layers::MAX_EXTRA_CAMERAS]; + for (slot, frame) in slots.iter_mut().zip(frames) { + *slot = *frame as usize; + } + self.extra_camera_frames.set(slots); + } + + /// The frame set for scene camera `camera` (1..=3), null when there is none. + fn extra_camera_frame(&self, camera: usize) -> *const AVFrame { + let slots = self.extra_camera_frames.get(); + camera.checked_sub(1).and_then(|k| slots.get(k).copied()).unwrap_or(0) as *const AVFrame + } // -- seam frame (lit le carrier `data[0]`) -- @@ -1251,24 +1350,16 @@ impl Compositor { fn draw_screen_group( &self, rpass: &mut wgpu::RenderPass<'_>, - shadow: &Option<(wgpu::Buffer, wgpu::BindGroup)>, - window_frame: &Option<(wgpu::Buffer, wgpu::BindGroup)>, - screen: &wgpu::BindGroup, - device_frame: &Option<(wgpu::Buffer, wgpu::BindGroup)>, + shadow: &Option, + window_frame: &Option, + screen: &LayerBind, + device_frame: &Option, ) { - if let Some((_buf, bind)) = shadow { - rpass.set_bind_group(0, bind, &[]); - rpass.draw(0..4, 0..1); - } - if let Some((_buf, bind)) = window_frame { - rpass.set_bind_group(0, bind, &[]); - rpass.draw(0..4, 0..1); - } - rpass.set_bind_group(0, screen, &[]); - rpass.draw(0..4, 0..1); - if let Some((_buf, bind)) = device_frame { - rpass.set_bind_group(0, bind, &[]); - rpass.draw(0..4, 0..1); + for layer in [shadow.as_ref(), window_frame.as_ref(), Some(screen), device_frame.as_ref()] + .into_iter() + .flatten() + { + self.draw_layer(rpass, layer, false); } } @@ -1277,12 +1368,13 @@ impl Compositor { cb: &LayerCB, planes: Option<(&wgpu::TextureView, &wgpu::TextureView, &wgpu::TextureView)>, dummy: &wgpu::TextureView, - ) -> (wgpu::Buffer, wgpu::BindGroup) { + ) -> LayerBind { self.make_bind_b4(cb, planes, dummy, None, None) } /// `make_bind`, binding 4 impose (le rendu isole du mode 18, a la place du masque webcam /// qu'il ne lit pas), et la pyramide de profondeur de champ en binding 6 (modes 8 et 18). + /// Le mode 15 y met le champ de son sprite et le sprite. fn make_bind_b4( &self, cb: &LayerCB, @@ -1290,7 +1382,7 @@ impl Compositor { dummy: &wgpu::TextureView, b4: Option<&wgpu::TextureView>, dof: Option<&wgpu::TextureView>, - ) -> (wgpu::Buffer, wgpu::BindGroup) { + ) -> LayerBind { let uniform = self.gpu.device.create_buffer_init(&wgpu::util::BufferInitDescriptor { label: Some("layer-uniform"), contents: layer_bytes(cb), @@ -1338,9 +1430,29 @@ impl Compositor { binding: 6, resource: wgpu::BindingResource::TextureView(dof.unwrap_or(dummy)), }, + wgpu::BindGroupEntry { binding: 7, resource: self.prism_mesh.as_entire_binding() }, ], }); - (uniform, bind) + LayerBind { _buf: uniform, bind, models: cb.needs_models() } + } + + /// Dessine un calque par le pipeline plat que l'appelant a lie (`pipeline`, ou `pipeline_add` + /// si `add`) -- sauf un modele 3D, qui passe le temps de son draw par la variante + /// « modeles » de ce pipeline, la seule a le compiler (pourquoi : le haut de `layer.wgsl`). + fn draw_layer(&self, rpass: &mut wgpu::RenderPass<'_>, layer: &LayerBind, add: bool) { + rpass.set_bind_group(0, &layer.bind, &[]); + if layer.models { + let (models, flat) = if add { + (&self.pipeline_add_models, &self.pipeline_add) + } else { + (&self.pipeline_models, &self.pipeline) + }; + rpass.set_pipeline(models); + rpass.draw(0..4, 0..1); + rpass.set_pipeline(flat); + } else { + rpass.draw(0..4, 0..1); + } } /// Charge un PNG/JPEG (chemin fichier ou data URI) en texture RGBA8. Port @@ -1484,7 +1596,8 @@ impl Compositor { /// son propre repli, et un echec silencieux redonnerait le noir qu'on corrige. /// /// `motion` anime l'image au temps programme `programme_t` ; la bulle webcam - /// passe `WallpaperMotion::None`. + /// passe `WallpaperMotion::None`. `transparency` = `layer_fx.x` (0 = opaque): a camera + /// layer's background box fades with the layer. #[allow(clippy::too_many_arguments)] fn image_bg_draw( &self, @@ -1495,6 +1608,7 @@ impl Compositor { aspect: f32, motion: WallpaperMotion, programme_t: f32, + transparency: f32, dummy: &wgpu::TextureView, ) -> Result { let (tex, iw, ih) = self.cached_image(path)?; @@ -1516,11 +1630,12 @@ impl Compositor { mode: 6.0, fx: [0.0, 0.0, anim[0], anim[1]], mb, + layer_fx: [transparency, 0.0, 0.0, 0.0], ..Default::default() }; let view = tex.create_view(&wgpu::TextureViewDescriptor::default()); - let (buf, bind) = self.make_bind(&cb, Some((&view, &view, &view)), dummy); - Ok(BgDraw { _buf: buf, _tex: Some(tex), _view: Some(view), bind }) + let layer = self.make_bind(&cb, Some((&view, &view, &view)), dummy); + Ok(BgDraw { layer, _tex: Some(tex), _view: Some(view) }) } /// Prepare le fond du mode « personnalise », peint DANS la bulle webcam juste @@ -1536,25 +1651,27 @@ impl Compositor { /// /// `quad_px` / `radius_px` sont ceux de la bulle : le fond doit epouser ses /// coins arrondis, sinon un rectangle deborde derriere la camera. + /// + /// `transparency` = the camera layer's `layer_fx.x`: the box fades with the camera. fn webcam_bg_draw( &self, bg: Option<&SceneBackground>, dst: [f32; 4], quad_px: [f32; 2], radius_px: f32, + transparency: f32, dummy: &wgpu::TextureView, ) -> BgDraw { const BLACK: [f32; 4] = [0.0, 0.0, 0.0, 1.0]; - let flat = |cb: LayerCB| { - let (buf, bind) = self.make_bind(&cb, None, dummy); - BgDraw { _buf: buf, _tex: None, _view: None, bind } - }; + let layer_fx = [transparency, 0.0, 0.0, 0.0]; + let flat = |cb: LayerCB| BgDraw { layer: self.make_bind(&cb, None, dummy), _tex: None, _view: None }; let solid = |color: [f32; 4]| LayerCB { dst, quad_px, radius_px, mode: 1.0, color, + layer_fx, ..Default::default() }; match bg { @@ -1572,6 +1689,7 @@ impl Compositor { quad_px, radius_px, fx: [a.sin(), -a.cos(), 0.0, 0.0], + layer_fx, ..crate::frame_geometry::gradient_layer(stops, offsets, BLACK) }) } @@ -1580,7 +1698,10 @@ impl Compositor { // elle que l'image doit remplir sans etirement. let aspect = if quad_px[1] > 0.0 { quad_px[0] / quad_px[1] } else { 1.0 }; let still = WallpaperMotion::None; - match self.image_bg_draw(path, dst, quad_px, radius_px, aspect, still, 0.0, dummy) { + let drawn = self.image_bg_draw( + path, dst, quad_px, radius_px, aspect, still, 0.0, transparency, dummy, + ); + match drawn { Ok(d) => d, Err(e) => { // Meme contrat que le fond d'ecran : un chemin casse est @@ -1685,7 +1806,7 @@ impl Compositor { // `color.a = 1` n'est pas decoratif : `fs_main` calcule son alpha en // `layer.color.a * alpha_mask`, donc le defaut (0) rendrait un quad // entierement transparent. - let (_uniform, bind) = self.make_bind( + let layer = self.make_bind( &LayerCB { dst: [0.0, 0.0, 1.0, 1.0], src, @@ -1718,8 +1839,7 @@ impl Compositor { occlusion_query_set: None, }); rpass.set_pipeline(&self.pipeline); - rpass.set_bind_group(0, &bind, &[]); - rpass.draw(0..4, 0..1); + self.draw_layer(&mut rpass, &layer, false); } encoder.copy_texture_to_buffer( wgpu::TexelCopyTextureInfo { @@ -2024,6 +2144,11 @@ impl Compositor { *self.webcam_mask.borrow_mut() = None; } + /// Le métrage dans la dernière image composée : ses coins et son warp (`FootageQuad`). + pub fn footage_quad(&self) -> Option { + self.footage.get() + } + /// Rend une frame dans le RT interne. Le screen `screen`/`webcam` sont des /// carriers `linux_frames` ; la geometrie vient de `plan_frame`. Coeur : /// fond uni + ecran cover-fit. `readback_direct` lit ensuite le RT. @@ -2099,6 +2224,7 @@ impl Compositor { timeline_t_override: *self.timeline_time.borrow(), programme_time: *self.programme_time.borrow(), }); + self.footage.set(Some(g.footage_quad([rw, rh]))); // (`wtw`/`wth` sont les dims de la TEXTURE webcam, consommees par le // cover-crop du calque PiP plus bas.) @@ -2210,12 +2336,12 @@ impl Compositor { g.tilt_pixel_trail([rw, rh]), ), }; - // Bind group construit AVANT le pass (doit vivre pendant tout le pass) ; - // `_screen_uniform` garde le buffer uniforme en vie (reference par le bind). + // Bind group construit AVANT le pass (doit vivre pendant tout le pass) ; `LayerBind` + // garde le buffer uniforme en vie (reference par le bind). let dummy = self.dummy_view(); // Binding 6 du mode 8 : la pyramide si l'effet tourne, sinon `dummy` (`k = 0`, le // shader n'y lit rien). - let (_screen_uniform, screen_bind) = self.make_bind_b4( + let screen_bind = self.make_bind_b4( &screen_layer, Some((&sy, &su, &sv)), &dummy, @@ -2291,18 +2417,30 @@ impl Compositor { ) }); + // Le fond de la frame d'avant, s'il est celui qu'on demande (`BackgroundKey`) : la passe 1 + // le recopie au lieu de le redessiner. + let bg_desc = scene_ref.as_ref().map(|s| &s.background); + let bg_cached = self + .bg_cache + .borrow() + .as_ref() + .is_some_and(|c| c.key.matches(bg_desc, cfg.bg_blur, lp.bg_color, [rw, rh])); + let bg_wanted = bg_layer.is_some(); + // Fond (gradient mode 5 OU image mode 6), dessine dans la passe de fond. - let bg_draw = bg_layer.and_then(|bl| match bl { + let bg_draw = bg_layer.filter(|_| !bg_cached).and_then(|bl| match bl { BgLayer::Gradient(cb) => { - let (buf, bind) = self.make_bind(&cb, None, &dummy); - Some(BgDraw { _buf: buf, _tex: None, _view: None, bind }) + Some(BgDraw { layer: self.make_bind(&cb, None, &dummy), _tex: None, _view: None }) } // Le wallpaper couvre tout le cadre, donc dst plein et pas de coins : // `image_bg_draw` sert aussi la bulle webcam, qui elle en a. BgLayer::Image(path, motion) => { let full = [0.0, 0.0, 1.0, 1.0]; let t = g.programme_t; - match self.image_bg_draw(&path, full, [0.0, 0.0], 0.0, rw / rh, motion, t, &dummy) { + let drawn = self.image_bg_draw( + &path, full, [0.0, 0.0], 0.0, rw / rh, motion, t, 0.0, &dummy, + ); + match drawn { Ok(d) => Some(d), Err(e) => { eprintln!("[fond image] \"{path}\" : {e:#}"); @@ -2334,7 +2472,10 @@ impl Compositor { // lancement gracieux, le temps que l'inference rende son premier masque. // // Calcule ICI, avant le draw comme avant l'ombre : les deux en dependent. - let (effect_code, blur_intensity, webcam_bg) = { + // Without layout regions camera 0 is drawn from `w_dst` (`webcam_bg`, `webcam_draw`, + // `webcam_shadow` below); with them every planned layer is (`camera_layer_draws`). + let unplanned = g.camera_layers.is_empty(); + let (effect_code, blur_intensity, custom_bg) = { let has_mask = self.webcam_mask.borrow().is_some(); let effect = scene_ref .as_ref() @@ -2350,24 +2491,16 @@ impl Compositor { // fond. Le shader ne sait peindre qu'une couleur plate sous le // masque ; degrades et images y tombaient sur du noir, et le defaut // EST une image. - Some((code, e)) if code > 2.5 => { - // Sans piste webcam le fond peindrait un rectangle seul dans le - // cadre : il ne se prepare que si la camera se dessine. - let bg = webcam_planes.is_some().then(|| { - self.webcam_bg_draw( - e.background.as_ref(), - g.w_dst, - g.w_px, - g.w_radius, - &dummy, - ) - }); - (1.0, 0.0, bg) - } + Some((code, e)) if code > 2.5 => (1.0, 0.0, Some(e.background.as_ref())), Some((code, e)) => (code, e.blur_intensity.clamp(0.0, 1.0), None), None => (0.0, 0.0, None), } }; + // Sans piste webcam le fond peindrait un rectangle seul dans le cadre : il ne se + // prepare que si la camera se dessine. + let webcam_bg = custom_bg + .filter(|_| unplanned && webcam_planes.is_some()) + .map(|bg| self.webcam_bg_draw(bg, g.w_dst, g.w_px, g.w_radius, 0.0, &dummy)); // L'ombre se juge sur le mode DE LA SCENE, pas sur `effect_code` : le fond // personnalise se compose desormais en detourage (code 1) tout en gardant // sa bulle, et tester le code compose la lui retirerait. Meme lecture que @@ -2376,7 +2509,10 @@ impl Compositor { scene_ref.as_ref().and_then(|s| s.webcam_effect.as_ref()), Some(e) if e.shader_code() == 1.0 ) && self.webcam_mask.borrow().is_some(); - let webcam_draw = webcam_planes.as_ref().map(|(wy, wu, wv)| { + // Camera 0's crop, mirror and desk turn as camera settings, plus its homography + // (`Scene::camera(0)`): with one, `src` is not read and the crop is dropped. + let cam0 = g.camera0_settings(scene_ref.as_ref()); + let webcam_draw = webcam_planes.as_ref().filter(|_| unplanned).map(|(wy, wu, wv)| { // COVER-CROP. `src` etait cable a [0,0,1,1], donc la texture entiere // etait etiree sur la boite quelle que soit sa forme : le facteur de // deformation valait exactement `box_ar / cam_ar`. Invisible en PiP @@ -2386,45 +2522,27 @@ impl Compositor { // // `cover_crop_uv` est la primitive partagee que macOS et Windows // utilisent ; elle rend le rect inchange quand il a deja le bon - // ratio, donc aucun placement correct ne bouge. - let [cu0, cv0, cu1, cv1] = crate::frame_geometry::webcam_source_rect( + // ratio, donc aucun placement correct ne bouge. Mirror and the desk-shot + // turn are both bound swaps: u for horizontal, v for vertical. + let src = crate::frame_geometry::camera_source_rect( + Some(&cam0), [wcw, wch], [wtw as f32, wth as f32], - scene_ref - .as_ref() - .and_then(|scene| scene.layout.webcam_crop), g.w_px[0] / g.w_px[1].max(0.0001), ); - // MIROIR : on inverse l'intervalle u. Le VS interpole `src` - // lineairement et `fs_main` ne re-clampe pas `i.uv`, donc un - // intervalle a l'envers suffit -- aucune retouche du WGSL. Apres le - // cover-crop les deux bornes sont strictement a l'interieur de la - // texture, donc le sampler ClampToEdge ne bave pas sur les bords. - let (u0, u1) = if lp.webcam_mirror { (cu1, cu0) } else { (cu0, cu1) }; // `src_prev` doit valoir EXACTEMENT le `src` de ce draw, miroir // compris : le shader s'en sert pour reconstruire l'UV de la frame // precedente, et un rect source qui ne correspond pas au calque // dessine ferait diverger la trainee vers une zone de la texture qui // n'a jamais ete affichee. Seul `dst_prev` porte le mouvement. - let cb = LayerCB { - dst: g.w_dst, - src: [u0, cv0, u1, cv1], - quad_px: g.w_px, - radius_px: g.w_radius, - mode: 0.0, - // `color.a` porte l'alpha du decoupage (`color.a * personne`) ; le - // RGB n'est plus lu, le fond ayant deja ete peint sous la camera. - color: [0.0, 0.0, 0.0, 1.0], - // `fx.xy` = etendue valide de la texture webcam, par quoi le - // shader divise `uv` pour retomber dans l'espace du masque ; - // `fx.z` = mode, `fx.w` = intensite du flou. Contrat commun aux - // trois back-ends, cf. `layer.wgsl` et `webcam-segmentation.md`. - fx: [w_valid[0], w_valid[1], effect_code, blur_intensity], - src_prev: [u0, cv0, u1, cv1], - dst_prev: g.w_dst_prev, - mb: [g.mb_taps, g.mb_amount, 1.0, 0.0], - ..Default::default() - }; + // `fx.xy` = etendue valide de la texture webcam, par quoi le + // shader divise `uv` pour retomber dans l'espace du masque ; + // `fx.z` = mode, `fx.w` = intensite du flou. Contrat commun aux + // trois back-ends, cf. `layer.wgsl` et `webcam-segmentation.md`. + let cb = crate::frame_geometry::with_camera_homography( + g.webcam_video_cb(src, w_valid, effect_code, blur_intensity), + Some(&cam0), + ); // Le masque est lie par `make_bind` sur tous les draws, pas seulement // celui-ci : le layout l'exige (cf. `tex_entry(4)`). self.make_bind(&cb, Some((wy, wu, wv)), &dummy) @@ -2458,6 +2576,71 @@ impl Compositor { self.make_bind(&cb, None, &dummy) }); + // Every planned camera layer, in order, each as shadow, then (camera 0) the custom + // background, then the video: one uniform buffer and bind group per draw, drawn in + // the "webcam-pass" below. Camera 0 keeps its frame, effects and mask; the extra + // cameras draw from `set_extra_camera_frames` without effects, and a camera without a + // frame is skipped. `extra_planes` keeps their views alive until the pass is encoded. + let mut camera_layer_draws: Vec = Vec::new(); + let mut extra_planes = Vec::new(); + let render = [rw, rh]; + let cam0_base = g.webcam_video_cb([0.0; 4], w_valid, effect_code, blur_intensity); + let plain = |layer: LayerBind| BgDraw { layer, _tex: None, _view: None }; + for plan in &g.camera_layers { + if plan.camera == 0 { + // `webcam_planes` is already gated on `lp.has_webcam`. + let Some((wy, wu, wv)) = webcam_planes.as_ref() else { continue }; + let video = crate::frame_geometry::camera_layer_cb( + plan, + Some(&cam0), + [wcw, wch], + [wtw as f32, wth as f32], + render, + &cam0_base, + ); + if let Some(shadow) = + g.camera_layer_shadow(plan, &video, render, cfg.shadow, is_cutout) + { + camera_layer_draws.push(plain(self.make_bind(&shadow, None, &dummy))); + } + if let Some(bg) = custom_bg { + let (dst, quad_px, radius_px) = (video.dst, video.quad_px, video.radius_px); + let t = video.layer_fx[0]; + let bg = self.webcam_bg_draw(bg, dst, quad_px, radius_px, t, &dummy); + camera_layer_draws.push(bg); + } + let bind = self.make_bind(&video, Some((wy, wu, wv)), &dummy); + camera_layer_draws.push(plain(bind)); + } else { + let frame = self.extra_camera_frame(plan.camera); + if Self::pixel_buffer_of(frame).is_none() { + continue; + } + let Ok(planes) = self.nv12_srvs(frame) else { continue }; + let (tw, th) = self.tex_dims(frame); + let visible = [(*frame).width as f32, (*frame).height as f32]; + let tex = [tw as f32, th as f32]; + let valid = [visible[0] / tex[0].max(1.0), visible[1] / tex[1].max(1.0)]; + let video = crate::frame_geometry::camera_layer_cb( + plan, + scene_ref.as_ref().and_then(|s| s.camera(plan.camera)), + visible, + tex, + render, + &crate::frame_geometry::extra_camera_base_cb(valid), + ); + if let Some(shadow) = + g.camera_layer_shadow(plan, &video, render, cfg.shadow, false) + { + camera_layer_draws.push(plain(self.make_bind(&shadow, None, &dummy))); + } + let (ey, eu, ev) = &planes; + let bind = self.make_bind(&video, Some((ey, eu, ev)), &dummy); + camera_layer_draws.push(plain(bind)); + extra_planes.push(planes); + } + } + // ANNOTATIONS -- calque le plus haut, place relativement au rect ecran // (les coords x/y/w/h de l'annotation sont des fractions de ce rect, cf. // `scene.rs`). Le rect est `g.s_ann`, l'ecran SANS ZOOM, et surtout pas @@ -2475,16 +2658,15 @@ impl Compositor { // replis, meme ordre. Seul le texte diverge, tinte cote shader (atlas R8) // au lieu d'une couleur bakee dans la texture. struct AnnDraw { - _buf: wgpu::Buffer, + layer: LayerBind, /// Gardent l'atlas / la texture image en vie jusqu'au submit. `None` /// pour les quads qui n'echantillonnent rien (plaque de fond, fleche). _glyphs: Option, _tex: Option, - bind: wgpu::BindGroup, } impl AnnDraw { - fn plain(buf: wgpu::Buffer, bind: wgpu::BindGroup) -> AnnDraw { - AnnDraw { _buf: buf, _glyphs: None, _tex: None, bind } + fn plain(layer: LayerBind) -> AnnDraw { + AnnDraw { layer, _glyphs: None, _tex: None } } } // FENETRE TEMPORELLE. Sans ce test, TOUTES les annotations du projet sont @@ -2502,6 +2684,8 @@ impl Compositor { .as_ref() .is_some_and(|s| s.annotations.iter().any(|a| a.kind == "blur" && visible(a))); let mut ann_draws: Vec = Vec::new(); + // Les flous de confidentialite, dessines sur le metrage avant le curseur. + let mut privacy_draws: Vec = Vec::new(); if let Some(scene) = scene_ref.as_ref() { // La liste arrive deja triee par zIndex cote app : l'ordre d'iteration // EST l'ordre de peinture. @@ -2543,8 +2727,7 @@ impl Compositor { mb: [1.0, half_stroke, 0.0, 0.0], ..Default::default() }; - let (buf, bind) = self.make_bind(&cb, None, &dummy); - ann_draws.push(AnnDraw::plain(buf, bind)); + ann_draws.push(AnnDraw::plain(self.make_bind(&cb, None, &dummy))); } "blur" => { let Some(blur) = a.blur.as_ref() else { continue }; @@ -2587,12 +2770,12 @@ impl Compositor { }; // La copie mipmappee au binding 1 (texY), la ou le mode 10 // la lit. - let (buf, bind) = self.make_bind( + let layer = self.make_bind( &cb, Some((&self.ann_copy_view, &self.ann_copy_view, &self.ann_copy_view)), &dummy, ); - ann_draws.push(AnnDraw::plain(buf, bind)); + privacy_draws.push(AnnDraw::plain(layer)); } "image" => { let Some(src) = a.image_path.as_ref().filter(|s| !s.is_empty()) else { @@ -2649,13 +2832,8 @@ impl Compositor { fx: [0.0, 0.0, 1.0, 1.0], ..Default::default() }; - let (buf, bind) = self.make_bind(&cb, Some((&view, &view, &view)), &dummy); - ann_draws.push(AnnDraw { - _buf: buf, - _glyphs: None, - _tex: Some(tex), - bind, - }); + let layer = self.make_bind(&cb, Some((&view, &view, &view)), &dummy); + ann_draws.push(AnnDraw { layer, _glyphs: None, _tex: Some(tex) }); } "text" => { let Some(raster) = self.text_raster.as_ref() else { continue }; @@ -2663,6 +2841,13 @@ impl Compositor { if text.content.trim().is_empty() { continue; } + // The desk label shows only while the camera is covered: skip the rest of + // its section before rasterizing anything. + if crate::text_anim::is_desk_cover(text.animation.as_deref()) + && g.webcam_cover <= 0.0 + { + continue; + } let color = parse_hex(&text.color).unwrap_or([1.0, 1.0, 1.0, 1.0]); let background = parse_hex(&text.background_color).unwrap_or([0.0, 0.0, 0.0, 0.0]); @@ -2697,10 +2882,11 @@ impl Compositor { // et remis a l'echelle de la sortie, comme la taille de // police : en px absolus la meme animation sauterait deux // fois plus haut dans un rendu 4K que dans l'apercu. - let anim = crate::text_anim::text_animation_state( + let anim = crate::text_anim::annotation_text_state( text.animation.as_deref(), (g.source_t - a.start_sec as f32) * 1000.0, ((a.end_sec - a.start_sec) * 1000.0) as f32, + g.webcam_cover, ); let anim_px = rh / crate::text_anim::ANIMATION_REFERENCE_HEIGHT; let (mut ax, mut ay, mut aw, mut ah) = ( @@ -2784,8 +2970,7 @@ impl Compositor { ), ..Default::default() }; - let (pbuf, pbind) = self.make_bind(&plate, None, &dummy); - ann_draws.push(AnnDraw::plain(pbuf, pbind)); + ann_draws.push(AnnDraw::plain(self.make_bind(&plate, None, &dummy))); } } @@ -2798,14 +2983,9 @@ impl Compositor { ..Default::default() }; // Atlas R8 au binding 1 (texY) que le mode 11 echantillonne. - let (buf, bind) = + let layer = self.make_bind(&cb, Some((&glyphs.view, &glyphs.view, &glyphs.view)), &dummy); - ann_draws.push(AnnDraw { - _buf: buf, - _glyphs: Some(glyphs), - _tex: None, - bind, - }); + ann_draws.push(AnnDraw { layer, _glyphs: Some(glyphs), _tex: None }); } _ => {} } @@ -2822,12 +3002,18 @@ impl Compositor { // sont PAS dessinees sur le RT mais dans `accum`, puis compositees en une // fois -- cf. le commentaire au point de dessin. struct CursorDraw { - _bufs: Vec, /// Le sprite, et pour le curseur modelise son champ de distance. _tex: Vec<(wgpu::Texture, wgpu::TextureView)>, - binds: Vec, + /// Une copie par echantillon de la trainee, le sprite (modes 7 et 13) ou le curseur + /// modelise (mode 15). + binds: Vec, + /// L'union des quads de `binds` (x0, y0, x1, y1 en fractions de la sortie) : tout ce + /// que la trainee peint dans `accum`, donc tout ce que sa recopie doit relire. + bounds: [f32; 4], /// L'impact des clics (mode 16), dessine SOUS le curseur, sur le RT. - impacts: Vec, + impacts: Vec, + /// Le cristal de Prism Glow : il lit la copie de l'image composee (`CursorPlan::glass`). + glass: bool, } let cursor_draw: Option = (|| { let track = cursor_ref.as_ref()?; @@ -2870,13 +3056,18 @@ impl Compositor { }) .collect() }; - let (mut bufs, mut binds) = (Vec::new(), Vec::new()); - let mut impacts = Vec::new(); - for cb in &plan.impacts { - let (buf, bind) = self.make_bind(cb, None, &dummy); - bufs.push(buf); - impacts.push(bind); - } + let mut binds = Vec::new(); + let mut bounds = [f32::MAX, f32::MAX, f32::MIN, f32::MIN]; + let mut grow = |dst: [f32; 4]| { + bounds = [ + bounds[0].min(dst[0]), + bounds[1].min(dst[1]), + bounds[2].max(dst[0] + dst[2]), + bounds[3].max(dst[1] + dst[3]), + ]; + }; + let impacts: Vec = + plan.impacts.iter().map(|cb| self.make_bind(cb, None, &dummy)).collect(); let sprites = scene_ref .as_ref() @@ -2906,7 +3097,9 @@ impl Compositor { let view = tex.create_view(&wgpu::TextureViewDescriptor::default()); // Curseur modelise (mode 15) : ce meme sprite extrude, `plan_cursor` en a tire la - // pose. Sprite au binding 1 (texY), champ au binding 2 (texU). Parite Windows/macOS. + // pose. La copie de l'image composee au binding 1 (texY), que refracte le cristal de + // Prism Glow, le champ au binding 4 (texMask), le sprite au binding 6 (texDof). + // Parite Windows/macOS. if let Some(pose) = plan.model { match self.cursor_sdf(&sprite.path) { Ok((sdf, shape)) => { @@ -2923,16 +3116,21 @@ impl Compositor { ) else { continue; }; - let (buf, bind) = - self.make_bind(&cb, Some((&view, &sdf_view, &view)), &dummy); - bufs.push(buf); - binds.push(bind); + binds.push(self.make_bind_b4( + &cb, + Some((&self.ann_copy_view, &dummy, &dummy)), + &dummy, + Some(&sdf_view), + Some(&view), + )); + grow(cb.dst); } return (!binds.is_empty()).then_some(CursorDraw { - _bufs: bufs, _tex: vec![(tex, view), (sdf, sdf_view)], binds, + bounds, impacts, + glass: plan.glass, }); } Err(e) => eprintln!("[curseur] champ de \"{}\" : {e:#}", sprite.path), @@ -2952,11 +3150,16 @@ impl Compositor { [rw, rh], ); // Sprite RGBA au binding 1 (texY) que le mode 7 echantillonne. - let (buf, bind) = self.make_bind(&cb, Some((&view, &view, &view)), &dummy); - bufs.push(buf); - binds.push(bind); + binds.push(self.make_bind(&cb, Some((&view, &view, &view)), &dummy)); + grow(cb.dst); } - Some(CursorDraw { _bufs: bufs, _tex: vec![(tex, view)], binds, impacts }) + Some(CursorDraw { + _tex: vec![(tex, view)], + binds, + bounds, + impacts, + glass: false, + }) })(); // Bind group de la passe de composition d'`accum` (layout du blur : // uniform + texture + sampler). Construit hors de la pass, comme les @@ -2998,7 +3201,7 @@ impl Compositor { }); // Passe 0 : pyramide de profondeur de champ, niveau 0 vide d'abord (le « over » // rend alors la source telle quelle), puis ses mips. - if let (Some(p), Some((_buf, bind))) = (dof_pyramid, &dof_fill) { + if let (Some(p), Some(fill)) = (dof_pyramid, &dof_fill) { { let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { label: Some("dof-pyramid-pass"), @@ -3015,8 +3218,7 @@ impl Compositor { occlusion_query_set: None, }); rpass.set_pipeline(&self.pipeline); - rpass.set_bind_group(0, bind, &[]); - rpass.draw(0..4, 0..1); + self.draw_layer(&mut rpass, fill, false); } self.generate_mips(&mut encoder, &p.mips); } @@ -3039,38 +3241,74 @@ impl Compositor { rpass.set_pipeline(&self.pipeline); self.draw_screen_group(&mut rpass, &screen_shadow, &window_frame, &screen_bind, &device_frame); } - // Passe 1 : fond (clear a `bg_clear` + gradient mode 5 eventuel). - { - let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { - label: Some("bg-pass"), - color_attachments: &[Some(wgpu::RenderPassColorAttachment { - view: &self.rt_view, - resolve_target: None, - ops: wgpu::Operations { - load: wgpu::LoadOp::Clear(wgpu::Color { - r: bg_clear[0] as f64, - g: bg_clear[1] as f64, - b: bg_clear[2] as f64, - a: bg_clear[3] as f64, - }), - store: wgpu::StoreOp::Store, - }, - })], - depth_stencil_attachment: None, - timestamp_writes: None, - occlusion_query_set: None, - }); - if let Some(bg) = &bg_draw { - rpass.set_pipeline(&self.pipeline); - rpass.set_bind_group(0, &bg.bind, &[]); - rpass.draw(0..4, 0..1); + // Passe 1 : fond (clear a `bg_clear` + gradient mode 5 eventuel) puis son flou -- ou, s'il + // n'a pas change (`BackgroundKey`), la copie qu'en a gardee la frame d'avant. + let extent = wgpu::Extent3d { + width: self.render_w, + height: self.render_h, + depth_or_array_layers: 1, + }; + if bg_cached { + let cache = self.bg_cache.borrow(); + let tex = &cache.as_ref().expect("bg_cached").tex; + encoder.copy_texture_to_texture(tex.as_image_copy(), self.rt.as_image_copy(), extent); + } else { + { + let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("bg-pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &self.rt_view, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color { + r: bg_clear[0] as f64, + g: bg_clear[1] as f64, + b: bg_clear[2] as f64, + a: bg_clear[3] as f64, + }), + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + }); + if let Some(bg) = &bg_draw { + rpass.set_pipeline(&self.pipeline); + self.draw_layer(&mut rpass, &bg.layer, false); + } + } + // Blur du fond (avant l'ecran), si active par la scene/l'inspector. + self.blur_bg(&mut encoder, cfg.bg_blur); + // Pas de cache pour un fond anime, ni pour un fond qui n'a pas pu se charger : la frame + // suivante reessaie. + let key = (bg_draw.is_some() == bg_wanted) + .then(|| { + crate::frame_geometry::BackgroundKey::of(bg_desc, cfg.bg_blur, lp.bg_color, [rw, rh]) + }) + .flatten(); + if let Some(key) = key { + let mut cache = self.bg_cache.borrow_mut(); + let tex = match cache.take() { + Some(c) if c.tex.size() == extent => c.tex, + _ => self.gpu.device.create_texture(&wgpu::TextureDescriptor { + label: Some("bg-cache"), + size: extent, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rgba8Unorm, + usage: wgpu::TextureUsages::COPY_SRC | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }), + }; + encoder.copy_texture_to_texture(self.rt.as_image_copy(), tex.as_image_copy(), extent); + *cache = Some(BgCache { key, tex }); } } - // Blur du fond (avant l'ecran), si active par la scene/l'inspector. - self.blur_bg(&mut encoder, cfg.bg_blur); - // Passe 2 : avant-plan (ecran + webcam), compose par-dessus le fond - // (eventuellement floute) avec `LoadOp::Load`. Les annotations sont dans - // une passe a part, cf. plus bas. + // Passe 2 : l'ecran, compose par-dessus le fond (eventuellement floute) avec + // `LoadOp::Load`. Puis les flous de confidentialite, le curseur, la camera et les autres + // annotations, dans cet ordre : celui de Windows et macOS. { let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { label: Some("fg-pass"), @@ -3091,10 +3329,7 @@ impl Compositor { // elle doit passer sous lui mais au-dessus du fond (et, pour la // camera, au-dessus de l'ecran). match &trail { - Some((_buf, bind)) => { - rpass.set_bind_group(0, bind, &[]); - rpass.draw(0..4, 0..1); - } + Some(layer) => self.draw_layer(&mut rpass, layer, false), None => self.draw_screen_group( &mut rpass, &screen_shadow, @@ -3103,40 +3338,22 @@ impl Compositor { &device_frame, ), } - if let Some((_buf, bind)) = &webcam_shadow { - rpass.set_bind_group(0, bind, &[]); - rpass.draw(0..4, 0..1); - } - // Fond personnalise : ENTRE l'ombre et la camera. C'est ce sandwich qui - // remplace la branche « mode 3 » du shader — la camera, decoupee, se - // fond dessus par alpha ; l'ombre reste dessous, elle appartient a la - // bulle et non a son contenu. - if let Some(bg) = &webcam_bg { - rpass.set_bind_group(0, &bg.bind, &[]); - rpass.draw(0..4, 0..1); - } - if let Some((_buf, bind)) = &webcam_draw { - rpass.set_bind_group(0, bind, &[]); - rpass.draw(0..4, 0..1); - } } // Fige la frame composee pour les annotations « flou ». ICI et nulle part - // ailleurs : apres l'ecran et la camera (sinon un flou masquerait du vide) - // et avant la premiere annotation (sinon deux flous qui se recouvrent + // ailleurs : apres l'ecran (sinon un flou masquerait du vide) + // et avant le premier flou (sinon deux flous qui se recouvrent // s'echantillonnent l'un l'autre). Une passe de rendu ne peut pas lire sa // propre cible, d'ou la copie -- et d'ou le fait que les annotations // doivent avoir leur propre passe. if needs_ann_copy { self.generate_ann_mips(&mut encoder); } - // Passe 3 : annotations puis curseur net, par-dessus tout le reste. Elle - // existe meme sans flou : deux passes consecutives sur la MEME cible avec - // `LoadOp::Load` ne coutent rien de plus qu'une seule sur un GPU - // desktop, et un seul chemin de code vaut mieux qu'un branchement qui ne - // serait exerce que dans un projet sur dix. - { + // Passe 2 bis : les flous de confidentialite, sur le metrage, AVANT le curseur (parite + // Windows et macOS). Le curseur reste net par-dessus, et le cristal de Prism Glow, qui + // refracte l'image composee, n'y voit que des pixels deja floutes. + if !privacy_draws.is_empty() { let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { - label: Some("ann-pass"), + label: Some("privacy-pass"), color_attachments: &[Some(wgpu::RenderPassColorAttachment { view: &self.rt_view, resolve_target: None, @@ -3150,21 +3367,44 @@ impl Compositor { occlusion_query_set: None, }); rpass.set_pipeline(&self.pipeline); - for a in &ann_draws { - rpass.set_bind_group(0, &a.bind, &[]); - rpass.draw(0..4, 0..1); + for a in &privacy_draws { + self.draw_layer(&mut rpass, &a.layer, false); } + } + // Le cristal de Prism Glow refracte l'image telle qu'elle est composee a cet instant, + // flous compris : sa copie dans `ann_copy` (libre, les flous l'ont deja lue ; le mip 0 + // suffit), une fois pour toutes les copies de la trainee. + if cursor_draw.as_ref().is_some_and(|c| c.glass) { + encoder.copy_texture_to_texture( + self.rt.as_image_copy(), + self.ann_copy.as_image_copy(), + wgpu::Extent3d { width: self.render_w, height: self.render_h, depth_or_array_layers: 1 }, + ); + } + // Le curseur net, et l'impact des clics sous lui. La trainee, elle, a besoin de sa + // propre cible : elle suit. + { + let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("cursor-pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &self.rt_view, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Load, + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + }); + rpass.set_pipeline(&self.pipeline); // L'impact des clics, sous le curseur et sa trainee (dessinee plus bas). - for bind in cursor_draw.iter().flat_map(|c| &c.impacts) { - rpass.set_bind_group(0, bind, &[]); - rpass.draw(0..4, 0..1); + for layer in cursor_draw.iter().flat_map(|c| &c.impacts) { + self.draw_layer(&mut rpass, layer, false); } - // Curseur en dernier : au-dessus de l'ecran et des annotations. - // Une seule copie = curseur net, il tient dans cette pass. La - // trainee, elle, a besoin de sa propre cible (voir plus bas). if let Some(c) = cursor_draw.as_ref().filter(|c| c.binds.len() == 1) { - rpass.set_bind_group(0, &c.binds[0], &[]); - rpass.draw(0..4, 0..1); + self.draw_layer(&mut rpass, &c.binds[0], false); } } // TRAINEE DU CURSEUR : flou REEL, pas des copies discretes. @@ -3200,11 +3440,10 @@ impl Compositor { occlusion_query_set: None, }); rpass.set_pipeline(&self.pipeline_add); - for (k, bind) in c.binds.iter().enumerate() { + for (k, layer) in c.binds.iter().enumerate() { let w = crate::frame_geometry::cursor_tap_weight(k as u32, c.binds.len() as u32) as f64; rpass.set_blend_constant(wgpu::Color { r: w, g: w, b: w, a: w }); - rpass.set_bind_group(0, bind, &[]); - rpass.draw(0..4, 0..1); + self.draw_layer(&mut rpass, layer, true); } } let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { @@ -3221,9 +3460,79 @@ impl Compositor { timestamp_writes: None, occlusion_query_set: None, }); - rpass.set_pipeline(&self.pipeline_copy); - rpass.set_bind_group(0, abind, &[]); - rpass.draw(0..3, 0..1); + // Rien n'a ete peint dans `accum` hors des quads de la trainee : la recopie s'y borne + // au lieu de repasser sur toute la sortie, ce qui coutait 0,5 ms par frame 1080p + // (Radeon 610M) pour un curseur de quelques dizaines de pixels. + let (w, h) = (self.render_w as f32, self.render_h as f32); + let x0 = (c.bounds[0] * w).floor().clamp(0.0, w) as u32; + let y0 = (c.bounds[1] * h).floor().clamp(0.0, h) as u32; + let x1 = (c.bounds[2] * w).ceil().clamp(0.0, w) as u32; + let y1 = (c.bounds[3] * h).ceil().clamp(0.0, h) as u32; + if x1 > x0 && y1 > y0 { + rpass.set_scissor_rect(x0, y0, x1 - x0, y1 - y0); + rpass.set_pipeline(&self.pipeline_copy); + rpass.set_bind_group(0, abind, &[]); + rpass.draw(0..3, 0..1); + } + } + // La camera, au-dessus de l'ecran et du curseur. + { + let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("webcam-pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &self.rt_view, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Load, + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + }); + rpass.set_pipeline(&self.pipeline); + if let Some(layer) = &webcam_shadow { + self.draw_layer(&mut rpass, layer, false); + } + // Fond personnalise : ENTRE l'ombre et la camera. C'est ce sandwich qui + // remplace la branche « mode 3 » du shader — la camera, decoupee, se + // fond dessus par alpha ; l'ombre reste dessous, elle appartient a la + // bulle et non a son contenu. + if let Some(bg) = &webcam_bg { + self.draw_layer(&mut rpass, &bg.layer, false); + } + if let Some(layer) = &webcam_draw { + self.draw_layer(&mut rpass, layer, false); + } + for draw in &camera_layer_draws { + self.draw_layer(&mut rpass, &draw.layer, false); + } + } + // Passe 3 : les autres annotations, par-dessus tout le reste (les flous sont passes avant + // le curseur). Elle existe meme sans annotation : deux passes consecutives sur la MEME + // cible avec `LoadOp::Load` ne coutent rien de plus qu'une seule sur un GPU desktop, et un + // seul chemin de code vaut mieux qu'un branchement qui ne serait exerce que dans un projet + // sur dix. + { + let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("ann-pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &self.rt_view, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Load, + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + }); + rpass.set_pipeline(&self.pipeline); + for a in &ann_draws { + self.draw_layer(&mut rpass, &a.layer, false); + } } self.gpu.context.submit(std::iter::once(encoder.finish())); Ok(()) @@ -4041,7 +4350,7 @@ mod tests { planes: (&wgpu::TextureView, &wgpu::TextureView, &wgpu::TextureView), ) -> (u32, u32, Vec) { let dummy = comp.dummy_view(); - let (_buf, bind) = comp.make_bind(cb, Some(planes), &dummy); + let layer = comp.make_bind(cb, Some(planes), &dummy); let mut encoder = comp.gpu.device.create_command_encoder( &wgpu::CommandEncoderDescriptor { label: Some("test-layer") }, ); @@ -4061,8 +4370,7 @@ mod tests { occlusion_query_set: None, }); rpass.set_pipeline(&comp.pipeline); - rpass.set_bind_group(0, &bind, &[]); - rpass.draw(0..4, 0..1); + comp.draw_layer(&mut rpass, &layer, false); } comp.gpu.context.submit(std::iter::once(encoder.finish())); unsafe { comp.readback_direct().expect("readback_direct") } @@ -4559,12 +4867,37 @@ mod tests { /// `compose_pip` sur une scene deja construite, pour les tests qui la retouchent. fn compose_pip_scene(comp: &Compositor, gpu: &Gpu, scene: Scene, shadow: bool) -> Vec { + let screen = FakeFrame::new(gpu, 128, 128, |_, _| 126); + let webcam = FakeFrame::new(gpu, 64, 64, |_, _| Y_WHITE); + compose_pip_frames(comp, scene, shadow, &screen, &webcam) + } + + /// `compose_pip_scene` with the caller's own screen and camera frames. + fn compose_pip_frames( + comp: &Compositor, + scene: Scene, + shadow: bool, + screen: &FakeFrame, + webcam: &FakeFrame, + ) -> Vec { + compose_pip_frames_with_extras(comp, scene, shadow, screen, webcam, &[]) + } + + /// `compose_pip_frames` with extra camera frames. They are handed over AFTER `set_scene`, + /// which forgets the previous ones, and before the compose, as the product code does. + fn compose_pip_frames_with_extras( + comp: &Compositor, + scene: Scene, + shadow: bool, + screen: &FakeFrame, + webcam: &FakeFrame, + extra: &[*const AVFrame], + ) -> Vec { comp.set_live_params(live_params_from_scene(&scene)); comp.set_has_webcam(true); comp.set_scene(Some(scene)); + unsafe { comp.set_extra_camera_frames(extra) }; - let screen = FakeFrame::new(gpu, 128, 128, |_, _| 126); - let webcam = FakeFrame::new(gpu, 64, 64, |_, _| Y_WHITE); let mut cfg = Cfg::c8(); cfg.bg_blur = 0.0; cfg.zoom = false; @@ -4679,6 +5012,9 @@ mod tests { clip_index: None, start_sec: 2.0, end_sec: 8.0, + rotation: 0, + mirror: None, + full_frame: false, }); // 5 s : la montee (~1 s) est finie, le retour (~1,5 s) n'a pas commence. comp.set_timeline_time(Some(5.0)); @@ -4690,6 +5026,336 @@ mod tests { ); } + /// `pip_scene_json` on a black background: with a black screen too, the camera is drawn over + /// black wherever it lands, so its colour can be read without knowing where the PiP is. + fn black_pip_scene() -> Scene { + let json = pip_scene_json(NO_EFFECT).replace("#0080ff", "#000000"); + Scene::from_json(&json).expect("scene json") + } + + /// The x of every pixel of a `width`-wide RGBA image that `pred` accepts. + fn xs_where(rgba: &[u8], width: usize, pred: impl Fn(&[u8]) -> bool) -> Vec { + rgba.chunks_exact(4) + .enumerate() + .filter(|(_, px)| pred(px)) + .map(|(i, _)| i % width) + .collect() + } + + /// The RGBA of pixel (x, y) of a 320-wide readback. + fn pixel_at(rgba: &[u8], x: usize, y: usize) -> [u8; 4] { + let i = (y * 320 + x) * 4; + [rgba[i], rgba[i + 1], rgba[i + 2], rgba[i + 3]] + } + + /// A 64x64 camera frame of one RGB colour (BT.709 limited, `rgb_to_nv12`). + fn solid_camera(gpu: &Gpu, rgb: [u8; 3]) -> FakeFrame { + let (w, h) = (64u32, 64u32); + let pixels: Vec = (0..w * h).flat_map(|_| rgb).collect(); + let (y, uv) = rgb_to_nv12(&pixels, w, h); + FakeFrame::from_planes(gpu, w, h, &y, &uv) + } + + /// Camera 1 over the whole frame, and camera 0 as a PiP in the bottom-right corner + /// (x 224..304, y 126..171 of a 320x180 frame). + const CAMERA_1_FULL: &str = r#"{"camera":1,"rect":{"x":0,"y":0,"width":1,"height":1},"shape":"rectangle","fillsFrame":true}"#; + const CAMERA_0_PIP: &str = r#"{"camera":0,"rect":{"x":0.7,"y":0.7,"width":0.25,"height":0.25},"shape":"rectangle"}"#; + + /// A camera layout region from 2 s to 8 s with these layers (JSON objects, comma-separated). + fn layout_region(layers: &str) -> crate::scene::SceneCameraLayoutRegion { + serde_json::from_str(&format!(r#"{{"startSec":2.0,"endSec":8.0,"layers":[{layers}]}}"#)) + .expect("region json") + } + + /// The time in the lead-in of `layout_region` where a layer only the region shows is drawn + /// at opacity 0.5 (and one only the default shows, at 0.5 too). Checked against the plan. + fn half_way_in(region: &crate::scene::SceneCameraLayoutRegion) -> f32 { + let (mut lo, mut hi) = (0.0f32, 1.0f32); + for _ in 0..40 { + let mid = 0.5 * (lo + hi); + if crate::regions::ease_out_screen_studio(mid) < 0.5 { + lo = mid; + } else { + hi = mid; + } + } + let t = 2.0 + lo * crate::regions::TRANSITION_WINDOW_S; + let default_cam0 = crate::camera_layers::CameraLayerPlan { + camera: 0, + dst: [0.7, 0.7, 0.25, 0.25], + radius_frac: 0.0, + shape: 0, + opacity: 1.0, + fills_frame: false, + }; + let clock = crate::regions::ScreenClock::new(&[], 0); + let layers = crate::camera_layers::camera_layers_at( + std::slice::from_ref(region), + &[], + t, + &clock, + Some(default_cam0), + ); + assert!( + layers.iter().any(|l| (l.opacity - 0.5).abs() < 1e-3), + "no layer at half opacity at {t}: {layers:?}" + ); + t + } + + /// `compose_pip_frames` at source time `t`, with these extra camera frames set (and cleared + /// again afterwards). + fn compose_layers( + comp: &Compositor, + scene: Scene, + t: f32, + screen: &FakeFrame, + camera_0: &FakeFrame, + extra: &[*const AVFrame], + ) -> Vec { + comp.set_timeline_time(Some(t)); + let rgba = compose_pip_frames_with_extras(comp, scene, false, screen, camera_0, extra); + unsafe { comp.set_extra_camera_frames(&[]) }; + rgba + } + + fn is_green(px: [u8; 4]) -> bool { + px[1] > 200 && px[0] < 60 && px[2] < 60 + } + + /// A `camera-full-pip` region: camera 1 (green) fills the frame, camera 0 (white) is the PiP + /// over it. The centre shows camera 1, the PiP rect camera 0. + #[test] + fn two_cameras_draw_in_their_planned_rects() { + let Some(gpu) = gpu() else { return }; + let comp = Compositor::new_sized(&gpu, 320, 180).expect("Compositor::new_sized"); + let screen = FakeFrame::new(&gpu, 128, 128, |_, _| Y_BLACK); + let white = FakeFrame::new(&gpu, 64, 64, |_, _| Y_WHITE); + let green = solid_camera(&gpu, [0, 255, 0]); + let mut scene = black_pip_scene(); + scene.camera_layout_regions.push(layout_region(&format!("{CAMERA_1_FULL},{CAMERA_0_PIP}"))); + // 5 s: past the lead-in, before the lead-out. + let rgba = compose_layers(&comp, scene, 5.0, &screen, &white, &[green.as_ptr()]); + let centre = pixel_at(&rgba, 160, 90); + assert!(is_green(centre), "the centre is not camera 1: {centre:?}"); + let pip = pixel_at(&rgba, 264, 148); + assert!(pip[..3].iter().all(|&c| c > 240), "the PiP rect is not camera 0: {pip:?}"); + let corner = pixel_at(&rgba, 10, 10); + assert!(is_green(corner), "camera 1 does not fill the frame: {corner:?}"); + } + + /// Without a frame for camera 1 its layer is skipped, empty slice or null pointer alike: + /// the centre shows the screen, as it does without any region. + #[test] + fn a_missing_extra_camera_skips_its_layer() { + let Some(gpu) = gpu() else { return }; + let comp = Compositor::new_sized(&gpu, 320, 180).expect("Compositor::new_sized"); + let screen = FakeFrame::new(&gpu, 128, 128, |_, _| 126); + let white = FakeFrame::new(&gpu, 64, 64, |_, _| Y_WHITE); + let control = compose_layers(&comp, black_pip_scene(), 5.0, &screen, &white, &[]); + let screen_px = pixel_at(&control, 160, 90); + assert!(screen_px[1] > 100 && screen_px[1] < 160, "control: the centre is not the screen"); + + let mut scene = black_pip_scene(); + scene.camera_layout_regions.push(layout_region(&format!("{CAMERA_1_FULL},{CAMERA_0_PIP}"))); + for extra in [&[][..], &[std::ptr::null()][..]] { + let rgba = compose_layers(&comp, scene.clone(), 5.0, &screen, &white, extra); + assert_eq!(pixel_at(&rgba, 160, 90), screen_px, "{} frames set", extra.len()); + let pip = pixel_at(&rgba, 264, 148); + assert!(pip[..3].iter().all(|&c| c > 240), "camera 0 is no longer drawn: {pip:?}"); + } + } + + /// Half-way into the lead-in camera 1 fades in at opacity 0.5: over the black screen the + /// centre is half as green as in the hold. + #[test] + fn a_fading_extra_camera_is_half_transparent() { + let Some(gpu) = gpu() else { return }; + let comp = Compositor::new_sized(&gpu, 320, 180).expect("Compositor::new_sized"); + let screen = FakeFrame::new(&gpu, 128, 128, |_, _| Y_BLACK); + let white = FakeFrame::new(&gpu, 64, 64, |_, _| Y_WHITE); + let green = solid_camera(&gpu, [0, 255, 0]); + let region = layout_region(&format!("{CAMERA_1_FULL},{CAMERA_0_PIP}")); + let t = half_way_in(®ion); + let mut scene = black_pip_scene(); + scene.camera_layout_regions.push(region); + + let hold = compose_layers(&comp, scene.clone(), 5.0, &screen, &white, &[green.as_ptr()]); + let full = pixel_at(&hold, 160, 90); + assert!(is_green(full), "control: the centre is not camera 1 in the hold: {full:?}"); + let half = compose_layers(&comp, scene, t, &screen, &white, &[green.as_ptr()]); + let px = pixel_at(&half, 160, 90); + let expected = full[1] as i32 / 2; + assert!( + (px[1] as i32 - expected).abs() <= 8 && px[0] < 30 && px[2] < 30, + "half-way in: {px:?}, expected green near {expected}" + ); + } + + /// Camera 0 fading out at opacity 0.5 (a region that shows only camera 1, which has no + /// frame) draws the (white) camera at half strength over the (black) screen: every camera + /// pixel is mid grey, none of them is white any more. + #[test] + fn transparency_half_halves_the_camera_over_the_screen() { + let Some(gpu) = gpu() else { return }; + let comp = Compositor::new_sized(&gpu, 320, 180).expect("Compositor::new_sized"); + let screen = FakeFrame::new(&gpu, 128, 128, |_, _| Y_BLACK); + let webcam = FakeFrame::new(&gpu, 64, 64, |_, _| Y_WHITE); + let opaque = compose_pip_frames(&comp, black_pip_scene(), false, &screen, &webcam); + let whole = camera_pixels(&opaque); + assert!(whole > 200, "the camera is not on screen, the test proves nothing"); + + let region = layout_region(CAMERA_1_FULL); + let t = half_way_in(®ion); + let mut scene = black_pip_scene(); + scene.camera_layout_regions.push(region); + let half = compose_layers(&comp, scene, t, &screen, &webcam, &[]); + assert_eq!(camera_pixels(&half), 0, "a half transparent camera still has white pixels"); + let grey = half + .chunks_exact(4) + .filter(|px| px[..3].iter().all(|&c| (126..=130).contains(&c))) + .count(); + assert!( + grey as f32 >= whole as f32 * 0.9, + "{grey} mid-grey pixels for {whole} camera pixels drawn opaque" + ); + } + + /// Camera 0's perspective applies without any layout region. A homography that flips the + /// camera horizontally draws its right half on the left of the layer: a camera red on the + /// left and blue on the right shows blue left of red. The control (no perspective) shows it + /// the other way round, so the flip is the homography's doing. + #[test] + fn camera_0_perspective_applies_without_layout_regions() { + let Some(gpu) = gpu() else { return }; + let comp = Compositor::new_sized(&gpu, 320, 180).expect("Compositor::new_sized"); + let screen = FakeFrame::new(&gpu, 128, 128, |_, _| Y_BLACK); + // BT.709 limited: red = (Y 63, Cb 102, Cr 240), blue = (Y 32, Cb 240, Cr 118). + let (w, h) = (64u32, 64u32); + let y: Vec = (0..w * h).map(|i| if i % w < w / 2 { 63 } else { 32 }).collect(); + let uv: Vec = (0..h / 2) + .flat_map(|_| (0..w / 2).flat_map(|j| if j < w / 4 { [102, 240] } else { [240, 118] })) + .collect(); + let webcam = FakeFrame::from_planes(&gpu, w, h, &y, &uv); + let red = |px: &[u8]| px[0] > 200 && px[1] < 60 && px[2] < 60; + let blue = |px: &[u8]| px[2] > 200 && px[0] < 60 && px[1] < 60; + + let plain = compose_pip_frames(&comp, black_pip_scene(), false, &screen, &webcam); + let (r, b) = (xs_where(&plain, 320, red), xs_where(&plain, 320, blue)); + assert!(r.len() > 50 && b.len() > 50, "control: {} red, {} blue pixels", r.len(), b.len()); + assert!(r.iter().max() < b.iter().min(), "control: the camera's red half is on the left"); + + let mut scene = black_pip_scene(); + assert!(scene.camera_layout_regions.is_empty()); + scene.cameras.push(crate::scene::SceneCamera { + index: 0, + rotation: 0, + mirror: None, + crop: None, + homography: Some([-1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0]), + aspect: None, + }); + let flipped = compose_pip_frames(&comp, scene, false, &screen, &webcam); + let (r, b) = (xs_where(&flipped, 320, red), xs_where(&flipped, 320, blue)); + assert!(r.len() > 50 && b.len() > 50, "flipped: {} red, {} blue pixels", r.len(), b.len()); + assert!(b.iter().max() < r.iter().min(), "flipped: the left half of the layer is blue"); + } + + /// The homography `cam.x = 2 u - 0.5`: the outer quarters of the layer (u < 0.25, u > 0.75) + /// find no camera point, the middle half shows the whole camera stretched. + const PAST_THE_PICTURE: [f32; 9] = [2.0, 0.0, -0.5, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0]; + + fn corrected(index: usize, homography: [f32; 9]) -> crate::scene::SceneCamera { + crate::scene::SceneCamera { + index, + rotation: 0, + mirror: None, + crop: None, + homography: Some(homography), + aspect: None, + } + } + + fn is_opaque_black(px: [u8; 4]) -> bool { + px[..3].iter().all(|&c| c < 20) && px[3] == 255 + } + + /// Camera 0 corrected past its picture (spec: "der Rand wird schwarz"): where the + /// homography finds no camera point the PiP is opaque black, not the blue background + /// behind it. The PiP rect comes from the uncorrected control. + #[test] + fn camera_0_corrected_past_its_picture_is_black_there() { + let Some(gpu) = gpu() else { return }; + let comp = Compositor::new_sized(&gpu, 320, 180).expect("Compositor::new_sized"); + let screen = FakeFrame::new(&gpu, 128, 128, |_, _| 126); + let green = solid_camera(&gpu, [0, 255, 0]); + let scene = Scene::from_json(&pip_scene_json(NO_EFFECT)).expect("scene json"); + + let plain = compose_pip_frames(&comp, scene.clone(), false, &screen, &green); + let pip: Vec<(usize, usize)> = plain + .chunks_exact(4) + .enumerate() + .filter(|(_, px)| is_green([px[0], px[1], px[2], px[3]])) + .map(|(i, _)| (i % 320, i / 320)) + .collect(); + assert!(pip.len() > 200, "control: {} camera pixels", pip.len()); + let x0 = pip.iter().map(|p| p.0).min().unwrap(); + let x1 = pip.iter().map(|p| p.0).max().unwrap(); + let y0 = pip.iter().map(|p| p.1).min().unwrap(); + let y1 = pip.iter().map(|p| p.1).max().unwrap(); + let (w, y) = ((x1 - x0) as f32, (y0 + y1) / 2); + let at = |f: f32| x0 + (w * f).round() as usize; + + let mut scene = scene; + scene.cameras.push(corrected(0, PAST_THE_PICTURE)); + let rgba = compose_pip_frames(&comp, scene, false, &screen, &green); + for f in [0.1, 0.9] { + let px = pixel_at(&rgba, at(f), y); + assert!(is_opaque_black(px), "at {f} of the PiP, past the picture: {px:?}"); + } + let mid = pixel_at(&rgba, at(0.5), y); + assert!(is_green(mid), "the middle of the PiP is not the camera: {mid:?}"); + } + + /// A frame-filling camera layer corrected past its picture: opaque black in the outer + /// quarters (the blue background no longer shows through), the camera in the middle. + /// Half-way into the lead-in the black is drawn at opacity 0.5 like the rest of the layer: + /// what is behind it shows at half strength. + #[test] + fn a_camera_layer_corrected_past_its_picture_is_black_there() { + let Some(gpu) = gpu() else { return }; + let comp = Compositor::new_sized(&gpu, 320, 180).expect("Compositor::new_sized"); + let screen = FakeFrame::new(&gpu, 128, 128, |_, _| 126); + let white = FakeFrame::new(&gpu, 64, 64, |_, _| Y_WHITE); + let green = solid_camera(&gpu, [0, 255, 0]); + let region = layout_region(CAMERA_1_FULL); + let mut scene = Scene::from_json(&pip_scene_json(NO_EFFECT)).expect("scene json"); + scene.cameras.push(corrected(1, PAST_THE_PICTURE)); + let mut layered = scene.clone(); + layered.camera_layout_regions.push(region.clone()); + + let hold = compose_layers(&comp, layered.clone(), 5.0, &screen, &white, &[green.as_ptr()]); + for x in [10, 300] { + let px = pixel_at(&hold, x, 90); + assert!(is_opaque_black(px), "x {x}, past the picture: {px:?}"); + } + let mid = pixel_at(&hold, 160, 90); + assert!(is_green(mid), "the middle is not camera 1: {mid:?}"); + + let t = half_way_in(®ion); + let under = compose_layers(&comp, scene, t, &screen, &white, &[green.as_ptr()]); + let half = compose_layers(&comp, layered, t, &screen, &white, &[green.as_ptr()]); + let (below, px) = (pixel_at(&under, 10, 90), pixel_at(&half, 10, 90)); + assert!(below[..3].iter().any(|&c| c > 60), "control: nothing visible under the layer"); + for (&got, &under) in px[..3].iter().zip(&below[..3]) { + let expected = under as i32 / 2; + assert!( + (got as i32 - expected).abs() <= 8, + "half-way in, past the picture: {px:?}, expected half of {below:?}" + ); + } + } + /// Le tour complet, celui qui a besoin d'ONNX Runtime : capture -> inference /// -> masque -> composite, entraine par `compose_frame` seul. Se saute /// proprement sans la bibliotheque, ce que fait la CI — cf. @@ -4915,12 +5581,23 @@ mod tests { /// Ecran gris sur fond gris moyen, avec ou sans cadre de fenetre (`frame` est insere tel /// quel dans `effects`), droit ou incline (`rotation`, JSON). fn compose_framed(comp: &Compositor, gpu: &Gpu, frame: &str, rotation: &str) -> Vec { + compose_framed_shadow(comp, gpu, frame, rotation, 0.6) + } + + /// `compose_framed`, avec l'ombre de l'ecran `shadow` (0 l'eteint). + fn compose_framed_shadow( + comp: &Compositor, + gpu: &Gpu, + frame: &str, + rotation: &str, + shadow: f32, + ) -> Vec { let json = format!( r##"{{"clips":[{{"screenPath":"/s.mp4","webcamPath":"","sourceStartSec":0,"sourceEndSec":10,"webcamOffsetSec":0,"hasAudio":false}}], "layout":{{"preset":"no-webcam","webcamSize":1,"webcamShape":"rounded", "webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false, "screenRect":{{"x":0.1,"y":0.1,"width":0.8,"height":0.8}}}}, - "effects":{{"padding":0.2,"blur":false,"shadow":0.6,"roundnessFrac":0.03,"motionBlur":0{frame}}}, + "effects":{{"padding":0.2,"blur":false,"shadow":{shadow},"roundnessFrac":0.03,"motionBlur":0{frame}}}, "background":{{"kind":"color","color":"#6070a0"}}, "zoomRegions":[{{"clipIndex":0,"startSec":0,"endSec":10,"scale":1,"focusX":0.5,"focusY":0.5,"rotation":{rotation}}}], "annotations":[], @@ -5203,6 +5880,66 @@ mod tests { assert!(right[2] > 200 && right[0] < 60 && right[1] < 60, "droite {right:?} : bleu attendu"); } + /// Le fond fixe est recopie d'une frame a l'autre (`bg_cache`) au lieu d'etre redessine, et + /// un fond qui change ne laisse rien du precedent : un compositeur qui a rendu A, A, B puis A + /// rend chacun a l'octet comme un compositeur neuf. Un fond anime n'est jamais repris. + #[test] + fn a_static_background_is_reused_and_a_changed_one_redrawn() { + let Some(gpu) = gpu() else { return }; + let screen = FakeFrame::new(&gpu, 640, 360, |_, _| 60); + let webcam = FakeFrame::new(&gpu, 64, 64, |_, _| 60); + let render = |comp: &Compositor, background: &str| { + let json = format!( + r##"{{"clips":[{{"screenPath":"/s.mp4","webcamPath":"","sourceStartSec":0,"sourceEndSec":10,"webcamOffsetSec":0,"hasAudio":false}}], + "layout":{{"preset":"no-webcam","webcamSize":1,"webcamShape":"rounded", + "webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false, + "screenRect":{{"x":0.3,"y":0.3,"width":0.4,"height":0.4}}}}, + "effects":{{"padding":0.4,"blur":false,"shadow":0,"roundnessFrac":0,"motionBlur":0}}, + "background":{background}, + "zoomRegions":[],"annotations":[], + "cursor":{{"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0, + "clipToBounds":false,"theme":"default"}}, + "cropByClip":[null], + "output":{{"width":640,"height":360,"fps":30}}}}"## + ); + let scene = Scene::from_json(&json).expect("scene json"); + comp.set_live_params(live_params_from_scene(&scene)); + comp.set_has_webcam(false); + comp.set_scene(Some(scene)); + let mut cfg = Cfg::c8(); + cfg.bg_blur = 0.0; + cfg.zoom = false; + cfg.layout_anim = false; + cfg.cursor = false; + cfg.shadow = false; + cfg.mblur_n = 1; + unsafe { + comp.compose_frame(screen.as_ptr(), webcam.as_ptr(), 0.0, &cfg) + .expect("compose_frame"); + comp.readback_direct().expect("readback_direct").2 + } + }; + let fresh = || Compositor::new_sized(&gpu, 640, 360).expect("Compositor::new_sized"); + let a = r##"{"kind":"gradient","angleDeg":90,"stops":["rgb(255, 0, 0)","rgb(0, 0, 255)"],"offsets":[0,1]}"##; + let b = r##"{"kind":"color","color":"#00ff00"}"##; + + let comp = fresh(); + let a1 = render(&comp, a); + assert!(comp.bg_cache.borrow().is_some(), "un fond fixe doit etre garde"); + let a2 = render(&comp, a); + let b1 = render(&comp, b); + let a3 = render(&comp, a); + assert!(a1 == a2, "le fond recopie doit etre celui qui a ete dessine"); + assert!(b1 == render(&fresh(), b), "un autre fond ne doit rien garder du precedent"); + assert!(a3 == a1, "revenir au premier fond doit le redessiner a l'identique"); + assert!(a1 != b1, "garde : les deux fonds different"); + + let aurora = r##"{"kind":"gradient","angleDeg":90,"stops":["rgb(255, 0, 0)","rgb(0, 0, 255)"],"offsets":[0,1],"motion":"aurora"}"##; + let comp = fresh(); + render(&comp, aurora); + assert!(comp.bg_cache.borrow().is_none(), "un fond anime ne doit pas etre garde"); + } + /// Le cadre de fenetre (mode 14) se dessine sur Linux comme ailleurs : `"none"` rend /// l'image d'avant a l'octet, le theme clair ajoute une barre claire, droite comme /// inclinee, et les deux themes different. `OPENSCREEN_FRAME_OUT` recoit les PNG. @@ -5313,6 +6050,22 @@ mod tests { } } + /// L'ombre d'un appareil est un modele elle aussi (mode 17, `device_shadow_cb`) : comme + /// l'appareil, elle n'existe que dans `pipeline_models`, et le pipeline plat n'en peindrait + /// rien. Elle doit donc changer l'image, a plat comme incline. + #[test] + fn a_device_frame_casts_its_shadow() { + let Some(gpu) = gpu() else { return }; + let comp = Compositor::new_sized(&gpu, 1280, 720).expect("Compositor::new_sized"); + let laptop = r#","frame":"laptop""#; + for (name, rotation) in [("flat", "null"), ("iso", r#""iso""#)] { + let lit = compose_framed_shadow(&comp, &gpu, laptop, rotation, 0.6); + let bare = compose_framed_shadow(&comp, &gpu, laptop, rotation, 0.0); + let shaded = lit.chunks_exact(4).zip(bare.chunks_exact(4)).filter(|(a, b)| a != b).count(); + assert!(shaded > 2_000, "{name} : l'ombre de l'appareil ne change que {shaded} px"); + } + } + // ----------------------------------------------------------------------- // Curseur modélisé (mode 15) : pendant de `tests/cursor_model_render.rs` (Windows). // ----------------------------------------------------------------------- @@ -5523,6 +6276,22 @@ mod tests { .collect() } + /// Les pixels que couvre un curseur de VERRE (le cristal de Prism Glow, qui laisse voir le + /// contenu et le réfracte) : opaques, ou changés sur les deux teintes sans y être plus sombres + /// (une ombre ne fait qu'assombrir ; le verre réfracte, reflète et luit). + fn model_covered(on_blue: &[u8], on_orange: &[u8], bare: &[u8], bare_orange: &[u8]) -> Vec { + let opaque = model_opaque(on_blue, on_orange, bare); + (0..1280 * 720) + .map(|i| { + let lit = |on: &[u8], under: &[u8]| { + let (a, b) = (&on[i * 4..i * 4 + 3], &under[i * 4..i * 4 + 3]); + a != b && model_luma(a) >= model_luma(b) - 2.0 + }; + opaque[i] || (lit(on_blue, bare) && lit(on_orange, bare_orange)) + }) + .collect() + } + fn model_iou(a: &[bool], b: &[bool]) -> f32 { let inter = a.iter().zip(b).filter(|(x, y)| **x && **y).count(); let union = a.iter().zip(b).filter(|(x, y)| **x || **y).count(); @@ -5592,6 +6361,34 @@ mod tests { } } + /// La trainee du curseur modelise s'accumule par `pipeline_add_models` : le pipeline plat ne + /// compile pas le mode 15 et n'en peindrait rien. En mouvement, flou du curseur au maximum, + /// le modele doit donc bien apparaitre. + #[test] + fn the_modelled_cursor_draws_its_trail() { + let Some(gpu) = gpu() else { return }; + let comp = Compositor::new_sized(&gpu, 1280, 720).expect("Compositor::new_sized"); + let (y, uv) = model_screen_planes(false); + let screen = FakeFrame::from_planes(&gpu, 640, 360, &y, &uv); + // Traverse 60 % de la largeur en une seconde autour de t = 2 s : ~13 px par frame, donc + // plusieurs copies dans `accum`. + let moving = crate::cursor::CursorTrack::new( + vec![(1.5, 0.2, 0.45), (2.5, 0.8, 0.45)], + vec![], + vec![(0.0, "arrow".to_string())], + ); + let json = |show: bool| { + model_scene_json("null", Some(true), "default", show, 3.0) + .replace(r#""motionBlur":0,"clickBounce""#, r#""motionBlur":1,"clickBounce""#) + }; + assert!(json(true).contains(r#""motionBlur":1,"clickBounce""#), "garde : le flou du curseur"); + let bare = compose_model(&comp, &screen, &json(false), &moving); + let trail = compose_model(&comp, &screen, &json(true), &moving); + let drawn = bare.chunks_exact(4).zip(trail.chunks_exact(4)).filter(|(a, b)| a != b).count(); + // Taille 3 : ~1000 px de silhouette a l'arret, etales ici par la trainee. + assert!(drawn > 1000, "trainee du curseur modelise : {drawn} px dessines"); + } + /// Chaque état garde son art et sa silhouette : posé au centre de l'écran (vu de face), le /// modèle couvre ce que couvre le sprite plat, avec ses couleurs ; il porte une ombre en /// l'air et suit l'inclinaison du plan. Le réglage éteint garde le sprite plat. @@ -5681,7 +6478,8 @@ mod tests { /// Pendant de `the_sculpted_cursors_stand_at_the_hotspot` (Windows) : la flèche et la main /// sculptées des thèmes d'origine (`sculpt.rs`) passent par le WGSL, tiennent au hotspot, en - /// bas à droite de lui, et portent leur ombre en l'air. + /// bas à droite de lui, et portent leur ombre en l'air. Le cristal de Prism Glow est de verre : + /// son corps compte ce qu'il couvre, et une bonne part de lui change avec la teinte de l'écran. #[test] fn the_sculpted_cursors_stand_at_the_hotspot() { let Some(gpu) = gpu() else { return }; @@ -5693,6 +6491,7 @@ mod tests { let extruded = model_scene_json("null", Some(true), "default", true, 5.0); let hidden = model_scene_json("null", Some(true), "default", false, 5.0); let bare = compose_model(&comp, &blue, &hidden, &model_track("arrow", false, 0.5)); + let bare_orange = compose_model(&comp, &orange, &hidden, &model_track("arrow", false, 0.5)); let mut failures = Vec::new(); for state in ["arrow", "pointer"] { let still = model_track(state, false, 0.5); @@ -5707,7 +6506,12 @@ mod tests { failures.push(format!("{theme}/{state}: le sprite extrude au lieu du modele")); } let (tip, u) = model_tip(&json, &still); - let mask = model_opaque(&hover, &hover_b, &bare); + let glass = theme == "prism-glow"; + let mask = if glass { + model_covered(&hover, &hover_b, &bare, &bare_orange) + } else { + model_opaque(&hover, &hover_b, &bare) + }; let (mut body, mut c, mut near) = (0usize, [0.0f32; 2], f32::MAX); for (i, _) in mask.iter().enumerate().filter(|(_, m)| **m) { let (x, y) = ((i % 1280) as f32, (i / 1280) as f32); @@ -5735,6 +6539,124 @@ mod tests { if shadow * 10 < body * 3 { failures.push(format!("{theme}/{state}: pas d'ombre en l'air ({shadow} px)")); } + if glass { + let through = (0..1280 * 720).filter(|&i| mask[i] && hover[i * 4..i * 4 + 3] != hover_b[i * 4..i * 4 + 3]).count(); + println!("{theme}/{state} : {through} px du corps laissent voir l'ecran"); + if through * 10 < body * 3 { + failures.push(format!("{theme}/{state}: le cristal ne laisse voir l'ecran que sur {through} px")); + } + } + } + } + assert!(failures.is_empty(), "{failures:#?}"); + } + + /// Le métrage, puis ses flous de confidentialité, puis le curseur, net par-dessus (parité + /// Windows et macOS) : le cristal de Prism Glow réfracte l'image déjà floutée. Sous le flou, + /// des rayures fines décalées d'une colonne donnent la même image floutée ; si le verre lisait + /// la vidéo brute, il montrerait les rayures, et les deux rendus différeraient à travers lui. + #[test] + fn the_crystal_refracts_the_blurred_picture() { + let Some(gpu) = gpu() else { return }; + let comp = Compositor::new_sized(&gpu, 1280, 720).expect("Compositor::new_sized"); + // Moitié gauche grise, moitié droite (la zone floutée) rayée une colonne sur deux. + let (w, h) = (640u32, 360u32); + let stripes = |phase: u32| { + let y: Vec = (0..w * h) + .map(|i| if i % w < w / 2 { 150 } else if (i % w + phase) % 2 == 0 { 40 } else { 220 }) + .collect(); + FakeFrame::from_planes(&gpu, w, h, &y, &vec![128; (w * (h / 2)) as usize]) + }; + let (a, b) = (stripes(0), stripes(1)); + let blur = r#""annotations":[{"id":"b","startSec":0,"endSec":10,"kind":"blur","x":0.5,"y":0.0,"w":0.5,"h":1.0,"blur":{"style":"blur","shape":"rectangle","color":"white","intensity":24,"blockSize":16}}]"#; + let json = |show: bool| { + model_scene_json("null", Some(true), "default", show, 5.0) + .replace(r#"/arrow.png","#, r#"/arrow.png","sculpt":"prism-glow/arrow","#) + .replace(r#""annotations":[]"#, blur) + }; + let mut failures = Vec::new(); + // Tout entier dans la zone, puis à cheval sur son bord. + for x in [0.7f32, 0.48] { + let track = crate::cursor::CursorTrack::new(vec![(0.0, x, 0.45), (9.0, x, 0.45)], vec![], vec![(0.0, "arrow".to_string())]); + let (ra, rb) = (compose_model(&comp, &a, &json(true), &track), compose_model(&comp, &b, &json(true), &track)); + let bare = compose_model(&comp, &a, &json(false), &track); + model_save(&format!("glass-over-blur-{x}"), &ra); + let px = |img: &[u8], i: usize, c: usize| img[i * 4 + c] as i32; + let hidden = (0..1280 * 720).filter(|&i| (0..3).any(|c| (px(&ra, i, c) - px(&rb, i, c)).abs() > 24)).count(); + // Le serti net du curseur : des sauts francs, entre voisins, de ce qu'il change. + let d = |i: usize, c: usize| px(&ra, i, c) - px(&bare, i, c); + let sharp = (0..1280 * 720 - 1) + .filter(|&i| i % 1280 < 1279 && (0..3).any(|c| (d(i, c) - d(i + 1, c)).abs() > 60)) + .count(); + println!("cristal en x = {x} : {sharp} bords francs, {hidden} px qui suivent les rayures cachées"); + if sharp < 50 { + failures.push(format!("x = {x} : {sharp} bords francs, le curseur n'est pas net par-dessus le flou")); + } + if hidden > 0 { + failures.push(format!("x = {x} : {hidden} px montrent les rayures cachées sous le flou")); + } + } + assert!(failures.is_empty(), "{failures:#?}"); + } + + /// Les thèmes cerclés gardent le trait de leur dessin (`design/cursors/`) : sur la + /// fleche comme sur la main, le modele est pour une bonne part de la couleur du trait (le + /// plateau, le jonc, les rainures entre les doigts), autour de la couleur du corps, et montre + /// ce qu'il porte devant : l'etoile jaune de Star Sprout, le calque et les tirets jaunes de la + /// fleche de Pop Coral, les tirets corail de sa main. Les parts comptent les pixels francs : + /// ceux que l'antialiasing mêle aux bords n'entrent dans aucune. + #[test] + fn the_rimmed_models_keep_the_outline_of_their_art() { + type Rgb = [i32; 3]; + let navy = |[r, g, b]: Rgb| b > r + 20 && r < 110 && g < 130; + let black = |[r, g, b]: Rgb| r.max(g).max(b) < 70; + let mint = |[r, g, b]: Rgb| g > 200 && g > r + 15 && b > 150; + let ivory = |[r, g, b]: Rgb| r > 200 && g > 190 && b > 160 && r - b < 70; + let coral = |[r, g, b]: Rgb| r > 200 && g < 150 && b < 140; + let yellow = |[r, g, b]: Rgb| r > 200 && g > 150 && b < 120; + // (thème, état, trait, corps, ornement, parts minimales du trait, du corps, de l'ornement) + let cases: [(&str, &str, &dyn Fn(Rgb) -> bool, &dyn Fn(Rgb) -> bool, &dyn Fn(Rgb) -> bool, [f32; 3]); 6] = [ + ("studio-ink", "arrow", &black, &ivory, &|_| false, [0.3, 0.15, 0.0]), + ("studio-ink", "pointer", &black, &ivory, &|_| false, [0.2, 0.3, 0.0]), + ("pop-coral", "arrow", &navy, &coral, &yellow, [0.2, 0.2, 0.05]), + ("pop-coral", "pointer", &navy, &yellow, &coral, [0.18, 0.3, 0.02]), + ("star-sprout", "arrow", &navy, &mint, &yellow, [0.25, 0.2, 0.03]), + ("star-sprout", "pointer", &navy, &ivory, &yellow, [0.25, 0.2, 0.03]), + ]; + let Some(gpu) = gpu() else { return }; + let comp = Compositor::new_sized(&gpu, 1280, 720).expect("Compositor::new_sized"); + let (y, uv) = model_screen_planes(false); + let blue = FakeFrame::from_planes(&gpu, 640, 360, &y, &uv); + let (y, uv) = model_screen_planes(true); + let orange = FakeFrame::from_planes(&gpu, 640, 360, &y, &uv); + let extruded = model_scene_json("null", Some(true), "default", true, 5.0); + let hidden = model_scene_json("null", Some(true), "default", false, 5.0); + let bare = compose_model(&comp, &blue, &hidden, &model_track("arrow", false, 0.5)); + let mut failures = Vec::new(); + for (theme, state, rim, body, extra, [min_rim, min_body, min_extra]) in cases { + let still = model_track(state, false, 0.5); + let json = extruded + .replace(&format!(r#"/{state}.png","#), &format!(r#"/{state}.png","sculpt":"{theme}/{state}","#)); + let (hover, hover_b) = (compose_model(&comp, &blue, &json, &still), compose_model(&comp, &orange, &json, &still)); + let mask = model_opaque(&hover, &hover_b, &bare); + let (mut total, mut counts) = (0usize, [0usize; 3]); + for (i, _) in mask.iter().enumerate().filter(|(_, m)| **m) { + let p = [hover[i * 4] as i32, hover[i * 4 + 1] as i32, hover[i * 4 + 2] as i32]; + total += 1; + for (k, class) in [rim, body, extra].iter().enumerate() { + counts[k] += usize::from(class(p)); + } + } + let [rim_share, body_share, extra_share] = counts.map(|k| k as f32 / total.max(1) as f32); + println!("{theme}/{state} : {total} px, trait {rim_share:.3}, corps {body_share:.3}, ornement {extra_share:.3}"); + if !(min_rim..0.7).contains(&rim_share) { + failures.push(format!("{theme}/{state}: {rim_share:.3} de trait, il a disparu ou tout mange")); + } + if body_share < min_body { + failures.push(format!("{theme}/{state}: {body_share:.3} de couleur du corps")); + } + if extra_share < min_extra { + failures.push(format!("{theme}/{state}: {extra_share:.3} d'ornement")); } } assert!(failures.is_empty(), "{failures:#?}"); diff --git a/crates/compositor/src/compositor_macos.rs b/crates/compositor/src/compositor_macos.rs index b00a288eef..a9a571e98f 100644 --- a/crates/compositor/src/compositor_macos.rs +++ b/crates/compositor/src/compositor_macos.rs @@ -261,9 +261,14 @@ pub struct Compositor { scene: RefCell>, cursor: RefCell>, cursor_time: RefCell>, + /// Le métrage dans la dernière image composée, que l'éditeur lit avec elle (`live.rs`). + footage: std::cell::Cell>, timeline_time: RefCell>, /// Temps programme (secondes de sortie) — cf. `FrameGeometryInput::programme_time`. programme_time: RefCell>, + /// The frames of cameras 1..=3 (`set_extra_camera_frames`), as addresses (0 = none) so + /// the compositor's auto traits stay what they were without the raw pointers. + extra_camera_frames: std::cell::Cell<[usize; crate::camera_layers::MAX_EXTRA_CAMERAS]>, live_params: RefCell, metal_texture_cache: CVMetalTextureCache, /// Dernier command buffer soumis, gardé pour pouvoir l'attendre AU MOMENT où le CPU lit @@ -316,6 +321,17 @@ pub struct Compositor { pipeline_fs_tex: metal::RenderPipelineState, /// `vs_main` + `ps_main` en additif : les échantillons de traînée du curseur. pipeline_add: metal::RenderPipelineState, + /// `pipeline_main` et `pipeline_add` compilés AVEC les modèles 3D (`LAYER_MODELS`, cf. + /// l'en-tête de `shaders.metal`). Les deux autres ne les ont pas ; `draw_layer` bascule sur + /// ceux-ci le temps du draw d'un modèle (`LayerCB::needs_models`). + pipeline_main_models: metal::RenderPipelineState, + pipeline_add_models: metal::RenderPipelineState, + /// La passe en cours a été ouverte sur `pipeline_add` (cf. `begin_pass`) : dit à + /// `draw_layer` laquelle des deux variantes « modèles » prendre. + layer_add: std::cell::Cell, + /// Le dernier fond fixe composé et sa clé (`BackgroundKey`) : recopié dans le RT tant + /// que la clé tient, au lieu d'être redessiné. + bg_cache: RefCell>, /// Buffer d'accumulation ISOLÉ (transparent) pour la traînée. Accumuler directement sur /// le RT reviendrait à AJOUTER du blanc à ce qui est déjà dessous : sur un fond clair, /// le curseur disparaît. Même raisonnement que côté D3D11. @@ -537,6 +553,18 @@ enum Blend { Add, } +/// `shaders.metal` avec les modèles 3D : la source dont seuls les pipelines de ces calques sont +/// tirés (cf. son en-tête). +fn msl_source_with_models() -> String { + format!("#define LAYER_MODELS 1\n{}", include_str!("shaders.metal")) +} + +/// Le fond tel que la passe de fond (et son flou) l'a laissé dans le RT, et la clé qui le décrit. +struct BgCache { + key: crate::frame_geometry::BackgroundKey, + tex: metal::Texture, +} + /// Un pipeline state à une seule pièce jointe couleur. fn make_pipeline( device: &metal::Device, @@ -661,6 +689,25 @@ impl Compositor { metal::MTLPixelFormat::RGBA8Unorm, Blend::Add, )?; + let library_models = device + .new_library_with_source(&msl_source_with_models(), &metal::CompileOptions::new()) + .map_err(|e| anyhow!("MTLDevice::new_library_with_source (modèles) a échoué : {e}"))?; + let pipeline_main_models = make_pipeline( + device, + &library_models, + "vs_main", + "ps_main", + metal::MTLPixelFormat::RGBA8Unorm, + Blend::Over, + )?; + let pipeline_add_models = make_pipeline( + device, + &library_models, + "vs_main", + "ps_main", + metal::MTLPixelFormat::RGBA8Unorm, + Blend::Add, + )?; let pipeline_kdown = make_pipeline( device, &library, "vs_fs", "ps_kawase_down", metal::MTLPixelFormat::RGBA8Unorm, Blend::Replace, @@ -682,8 +729,10 @@ impl Compositor { scene: RefCell::new(None), cursor: RefCell::new(None), cursor_time: RefCell::new(None), + footage: std::cell::Cell::new(None), timeline_time: RefCell::new(None), programme_time: RefCell::new(None), + extra_camera_frames: std::cell::Cell::new([0; crate::camera_layers::MAX_EXTRA_CAMERAS]), live_params: RefCell::new(LiveParams::default()), metal_texture_cache: cache, last_cmd: RefCell::new(None), @@ -703,6 +752,10 @@ impl Compositor { pipeline_fs_uv, pipeline_fs_tex, pipeline_add, + pipeline_main_models, + pipeline_add_models, + layer_add: std::cell::Cell::new(false), + bg_cache: RefCell::new(None), accum, trail, blur_half, @@ -761,6 +814,7 @@ impl Compositor { blur_quarter, blur_eighth, ann_copy, + bg_cache: RefCell::new(None), ..self }) } @@ -786,6 +840,9 @@ impl Compositor { pub fn set_scene(&self, s: Option) { *self.scene.borrow_mut() = s; + // A new scene can drop the regions that drew an extra camera: forget its frames so a + // stale pointer is never drawn before the next `set_extra_camera_frames`. + self.extra_camera_frames.set([0; crate::camera_layers::MAX_EXTRA_CAMERAS]); } pub fn set_cursor(&self, track: crate::cursor::CursorTrack) { @@ -887,6 +944,26 @@ impl Compositor { /// IOSurface, pas par pointeur Rust. `flush()` est donc la vidange elle-même. pub fn clear_srv_cache(&self) { self.metal_texture_cache.flush(); + // The extra cameras' frames belong to the decoders just closed: forget them too, so + // a stale pointer is never read before the next `set_extra_camera_frames`. + self.extra_camera_frames.set([0; crate::camera_layers::MAX_EXTRA_CAMERAS]); + } + + /// Frames for cameras 1..=3 (index 0 = scene camera 1). A null or missing entry means that + /// camera is not drawn this frame. The compositor keeps the pointers and reads them in every + /// `compose_frame` until the next call, so they must stay valid until then. + pub unsafe fn set_extra_camera_frames(&self, frames: &[*const AVFrame]) { + let mut slots = [0usize; crate::camera_layers::MAX_EXTRA_CAMERAS]; + for (slot, frame) in slots.iter_mut().zip(frames) { + *slot = *frame as usize; + } + self.extra_camera_frames.set(slots); + } + + /// The frame set for scene camera `camera` (1..=3), null when there is none. + fn extra_camera_frame(&self, camera: usize) -> *const AVFrame { + let slots = self.extra_camera_frames.get(); + camera.checked_sub(1).and_then(|k| slots.get(k).copied()).unwrap_or(0) as *const AVFrame } /// Les verbes de dessin, côté Metal. Mêmes noms et mêmes paramètres que leurs @@ -909,7 +986,20 @@ impl Compositor { enc.set_fragment_texture(0, Some(y)); enc.set_fragment_texture(1, Some(uv)); } - enc.draw_primitives(metal::MTLPrimitiveType::TriangleStrip, 0, 4); + // Un modèle 3D passe, le temps de son draw, par la variante « modèles » du pipeline de la + // passe, la seule à le compiler ; le pipeline plat reprend ensuite la main. + if cb.needs_models() { + let (models, flat) = if self.layer_add.get() { + (&self.pipeline_add_models, &self.pipeline_add) + } else { + (&self.pipeline_main_models, &self.pipeline_main) + }; + enc.set_render_pipeline_state(models); + enc.draw_primitives(metal::MTLPrimitiveType::TriangleStrip, 0, 4); + enc.set_render_pipeline_state(flat); + } else { + enc.draw_primitives(metal::MTLPrimitiveType::TriangleStrip, 0, 4); + } } /// Quad de couleur pleine / gradient / ombre — tout ce qui n'échantillonne pas la vidéo. @@ -1087,13 +1177,15 @@ impl Compositor { programme_t: f32, ) -> Result<()> { let full = [0.0, 0.0, 1.0, 1.0]; - self.draw_image_in(enc, path, full, [0.0, 0.0], 0.0, output_aspect, motion, programme_t) + let (aspect, t) = (output_aspect, programme_t); + self.draw_image_in(enc, path, full, [0.0, 0.0], 0.0, aspect, motion, t, 0.0) } /// `draw_image_bg` pour un rect quelconque — la bulle webcam s'en sert avec ses coins /// arrondis. `output_aspect` est le ratio du RECT visé, pas celui de la sortie : le crop /// « cover » se calcule contre la zone qu'on remplit. `motion` anime l'image au temps - /// programme `programme_t` ; la bulle passe `WallpaperMotion::None`. + /// programme `programme_t` ; la bulle passe `WallpaperMotion::None`. `transparency` = + /// `layer_fx.x` (0 = opaque): a camera layer's background box fades with the layer. #[allow(clippy::too_many_arguments)] unsafe fn draw_image_in( &self, @@ -1105,6 +1197,7 @@ impl Compositor { output_aspect: f32, motion: WallpaperMotion, programme_t: f32, + transparency: f32, ) -> Result<()> { let (tex, iw, ih) = self.cached_image(path)?; let ai = iw as f32 / ih.max(1) as f32; @@ -1128,6 +1221,7 @@ impl Compositor { mode: 6.0, fx: [0.0, 0.0, anim[0], anim[1]], mb, + layer_fx: [transparency, 0.0, 0.0, 0.0], ..Default::default() }, ); @@ -1146,6 +1240,8 @@ impl Compositor { /// /// `quad_px` / `radius_px` sont ceux de la bulle : le fond doit épouser ses coins arrondis, /// sinon un rectangle déborde derrière la caméra. + /// + /// `transparency` = the camera layer's `layer_fx.x`: the box fades with the camera. unsafe fn draw_webcam_bg( &self, enc: &metal::RenderCommandEncoderRef, @@ -1153,14 +1249,17 @@ impl Compositor { dst: [f32; 4], quad_px: [f32; 2], radius_px: f32, + transparency: f32, ) { const BLACK: [f32; 4] = [0.0, 0.0, 0.0, 1.0]; + let layer_fx = [transparency, 0.0, 0.0, 0.0]; let solid = |color: [f32; 4]| LayerCB { dst, quad_px, radius_px, mode: 1.0, color, + layer_fx, ..Default::default() }; match bg { @@ -1178,6 +1277,7 @@ impl Compositor { quad_px, radius_px, fx: [a.sin(), -a.cos(), 0.0, 0.0], + layer_fx, ..crate::frame_geometry::gradient_layer(stops, offsets, BLACK) }, ); @@ -1188,7 +1288,9 @@ impl Compositor { let aspect = if quad_px[1] > 0.0 { quad_px[0] / quad_px[1] } else { 1.0 }; let still = WallpaperMotion::None; if let Err(e) = - self.draw_image_in(enc, path, dst, quad_px, radius_px, aspect, still, 0.0) + self.draw_image_in( + enc, path, dst, quad_px, radius_px, aspect, still, 0.0, transparency, + ) { eprintln!("[compositor] fond webcam \"{path}\" : {e:#}"); self.draw_solid(enc, &solid(BLACK)); @@ -1417,6 +1519,37 @@ impl Compositor { } + /// Le fond gardé (`bg_cache`) : copie le RT dans `tex`, ou `tex` dans le RT (`into_rt`). + unsafe fn blit_background(&self, cmd: &metal::CommandBufferRef, tex: &metal::Texture, into_rt: bool) { + let (src, dst) = if into_rt { (tex, &self.rt) } else { (&self.rt, tex) }; + let blit = cmd.new_blit_command_encoder(); + blit.copy_from_texture( + src, 0, 0, + metal::MTLOrigin { x: 0, y: 0, z: 0 }, + metal::MTLSize { width: self.render_w as u64, height: self.render_h as u64, depth: 1 }, + dst, 0, 0, + metal::MTLOrigin { x: 0, y: 0, z: 0 }, + ); + blit.end_encoding(); + } + + /// Fige l'image composée dans `ann_copy` (mip 0), qu'y lisent les flous (mips dérivés, + /// `mips`) et le cristal de Prism Glow. + unsafe fn snapshot_rt(&self, cmd: &metal::CommandBufferRef, mips: bool) { + let blit = cmd.new_blit_command_encoder(); + blit.copy_from_texture( + &self.rt, 0, 0, + metal::MTLOrigin { x: 0, y: 0, z: 0 }, + metal::MTLSize { width: self.render_w as u64, height: self.render_h as u64, depth: 1 }, + &self.ann_copy, 0, 0, + metal::MTLOrigin { x: 0, y: 0, z: 0 }, + ); + if mips { + blit.generate_mipmaps(&self.ann_copy); + } + blit.end_encoding(); + } + /// Annotations : calque le plus haut, ancré sur `s_ann` — le rect écran SANS ZOOM, le /// conteneur que reçoit l'overlay web. Port de `compositor_windows::draw_annotations`. /// @@ -1425,6 +1558,9 @@ impl Compositor { /// grossir les sous-titres sous un zoom (issue #179, puis #397 sur Linux). L'arithmétique /// elle-même vit dans `frame_geometry::annotation_dst_in`, partagée par les trois backends. /// Le flou, lui, se place par `g.privacy_mask` : un masque doit rester sur ce qu'il cache. + /// + /// `privacy` : les flous de confidentialité seuls, dessinés sur le métrage avant le curseur ; + /// sinon toutes les autres annotations, le calque le plus haut. unsafe fn draw_annotations( &self, cmd: &metal::CommandBufferRef, @@ -1432,30 +1568,21 @@ impl Compositor { t: f32, s_ann: [f32; 4], g: &crate::frame_geometry::FrameGeometry, + privacy: bool, ) -> Result<()> { let Some(scene) = scene else { return Ok(()) }; - if scene.annotations.is_empty() { - return Ok(()); - } let (rw, rh) = (self.render_w as f32, self.render_h as f32); let visible = |a: &crate::scene::SceneAnnotation| { - t >= a.start_sec as f32 && t < a.end_sec as f32 + t >= a.start_sec as f32 && t < a.end_sec as f32 && (a.kind == "blur") == privacy }; + if !scene.annotations.iter().any(visible) { + return Ok(()); + } // UNE seule recopie pour toutes les annotations flou de la frame : leur lecture doit // voir l'image composée SANS les flous eux-mêmes, sinon deux zones qui se recouvrent // s'échantillonneraient l'une l'autre selon l'ordre de dessin. - if scene.annotations.iter().any(|a| a.kind == "blur" && visible(a)) { - let blit = cmd.new_blit_command_encoder(); - blit.copy_from_texture( - &self.rt, 0, 0, - metal::MTLOrigin { x: 0, y: 0, z: 0 }, - metal::MTLSize { width: rw as u64, height: rh as u64, depth: 1 }, - &self.ann_copy, 0, 0, - metal::MTLOrigin { x: 0, y: 0, z: 0 }, - ); - // Seul le mip 0 est rempli ; le GPU dérive le reste. - blit.generate_mipmaps(&self.ann_copy); - blit.end_encoding(); + if privacy { + self.snapshot_rt(cmd, true); } let enc = self.begin_pass(cmd, &self.rt, None, &self.pipeline_main)?; @@ -1585,6 +1712,13 @@ impl Compositor { if text.content.trim().is_empty() { continue; } + // The desk label shows only while the camera is covered: skip the rest of + // its section before rasterizing anything. + if crate::text_anim::is_desk_cover(text.animation.as_deref()) + && g.webcam_cover <= 0.0 + { + continue; + } let spec = crate::text::TextSpec { content: text.content.clone(), color: parse_hex(&text.color).unwrap_or([1.0, 1.0, 1.0, 1.0]), @@ -1618,10 +1752,11 @@ impl Compositor { }) else { continue; }; - let anim = crate::text_anim::text_animation_state( + let anim = crate::text_anim::annotation_text_state( text.animation.as_deref(), (t - a.start_sec as f32) * 1000.0, ((a.end_sec - a.start_sec) * 1000.0) as f32, + g.webcam_cover, ); let anim_px = rh / crate::text_anim::ANIMATION_REFERENCE_HEIGHT; let (mut ax, mut ay, mut aw, mut ah) = ( @@ -2058,6 +2193,8 @@ impl Compositor { ca.set_store_action(metal::MTLStoreAction::Store); let enc = cmd.new_render_command_encoder(&desc); enc.set_render_pipeline_state(pipeline); + // La famille du pipeline de calque de cette passe, pour `draw_layer`. + self.layer_add.set(std::ptr::eq(pipeline, &self.pipeline_add)); Ok(enc) } @@ -2066,7 +2203,8 @@ impl Compositor { /// `frame_geometry::cursor_sprite_cb`, partagée avec Windows et Linux. /// /// Avec `model`, le même sprite extrudé (mode 15, `cursor_model_cb`) : le sprite en - /// texture(2), son champ de distance en texture(4). + /// texture(2), son champ de distance en texture(4), l'enregistrement que l'appelant a lié en + /// texture(0) et (1). #[allow(clippy::too_many_arguments)] unsafe fn draw_cursor_sprite( &self, @@ -2077,21 +2215,22 @@ impl Compositor { sprite: &crate::scene::SceneCursorSprite, clip: [f32; 4], model: Option, - ) -> Result<()> { + ) -> Result> { let (tex, iw, ih) = self.cached_image(sprite.path.as_str())?; // Sans champ de distance, repli sur le sprite plat plutôt qu'aucun curseur. Parité Linux. if let Some(pose) = model { match self.cursor_sdf(sprite.path.as_str()) { Ok((sdf, shape)) => { let shape = crate::frame_geometry::model_shape(sprite, shape); - if let Some(cb) = crate::frame_geometry::cursor_model_cb( + let Some(cb) = crate::frame_geometry::cursor_model_cb( placement, size_px, pose, shape, a, clip, - ) { - enc.set_fragment_texture(2, Some(&tex)); - enc.set_fragment_texture(4, Some(&sdf)); - self.draw_solid(enc, &cb); - } - return Ok(()); + ) else { + return Ok(None); + }; + enc.set_fragment_texture(2, Some(&tex)); + enc.set_fragment_texture(4, Some(&sdf)); + self.draw_solid(enc, &cb); + return Ok(Some(cb.dst)); } Err(e) => eprintln!("[curseur] champ de \"{}\" : {e:#}", sprite.path), } @@ -2109,7 +2248,7 @@ impl Compositor { ); enc.set_fragment_texture(2, Some(&tex)); self.draw_solid(enc, &cb); - Ok(()) + Ok(Some(cb.dst)) } /// Curseur thématisé : le sprite de l'état courant, sinon la flèche, sinon rien. @@ -2132,14 +2271,17 @@ impl Compositor { a: f32, clip: [f32; 4], model: Option, - ) { - let sprite = cursor_type.and_then(|t| sprites.get(t)).or_else(|| sprites.get("arrow")); - if let Some(sprite) = sprite { - if let Err(e) = self.draw_cursor_sprite(enc, placement, size_px, a, sprite, clip, model) - { - eprintln!("[compositor] sprite curseur \"{}\" : {e:#}", sprite.path); - } - } + ) -> Option<[f32; 4]> { + let sprite = cursor_type.and_then(|t| sprites.get(t)).or_else(|| sprites.get("arrow"))?; + self.draw_cursor_sprite(enc, placement, size_px, a, sprite, clip, model).unwrap_or_else(|e| { + eprintln!("[compositor] sprite curseur \"{}\" : {e:#}", sprite.path); + None + }) + } + + /// Le métrage dans la dernière image composée : ses coins et son warp (`FootageQuad`). + pub fn footage_quad(&self) -> Option { + self.footage.get() } /// Compose la frame : fond, ombre écran, écran, ombre caméra, caméra — puis miroir @@ -2211,6 +2353,7 @@ impl Compositor { timeline_t_override: *self.timeline_time.borrow(), programme_time: *self.programme_time.borrow(), }); + self.footage.set(Some(g.footage_quad([rw, rh]))); let cmd_buf = self.gpu.context.new_command_buffer(); // Profondeur de champ : pyramide remplie seulement sur une frame inclinée qui la lit, @@ -2220,46 +2363,75 @@ impl Compositor { } else { None }; - let enc = self.begin_pass( - cmd_buf, - &self.rt, - Some(metal::MTLClearColor::new(0.0, 0.0, 0.0, 1.0)), - &self.pipeline_main, - )?; - // Les deux plans écran restent liés par défaut : les quads de couleur ne les - // échantillonnent pas, mais Metal veut des slots renseignés pour les draws qui, eux, - // le font. - enc.set_fragment_texture(0, Some(&sy)); - enc.set_fragment_texture(1, Some(&suv)); - - // --- fond --- (parité `compositor_windows.rs`, section « fond ») - match scene_ref.as_ref().map(|s| s.background.clone()) { - Some(SceneBackground::Color { color }) => { - let c = parse_hex(&color).unwrap_or(lp.bg_color); - self.draw_solid( - enc, - &LayerCB { dst: [0.0, 0.0, 1.0, 1.0], mode: 1.0, color: c, ..Default::default() }, - ); - } - Some(SceneBackground::Gradient { angle_deg, stops, offsets, motion }) => { - let a = angle_deg.to_radians(); - let (anim, mb) = - crate::frame_geometry::wallpaper_motion_slots(motion, g.programme_t, rw / rh); - self.draw_solid( - enc, - &LayerCB { - dst: [0.0, 0.0, 1.0, 1.0], - fx: [a.sin(), -a.cos(), anim[0], anim[1]], - mb, - ..crate::frame_geometry::gradient_layer(&stops, &offsets, lp.bg_color) - }, - ); + // Le fond de la frame d'avant, flou compris, s'il est celui qu'on demande + // (`BackgroundKey`) : recopié au lieu d'être redessiné. + let bg_desc = scene_ref.as_ref().map(|s| &s.background); + let bg_blur = scene_ref.as_ref().map_or(0.0, |s| s.effects.blur); + let bg_cached = self + .bg_cache + .borrow() + .as_ref() + .is_some_and(|c| c.key.matches(bg_desc, bg_blur, lp.bg_color, [rw, rh])); + if bg_cached { + if let Some(c) = self.bg_cache.borrow().as_ref() { + self.blit_background(cmd_buf, &c.tex, true); } - Some(SceneBackground::Image { path, motion }) => { - // Repli couleur en cas d'échec, mais LOGGÉ : un fallback silencieux masquerait - // un chemin cassé. - if let Err(e) = self.draw_image_bg(enc, &path, rw / rh, motion, g.programme_t) { - eprintln!("[compositor] wallpaper image \"{path}\" : {e:#}"); + } else { + let enc = self.begin_pass( + cmd_buf, + &self.rt, + Some(metal::MTLClearColor::new(0.0, 0.0, 0.0, 1.0)), + &self.pipeline_main, + )?; + // Les deux plans écran restent liés par défaut : les quads de couleur ne les + // échantillonnent pas, mais Metal veut des slots renseignés pour les draws qui, eux, + // le font. + enc.set_fragment_texture(0, Some(&sy)); + enc.set_fragment_texture(1, Some(&suv)); + + // --- fond --- (parité `compositor_windows.rs`, section « fond ») + // Faux si l'image du fond n'a pas pu se charger : son repli n'est pas gardé. + let mut bg_drawn = true; + match scene_ref.as_ref().map(|s| s.background.clone()) { + Some(SceneBackground::Color { color }) => { + let c = parse_hex(&color).unwrap_or(lp.bg_color); + self.draw_solid( + enc, + &LayerCB { dst: [0.0, 0.0, 1.0, 1.0], mode: 1.0, color: c, ..Default::default() }, + ); + } + Some(SceneBackground::Gradient { angle_deg, stops, offsets, motion }) => { + let a = angle_deg.to_radians(); + let (anim, mb) = + crate::frame_geometry::wallpaper_motion_slots(motion, g.programme_t, rw / rh); + self.draw_solid( + enc, + &LayerCB { + dst: [0.0, 0.0, 1.0, 1.0], + fx: [a.sin(), -a.cos(), anim[0], anim[1]], + mb, + ..crate::frame_geometry::gradient_layer(&stops, &offsets, lp.bg_color) + }, + ); + } + Some(SceneBackground::Image { path, motion }) => { + // Repli couleur en cas d'échec, mais LOGGÉ : un fallback silencieux masquerait + // un chemin cassé. + if let Err(e) = self.draw_image_bg(enc, &path, rw / rh, motion, g.programme_t) { + eprintln!("[compositor] wallpaper image \"{path}\" : {e:#}"); + bg_drawn = false; + self.draw_solid( + enc, + &LayerCB { + dst: [0.0, 0.0, 1.0, 1.0], + mode: 1.0, + color: lp.bg_color, + ..Default::default() + }, + ); + } + } + None => { self.draw_solid( enc, &LayerCB { @@ -2271,25 +2443,34 @@ impl Compositor { ); } } - None => { - self.draw_solid( - enc, - &LayerCB { - dst: [0.0, 0.0, 1.0, 1.0], - mode: 1.0, - color: lp.bg_color, - ..Default::default() - }, - ); - } - } - // « Blur BG » (parité web `blurredBackgroundLayer`) : floute CE wallpaper qu'on vient - // de dessiner, pas la vidéo. No-op visuel sur une couleur plate, effet réel sur un - // gradient ou une image. Il lui faut ses propres passes, d'où la coupure ici. - enc.end_encoding(); - if let Some(blur) = scene_ref.as_ref().map(|s| s.effects.blur) { - self.blur_bg(cmd_buf, blur)?; + // « Blur BG » (parité web `blurredBackgroundLayer`) : floute CE wallpaper qu'on vient + // de dessiner, pas la vidéo. No-op visuel sur une couleur plate, effet réel sur un + // gradient ou une image. Il lui faut ses propres passes, d'où la coupure ici. + enc.end_encoding(); + if let Some(blur) = scene_ref.as_ref().map(|s| s.effects.blur) { + self.blur_bg(cmd_buf, blur)?; + } + // Pas de cache pour un fond animé (`of` rend `None`). + let key = bg_drawn + .then(|| crate::frame_geometry::BackgroundKey::of(bg_desc, bg_blur, lp.bg_color, [rw, rh])) + .flatten(); + if let Some(key) = key { + // L'ancienne texture est reprise : `resized` vide le cache, sa taille est la bonne. + let old = self.bg_cache.borrow_mut().take().map(|c| c.tex); + let tex = old.unwrap_or_else(|| { + make_texture( + &self.gpu.device, + metal::MTLPixelFormat::RGBA8Unorm, + self.render_w, + self.render_h, + metal::MTLStorageMode::Private, + metal::MTLTextureUsage::ShaderRead, + ) + }); + self.blit_background(cmd_buf, &tex, false); + *self.bg_cache.borrow_mut() = Some(BgCache { key, tex }); + } } // --- écran : ombre puis vidéo --- let s_px = [g.s_dst[2] * rw, g.s_dst[3] * rh]; @@ -2400,6 +2581,11 @@ impl Compositor { enc.end_encoding(); + // --- flous de confidentialité : sur le métrage, AVANT le curseur (parité Windows). Le + // curseur reste net par-dessus, et le cristal de Prism Glow, qui réfracte l'image + // composée, n'y voit que des pixels déjà floutés. + self.draw_annotations(cmd_buf, scene_ref.as_ref(), g.source_t, g.s_ann, &g, true)?; + // --- curseur --- (parité `compositor_windows.rs`, section « curseur custom ») if let Some(track) = cursor_ref.as_ref() { let plan = crate::frame_geometry::plan_cursor( @@ -2422,6 +2608,12 @@ impl Compositor { .map(|s| s.cursor.cursor_sprites.clone()) .unwrap_or_default(); let kind = plan.cursor_type.as_deref(); + // Le cristal de Prism Glow réfracte l'image telle qu'elle est composée à cet + // instant, flous compris : sa copie en texture(5), une fois pour toutes les copies + // de la traînée (`ann_copy` est libre, les flous l'ont déjà lue). + if plan.glass { + self.snapshot_rt(cmd_buf, false); + } // L'impact des clics (mode 16, sans texture), posé sur l'écran SOUS le curseur. if !plan.impacts.is_empty() { let e = self.begin_pass(cmd_buf, &self.rt, None, &self.pipeline_main)?; @@ -2432,6 +2624,9 @@ impl Compositor { } if plan.taps <= 1 { let e = self.begin_pass(cmd_buf, &self.rt, None, &self.pipeline_main)?; + if plan.glass { + e.set_fragment_texture(5, Some(&self.ann_copy)); + } self.draw_cur_themed( e, &sprites, @@ -2454,11 +2649,17 @@ impl Compositor { Some(metal::MTLClearColor::new(0.0, 0.0, 0.0, 0.0)), &self.pipeline_add, )?; + if plan.glass { + e.set_fragment_texture(5, Some(&self.ann_copy)); + } + // L'union des quads peints dans `accum` (x0, y0, x1, y1 en fractions de la + // sortie) : la seule région que la recopie doit relire. + let mut bounds = [f32::MAX, f32::MAX, f32::MIN, f32::MIN]; for k in 0..plan.taps { let f = k as f32 / (plan.taps - 1) as f32; let w = crate::frame_geometry::cursor_tap_weight(k, plan.taps); e.set_blend_color(w, w, w, w); - self.draw_cur_themed( + let drawn = self.draw_cur_themed( e, &sprites, kind, @@ -2468,12 +2669,36 @@ impl Compositor { plan.clip, plan.model, ); + if let Some(d) = drawn { + bounds = [ + bounds[0].min(d[0]), + bounds[1].min(d[1]), + bounds[2].max(d[0] + d[2]), + bounds[3].max(d[1] + d[3]), + ]; + } } e.end_encoding(); let c = self.begin_pass(cmd_buf, &self.rt, None, &self.pipeline_fs_tex)?; c.set_fragment_texture(0, Some(&self.accum)); - c.draw_primitives(metal::MTLPrimitiveType::Triangle, 0, 3); + // Rien n'a été peint dans `accum` hors des quads de la traînée : la recopie + // s'y borne au lieu de repasser sur toute la sortie (0,5 ms par frame 1080p + // mesurée sur Linux, Radeon 610M). + let (w, h) = (self.render_w as f32, self.render_h as f32); + let x0 = (bounds[0] * w).floor().clamp(0.0, w) as u64; + let y0 = (bounds[1] * h).floor().clamp(0.0, h) as u64; + let x1 = (bounds[2] * w).ceil().clamp(0.0, w) as u64; + let y1 = (bounds[3] * h).ceil().clamp(0.0, h) as u64; + if x1 > x0 && y1 > y0 { + c.set_scissor_rect(metal::MTLScissorRect { + x: x0, + y: y0, + width: x1 - x0, + height: y1 - y0, + }); + c.draw_primitives(metal::MTLPrimitiveType::Triangle, 0, 3); + } c.end_encoding(); } } @@ -2481,105 +2706,150 @@ impl Compositor { // --- caméra : ombre PiP puis vidéo --- let enc = self.begin_pass(cmd_buf, &self.rt, None, &self.pipeline_main)?; - if let (true, Some((wy, wuv))) = (lp.has_webcam, webcam_tex.as_ref()) { - let [cu0, cv0, cu1, cv1] = crate::frame_geometry::webcam_source_rect( - [wcw, wch], - [wtw as f32, wth as f32], - scene_ref.as_ref().and_then(|scene| scene.layout.webcam_crop), - g.w_px[0] / g.w_px[1].max(0.0001), - ); - let (u0, u1) = if lp.webcam_mirror { (cu1, cu0) } else { (cu0, cu1) }; - let webcam_is_block = matches!( - g.scene_preset.as_deref(), - Some("dual-frame") | Some("vertical-stack") - ); - // Effet d'arrière-plan : le mode vient de la scène, le masque par pixel de - // l'inférence. Les DEUX sont requis — un mode sans masque rendrait la webcam - // invisible en détourage, donc tant que rien n'a été segmenté on dessine la - // piste telle quelle. C'est aussi ce qui rend le premier lancement gracieux. - let mask = self.webcam_mask.borrow(); - let effect = scene_ref - .as_ref() - .and_then(|s| s.webcam_effect.as_ref()) - .filter(|_| mask.is_some()) - .map(|e| (e.shader_code(), e)) - .filter(|(code, _)| *code > 0.0); - - // L'ombre appartient à la bulle PiP. En détourage il n'y a plus de bulle — une - // ombre portée par un rectangle invisible se lit comme un artefact. Le test porte - // sur le code de la SCÈNE et non sur celui envoyé au shader : le fond personnalisé - // part lui aussi en détourage ci-dessous, mais sa bulle, elle, est bien peinte et - // garde donc son ombre. - let is_cutout = matches!(effect, Some((code, _)) if code == 1.0); - if cfg.shadow && !webcam_is_block && !is_cutout && g.shape_fade > 0.0 { - self.draw_shadow( - enc, - g.w_dst, - g.w_px, - g.w_radius, - WEBCAM_SHADOW_SPREAD_FRAC * g.frame_min_px, - [0.0, WEBCAM_SHADOW_OFFSET_FRAC * g.frame_min_px], - WEBCAM_SHADOW_OPACITY * g.shape_fade, + // Camera 0's crop, mirror and desk turn as camera settings, plus its homography + // (`Scene::camera(0)`): with one, `src` is not read and the crop is dropped. + let cam0 = g.camera0_settings(scene_ref.as_ref()); + let webcam_is_block = matches!( + g.scene_preset.as_deref(), + Some("dual-frame") | Some("vertical-stack") + ); + // Effet d'arrière-plan : le mode vient de la scène, le masque par pixel de + // l'inférence. Les DEUX sont requis — un mode sans masque rendrait la webcam + // invisible en détourage, donc tant que rien n'a été segmenté on dessine la + // piste telle quelle. C'est aussi ce qui rend le premier lancement gracieux. + let mask = self.webcam_mask.borrow(); + let effect = scene_ref + .as_ref() + .and_then(|s| s.webcam_effect.as_ref()) + .filter(|_| mask.is_some()) + .map(|e| (e.shader_code(), e)) + .filter(|(code, _)| *code > 0.0); + // L'ombre appartient à la bulle PiP. En détourage il n'y a plus de bulle — une + // ombre portée par un rectangle invisible se lit comme un artefact. Le test porte + // sur le code de la SCÈNE et non sur celui envoyé au shader : le fond personnalisé + // part lui aussi en détourage ci-dessous, mais sa bulle, elle, est bien peinte et + // garde donc son ombre. + let is_cutout = matches!(effect, Some((code, _)) if code == 1.0); + // Fond personnalisé : on PEINT le fond dans la bulle, puis on y découpe la caméra + // par-dessus — le mélange alpha donne `lerp(fond, caméra, personne)`, soit exactement + // ce que la branche « mode 3 » du shader calculait, mais pour les TROIS sortes de + // fond. Le shader ne sait peindre qu'une couleur plate sous le masque ; dégradés et + // images y tombaient sur du noir, et le défaut EST une image. L'ordre est imposé : + // ombre, puis fond, puis caméra. `custom_bg` = Some(background) in that mode. + let (effect_code, blur_intensity, custom_bg) = match effect { + Some((code, e)) if code > 2.5 => (1.0, 0.0, Some(e.background.as_ref())), + Some((code, e)) => (code, e.blur_intensity.clamp(0.0, 1.0), None), + None => (0.0, 0.0, None), + }; + // Metal tolère l'index 3 non lié tant que `fx.z` reste à 0 : la branche n'est + // pas prise, la texture n'est pas échantillonnée. Dès qu'il monte, elle doit + // l'être sur TOUT draw capable de la prendre — ici seul celui de la caméra 0. L'état + // d'un encodeur est rémanent, donc lier avant le draw suffit, et l'ombre puis le + // fond qui précèdent sont en modes 1/2/5/6, que `ps_main` garde hors de la branche + // (`mode < 0.5`). The extra cameras draw with `fx.z = 0` and never read it. + // + // Pas de déliaison après coup, contrairement au chemin Windows qui remet le slot + // t3 à `None` : cet état meurt avec l'encodeur, et les annotations en ouvrent un + // autre. Il n'y a rien sur quoi fuir. + if let Some(m) = mask.as_ref() { + enc.set_fragment_texture(3, Some(&m.tex)); + } + // The extra cameras' textures, held to the end of `compose_frame` like `webcam_tex`. + let mut extra_textures = Vec::new(); + if g.camera_layers.is_empty() { + // Without layout regions: camera 0 from `w_dst`, as before the layers. + if let (true, Some((wy, wuv))) = (lp.has_webcam, webcam_tex.as_ref()) { + // Mirror and the desk-shot turn are both bound swaps: u for horizontal, v for + // vertical. + let src = crate::frame_geometry::camera_source_rect( + Some(&cam0), + [wcw, wch], + [wtw as f32, wth as f32], + g.w_px[0] / g.w_px[1].max(0.0001), ); + if cfg.shadow && !webcam_is_block && !is_cutout && g.shape_fade > 0.0 { + self.draw_shadow( + enc, + g.w_dst, + g.w_px, + g.w_radius, + WEBCAM_SHADOW_SPREAD_FRAC * g.frame_min_px, + [0.0, WEBCAM_SHADOW_OFFSET_FRAC * g.frame_min_px], + WEBCAM_SHADOW_OPACITY * g.shape_fade, + ); + } + if let Some(bg) = custom_bg { + self.draw_webcam_bg(enc, bg, g.w_dst, g.w_px, g.w_radius, 0.0); + } + let cb = g.webcam_video_cb(src, w_valid, effect_code, blur_intensity); + let cb = crate::frame_geometry::with_camera_homography(cb, Some(&cam0)); + self.draw_video(enc, &cb, wy, wuv); } - - // Fond personnalisé : on PEINT le fond dans la bulle, puis on y découpe la caméra - // par-dessus — le mélange alpha donne `lerp(fond, caméra, personne)`, soit exactement - // ce que la branche « mode 3 » du shader calculait, mais pour les TROIS sortes de - // fond. Le shader ne sait peindre qu'une couleur plate sous le masque ; dégradés et - // images y tombaient sur du noir, et le défaut EST une image. L'ordre est imposé : - // ombre, puis fond, puis caméra. - let (effect_code, blur_intensity) = match effect { - Some((code, e)) if code > 2.5 => { - self.draw_webcam_bg(enc, e.background.as_ref(), g.w_dst, g.w_px, g.w_radius); - (1.0, 0.0) + } else { + // Every planned layer, in order: shadow, then (camera 0) the custom background, + // then the video. Camera 0 keeps its frame, effects and mask; the extra cameras + // draw from `set_extra_camera_frames` without effects, and a missing one is skipped. + let render = [rw, rh]; + let cam0_base = g.webcam_video_cb([0.0; 4], w_valid, effect_code, blur_intensity); + for plan in &g.camera_layers { + if plan.camera == 0 { + let (true, Some((wy, wuv))) = (lp.has_webcam, webcam_tex.as_ref()) else { + continue; + }; + let video = crate::frame_geometry::camera_layer_cb( + plan, + Some(&cam0), + [wcw, wch], + [wtw as f32, wth as f32], + render, + &cam0_base, + ); + if let Some(shadow) = + g.camera_layer_shadow(plan, &video, render, cfg.shadow, is_cutout) + { + self.draw_solid(enc, &shadow); + } + if let Some(bg) = custom_bg { + let (dst, quad_px, radius_px) = (video.dst, video.quad_px, video.radius_px); + self.draw_webcam_bg(enc, bg, dst, quad_px, radius_px, video.layer_fx[0]); + } + self.draw_video(enc, &video, wy, wuv); + } else { + // Tolerant like camera 0: no frame, or no pixel buffer in it, skips the layer. + let frame = self.extra_camera_frame(plan.camera); + let Ok((ey, euv)) = self.nv12_srvs(frame) else { continue }; + let (tw, th) = self.tex_dims(frame); + let visible = [(*frame).width as f32, (*frame).height as f32]; + let tex = [tw as f32, th as f32]; + let valid = [visible[0] / tex[0].max(1.0), visible[1] / tex[1].max(1.0)]; + let video = crate::frame_geometry::camera_layer_cb( + plan, + scene_ref.as_ref().and_then(|s| s.camera(plan.camera)), + visible, + tex, + render, + &crate::frame_geometry::extra_camera_base_cb(valid), + ); + if let Some(shadow) = + g.camera_layer_shadow(plan, &video, render, cfg.shadow, false) + { + self.draw_solid(enc, &shadow); + } + self.draw_video(enc, &video, &ey, &euv); + extra_textures.push((ey, euv)); } - Some((code, e)) => (code, e.blur_intensity.clamp(0.0, 1.0)), - None => (0.0, 0.0), - }; - - // Metal tolère l'index 3 non lié tant que `fx.z` reste à 0 : la branche n'est - // pas prise, la texture n'est pas échantillonnée. Dès qu'il monte, elle doit - // l'être sur TOUT draw capable de la prendre — ici il n'y en a qu'un. L'état - // d'un encodeur est rémanent, donc lier avant le draw suffit, et l'ombre puis le - // fond qui précèdent sont en modes 1/2/5/6, que `ps_main` garde hors de la branche - // (`mode < 0.5`). - // - // Pas de déliaison après coup, contrairement au chemin Windows qui remet le slot - // t3 à `None` : cet état meurt avec l'encodeur, et les annotations en ouvrent un - // autre. Il n'y a rien sur quoi fuir. - if let Some(m) = mask.as_ref() { - enc.set_fragment_texture(3, Some(&m.tex)); } - self.draw_video( - enc, - &LayerCB { - dst: g.w_dst, - src: [u0, cv0, u1, cv1], - quad_px: g.w_px, - radius_px: g.w_radius, - mode: 0.0, - // `color.a` porte l'alpha du découpage (`color.a * personne`) ; le RGB n'est - // plus lu, le fond ayant déjà été peint sous la caméra. - color: [0.0, 0.0, 0.0, 1.0], - fx: [w_valid[0], w_valid[1], effect_code, blur_intensity], - src_prev: [u0, cv0, u1, cv1], - dst_prev: g.w_dst_prev, - mb: [g.mb_taps, g.mb_amount, 1.0, 0.0], - ..Default::default() - }, - wy, - wuv, - ); } + drop(mask); enc.end_encoding(); // --- annotations : calque le plus haut, ancré sur le rect ÉCRAN SANS ZOOM --- // `s_ann`, pas `s_dst` : le zoom vit dans la boîte depuis l'issue #179, donc `s_dst` // grandit avec lui et emmenait annotations et sous-titres dans le mouvement. Le flou de - // confidentialité est l'exception : il suit le contenu, d'où `&g`. - self.draw_annotations(cmd_buf, scene_ref.as_ref(), g.source_t, g.s_ann, &g)?; + // confidentialité est l'exception : il suit le contenu, d'où `&g` ; il est passé avant le + // curseur (plus haut). + self.draw_annotations(cmd_buf, scene_ref.as_ref(), g.source_t, g.s_ann, &g, false)?; // Ni miroir RGBA ni attente ici : le miroir ne sert qu'à `readback_direct` (la // preview), et l'export ne lit jamais le RGBA — le blit pleine résolution était payé @@ -4267,6 +4537,12 @@ mod tests { &metal::CompileOptions::new(), ) .expect("shaders.metal doit compiler"); + // La variante des modèles 3D (`LAYER_MODELS`) : la seule à compiler leurs branches. + device + .new_library_with_source(&super::msl_source_with_models(), &metal::CompileOptions::new()) + .expect("shaders.metal doit compiler avec LAYER_MODELS") + .get_function("ps_main", None) + .expect("ps_main absent de la variante LAYER_MODELS"); for name in [ "vs_main", "vs_fs", diff --git a/crates/compositor/src/compositor_windows.rs b/crates/compositor/src/compositor_windows.rs index 59431e383d..74c60b07ff 100644 --- a/crates/compositor/src/compositor_windows.rs +++ b/crates/compositor/src/compositor_windows.rs @@ -24,8 +24,9 @@ use std::cell::{Cell, RefCell}; use std::collections::HashMap; use std::ffi::c_void; use windows::core::Interface; +use windows::Win32::Foundation::RECT; use windows::Win32::Graphics::Direct3D::{ - D3D11_SRV_DIMENSION_TEXTURE2DARRAY, D3D_PRIMITIVE_TOPOLOGY_TRIANGLESTRIP, + D3D11_SRV_DIMENSION_TEXTURE2D, D3D_PRIMITIVE_TOPOLOGY_TRIANGLESTRIP, }; use windows::Win32::Graphics::Direct3D11::*; use windows::Win32::Graphics::Dxgi::Common::*; @@ -78,6 +79,12 @@ struct WebcamMask { /// demi-résolution de la texture DÉCODEUR, avec sa chaîne de mips. Dimensionnée sur la texture /// décodeur et remplie en UV plein (0..1) : ses UV sont ceux de t0/t1, et le calcul d'UV du /// mode 8 ne change pas. +/// Le fond tel que la passe de fond (et son flou) l'a laissé dans le RT, et la clé qui le décrit. +struct BgCache { + key: crate::frame_geometry::BackgroundKey, + tex: ID3D11Texture2D, +} + struct DofPyramid { rtv: ID3D11RenderTargetView, srv: ID3D11ShaderResourceView, @@ -85,6 +92,15 @@ struct DofPyramid { height: u32, } +/// An extra camera's frame for this `compose_frame`: its private copy's plane views, and the +/// visible and texture sizes `camera_layer_cb` crops against. +struct ExtraCamera { + y: ID3D11ShaderResourceView, + uv: ID3D11ShaderResourceView, + visible: [f32; 2], + tex: [f32; 2], +} + pub struct Compositor { dev: ID3D11Device, ctx: ID3D11DeviceContext, @@ -93,7 +109,10 @@ pub struct Compositor { rt_srv: ID3D11ShaderResourceView, staging: ID3D11Texture2D, vs: ID3D11VertexShader, + /// Le pixel shader des calques, SANS les modèles 3D (modes 15 à 17) : ils passent par + /// `ps_models`, et `draw_layer` choisit d'après le mode (le haut de `shaders.hlsl`). ps: ID3D11PixelShader, + ps_models: ID3D11PixelShader, vs_fs: ID3D11VertexShader, ps_y: ID3D11PixelShader, ps_uv: ID3D11PixelShader, @@ -129,11 +148,16 @@ pub struct Compositor { accum_rtv: ID3D11RenderTargetView, accum_srv: ID3D11ShaderResourceView, blend_add: ID3D11BlendState, + /// Rastérisation avec scissor, pour la seule recopie de la traînée du curseur. + rs_scissor: ID3D11RasterizerState, /// Rendu isolé de l'écran cadré (ombre, cadre, métrage, appareil), transparent autour, que le /// mode 18 recompose le long de sa trajectoire (`FrameGeometry::screen_trail`). Distinct de /// `accum`, que le curseur et `compose_frame_mb` remplissent dans la même frame. trail_rtv: ID3D11RenderTargetView, trail_srv: ID3D11ShaderResourceView, + /// Le dernier fond fixe composé et sa clé (`BackgroundKey`) : recopié dans le RT tant + /// que la clé tient, au lieu d'être redessiné. + bg_cache: RefCell>, /// RefCell (pas un simple champ) pour que `set_cursor` reste `&self`, comme `set_scene` / /// `set_live_params` — nécessaire pour le rebrancher par clip dans l'export multiclip, qui /// n'a qu'une référence partagée au `Compositor`. @@ -142,6 +166,8 @@ pub struct Compositor { /// (`frame / FPS`). L'export multiclip et le live le positionnent au PTS écran courant, /// c'est-à-dire au temps source absolu du clip actif. cursor_t_override: RefCell>, + /// Le métrage dans la dernière image composée, que l'éditeur lit avec elle (`live.rs`). + footage: std::cell::Cell>, /// Override du temps des zoom/full-camera regions (secondes source du clip actif). Le nom /// `timeline_t_override` est conservé pour l'API existante, mais ce temps n'est plus cumulé /// entre clips : les régions projetées par l'app portent elles aussi des temps source. @@ -149,9 +175,14 @@ pub struct Compositor { timeline_t_override: RefCell>, /// Temps programme (secondes de sortie) — cf. `FrameGeometryInput::programme_time`. programme_time: RefCell>, - // cache des SRV décodeur par (texture array, slice) : le pool réutilise ~32 textures, - // donc après warmup plus aucune création de SRV par frame (overhead CPU supprimé). - srv_cache: RefCell>, + /// A private copy of a decoder surface, with its two plane views (Y, UV). + /// Keyed by the decoder texture's pointer, which pins nothing: `nv12_srvs` re-checks + /// the copy's size and format on every hit, so a new texture landing on an old address + /// is never handed a copy sized for the old one. `clear_srv_cache` frees stale entries. + srv_cache: RefCell>, + /// The frames of cameras 1..=3 (`set_extra_camera_frames`), as addresses (0 = none) so + /// the compositor's auto traits stay what they were without the raw pointers. + extra_camera_frames: Cell<[usize; crate::camera_layers::MAX_EXTRA_CAMERAS]>, live_params: RefCell, /// Scène pilotée par l'app (contrat) : quand présente, remplace le layout fixture de /// `timeline()`. Voir `scene.rs` / `SceneDescription` (TS). @@ -404,6 +435,7 @@ impl Compositor { render_size: Cell::new((w, h)), // Caches dimensionnés à l'ancienne taille : recréés à la demande. resize_target: RefCell::new(None), + bg_cache: RefCell::new(None), live_readback_staging: RefCell::new(None), nv12_readback_staging: RefCell::new(None), ..self @@ -600,6 +632,8 @@ impl Compositor { dev.CreateVertexShader(shader!("vs_main"), None, Some(&mut vs))?; let mut ps: Option = None; dev.CreatePixelShader(shader!("ps_main"), None, Some(&mut ps))?; + let mut ps_models: Option = None; + dev.CreatePixelShader(shader!("ps_main_models"), None, Some(&mut ps_models))?; // shaders RGB->NV12 let mut vs_fs: Option = None; @@ -680,6 +714,17 @@ impl Compositor { }; let mut blend_add: Option = None; dev.CreateBlendState(&bla, Some(&mut blend_add))?; + // L'état par défaut (aucun lié), plus le scissor. Sans culling : le triangle plein écran + // n'a pas à dépendre de son sens de parcours. + let rsd = D3D11_RASTERIZER_DESC { + FillMode: D3D11_FILL_SOLID, + CullMode: D3D11_CULL_NONE, + ScissorEnable: true.into(), + DepthClipEnable: true.into(), + ..Default::default() + }; + let mut rs_scissor: Option = None; + dev.CreateRasterizerState(&rsd, Some(&mut rs_scissor))?; Ok(Compositor { dev, @@ -690,6 +735,7 @@ impl Compositor { staging, vs: vs.unwrap(), ps: ps.unwrap(), + ps_models: ps_models.unwrap(), vs_fs: vs_fs.unwrap(), ps_y: ps_y.unwrap(), ps_uv: ps_uv.unwrap(), @@ -718,13 +764,17 @@ impl Compositor { accum_rtv, accum_srv, blend_add: blend_add.unwrap(), + rs_scissor: rs_scissor.unwrap(), trail_rtv, trail_srv, + bg_cache: RefCell::new(None), cursor: RefCell::new(None), cursor_t_override: RefCell::new(None), + footage: std::cell::Cell::new(None), timeline_t_override: RefCell::new(None), programme_time: RefCell::new(None), srv_cache: RefCell::new(HashMap::new()), + extra_camera_frames: Cell::new([0; crate::camera_layers::MAX_EXTRA_CAMERAS]), live_params: RefCell::new(LiveParams::default()), scene: RefCell::new(None), text_raster: match crate::text::TextRasterizer::new() { @@ -803,43 +853,150 @@ impl Compositor { /// depuis le layout preset au lieu du planning fixture. pub fn set_scene(&self, s: Option) { *self.scene.borrow_mut() = s; + // A new scene can drop the regions that drew an extra camera: forget its frames so a + // stale pointer is never drawn before the next `set_extra_camera_frames`. + self.extra_camera_frames.set([0; crate::camera_layers::MAX_EXTRA_CAMERAS]); + } + + /// Frames for cameras 1..=3 (index 0 = scene camera 1). A null or missing entry means that + /// camera is not drawn this frame. The compositor keeps the pointers and reads them in every + /// `compose_frame` until the next call, so they must stay valid until then. They go through + /// `nv12_srvs` and its cache like camera 0's frame (`clear_srv_cache` clears them alike). + pub unsafe fn set_extra_camera_frames(&self, frames: &[*const AVFrame]) { + let mut slots = [0usize; crate::camera_layers::MAX_EXTRA_CAMERAS]; + for (slot, frame) in slots.iter_mut().zip(frames) { + *slot = *frame as usize; + } + self.extra_camera_frames.set(slots); + } + + /// The frame set for scene camera `camera` (1..=3), null when there is none. + fn extra_camera_frame(&self, camera: usize) -> *const AVFrame { + let slots = self.extra_camera_frames.get(); + camera.checked_sub(1).and_then(|k| slots.get(k).copied()).unwrap_or(0) as *const AVFrame } - /// Crée les SRV Y (R8) et UV (R8G8) sur la tranche d'array de la frame décodeur. + /// The copies (`nv12_srvs`) of every extra camera `layers` draws, with their visible and + /// texture sizes. Tolerant: a camera without a frame, or whose frame has no D3D11 texture, + /// is `None` and simply not drawn. + unsafe fn extra_camera_srvs( + &self, + layers: &[crate::camera_layers::CameraLayerPlan], + ) -> [Option; crate::camera_layers::MAX_EXTRA_CAMERAS] { + let mut out: [Option; crate::camera_layers::MAX_EXTRA_CAMERAS] = + Default::default(); + for (k, slot) in out.iter_mut().enumerate() { + let frame = self.extra_camera_frame(k + 1); + if frame.is_null() || !layers.iter().any(|l| l.camera == k + 1) { + continue; + } + let Ok((y, uv)) = self.nv12_srvs(frame) else { continue }; + let (tw, th) = self.tex_dims(frame); + *slot = Some(ExtraCamera { + y, + uv, + visible: [(*frame).width as f32, (*frame).height as f32], + tex: [tw as f32, th as f32], + }); + } + out + } + + /// The Y (R8) and UV (R8G8) views of the decoder frame — over a private COPY, never + /// over the decoder surface itself. + /// + /// Two documented D3D11 rules rule out the direct path. The first is explicit, on + /// `D3D11_BIND_DECODER`: "you cannot use texture arrays that are created with this + /// flag in calls to ID3D11Device::CreateShaderResourceView". The ffmpeg pool IS such + /// an array (`initial_pool_size` = 32, see `get_hw_format`). The second explains what + /// we were seeing: between the video engine and the 3D pipeline "there is no automatic + /// hazard tracking" — the surface stays the decoder's reference frame, so it may be + /// rewritten WHILE the shader samples it. ffmpeg's `ID3D11VideoContext` is in fact the + /// SAME object as our immediate context (it comes out of a QueryInterface on it), and + /// `SetMultithreadProtected(TRUE)` only makes an individual call atomic, never a + /// sequence. + /// + /// Measured on a Snapdragon X Elite (Adreno X1-85), same build, same recording, the + /// path picked by an environment variable: 768 black frames out of 792 when sampling + /// the decoder surface, 2 out of 545 through the copy. The black was OPAQUE and total, + /// so not "one frame late" but no frame at all. + /// + /// `CopySubresourceRegion` is issued on the immediate context, so it IS ordered against + /// the draws that follow; the destination texture is `ArraySize = 1` and carries only + /// `BIND_SHADER_RESOURCE`, which also takes it out of the restriction above. The cost + /// is one GPU→GPU copy per frame and per source, with nothing going back to system + /// memory. + /// + /// What hid the cause for so long: exporting the SAME scene never produced a black + /// frame (28 342 verified), because the export drains the GPU every frame through the + /// encoder. Everything that slowed the live loop — one more readback, a lock, a pause — + /// cut the black proportionally without ever removing it. pub unsafe fn nv12_srvs( &self, frame: *const AVFrame, ) -> Result<(ID3D11ShaderResourceView, ID3D11ShaderResourceView)> { let tex_ptr = (*frame).data[0] as *mut c_void; let slice = (*frame).data[1] as u32; - // cache hit : le pool réutilise les mêmes textures -> zéro création après warmup - let key = (tex_ptr as usize, slice); - if let Some((y, uv)) = self.srv_cache.borrow().get(&key) { - return Ok((y.clone(), uv.clone())); - } - let tex = ID3D11Texture2D::from_raw_borrowed(&tex_ptr) - .ok_or_else(|| anyhow::anyhow!("frame sans texture D3D11"))? + let src = ID3D11Texture2D::from_raw_borrowed(&tex_ptr) + .ok_or_else(|| anyhow::anyhow!("frame has no D3D11 texture"))? .clone(); - let mk = |fmt: DXGI_FORMAT| -> Result { - let mut d = D3D11_SHADER_RESOURCE_VIEW_DESC { - Format: fmt, - ViewDimension: D3D11_SRV_DIMENSION_TEXTURE2DARRAY, - ..Default::default() - }; - d.Anonymous.Texture2DArray = D3D11_TEX2D_ARRAY_SRV { - MostDetailedMip: 0, - MipLevels: 1, - FirstArraySlice: slice, - ArraySize: 1, - }; - let mut srv: Option = None; - self.dev.CreateShaderResourceView(&tex, Some(&d), Some(&mut srv))?; - Ok(srv.unwrap()) + // Cache hit: the pool reuses the same textures, so this allocates once per decoder + // after warmup. The COPY itself happens on EVERY frame — that is what freezes the + // slice's contents before the decoder takes it back. + // + // The key is a bare address and pins nothing. A decoder replaced without + // `clear_srv_cache` (a mid-stream resolution change, `CpuFrames::ensure_tex`) can + // hand a NEW texture the address of an old one, and a copy into a smaller destination + // is dropped without an error: the picture freezes. So a hit only counts while the + // destination still has the source's size and format. + let mut sd = D3D11_TEXTURE2D_DESC::default(); + src.GetDesc(&mut sd); + let key = tex_ptr as usize; + let cached = self.srv_cache.borrow().get(&key).cloned().filter(|(dst, ..)| { + let mut dd = D3D11_TEXTURE2D_DESC::default(); + dst.GetDesc(&mut dd); + (dd.Width, dd.Height, dd.Format) == (sd.Width, sd.Height, sd.Format) + }); + let (dst, y, uv) = match cached { + Some(v) => v, + None => { + let dd = D3D11_TEXTURE2D_DESC { + Width: sd.Width, + Height: sd.Height, + MipLevels: 1, + ArraySize: 1, + Format: sd.Format, + SampleDesc: DXGI_SAMPLE_DESC { Count: 1, Quality: 0 }, + Usage: D3D11_USAGE_DEFAULT, + BindFlags: D3D11_BIND_SHADER_RESOURCE.0 as u32, + CPUAccessFlags: 0, + MiscFlags: 0, + }; + let mut dst: Option = None; + self.dev.CreateTexture2D(&dd, None, Some(&mut dst))?; + let dst = dst.unwrap(); + let mk = |fmt: DXGI_FORMAT| -> Result { + let mut d = D3D11_SHADER_RESOURCE_VIEW_DESC { + Format: fmt, + ViewDimension: D3D11_SRV_DIMENSION_TEXTURE2D, + ..Default::default() + }; + d.Anonymous.Texture2D = D3D11_TEX2D_SRV { MostDetailedMip: 0, MipLevels: 1 }; + let mut srv: Option = None; + self.dev.CreateShaderResourceView(&dst, Some(&d), Some(&mut srv))?; + Ok(srv.unwrap()) + }; + let y = mk(DXGI_FORMAT_R8_UNORM)?; + let uv = mk(DXGI_FORMAT_R8G8_UNORM)?; + self.srv_cache + .borrow_mut() + .insert(key, (dst.clone(), y.clone(), uv.clone())); + (dst, y, uv) + } }; - let y = mk(DXGI_FORMAT_R8_UNORM)?; - let uv = mk(DXGI_FORMAT_R8G8_UNORM)?; - self.srv_cache.borrow_mut().insert(key, (y.clone(), uv.clone())); + + self.ctx.CopySubresourceRegion(&dst, 0, 0, 0, 0, &src, slice, None); Ok((y, uv)) } @@ -874,6 +1031,26 @@ impl Compositor { self.ctx.ClearRenderTargetView(&self.rtv, &clear); } + /// Garde le fond que le RT porte à cet instant (`bg_cache`), sous `key`. + unsafe fn keep_background(&self, key: crate::frame_geometry::BackgroundKey) -> Result<()> { + // L'ancienne texture est reprise : `resized` vide le cache, sa taille est la bonne. + let old = self.bg_cache.borrow_mut().take().map(|c| c.tex); + let tex = match old { + Some(tex) => tex, + None => { + let mut d = D3D11_TEXTURE2D_DESC::default(); + self.rt.GetDesc(&mut d); + d.BindFlags = 0; + let mut tex: Option = None; + self.dev.CreateTexture2D(&d, None, Some(&mut tex))?; + tex.ok_or_else(|| anyhow::anyhow!("CreateTexture2D (fond gardé)"))? + } + }; + self.ctx.CopyResource(&tex, &self.rt); + *self.bg_cache.borrow_mut() = Some(BgCache { key, tex }); + Ok(()) + } + /// Passe plein écran générique (triangle unique) : `srv` -> `rtv` via `ps`, avec `fx`. unsafe fn fs_pass( &self, @@ -1025,6 +1202,15 @@ impl Compositor { self.ctx.PSSetConstantBuffers(0, Some(&[Some(self.cbuf.clone())])); } + /// Dessine le quad d'un calque, ses ressources déjà liées. Le pixel shader suit son mode : + /// `ps_models` pour un modèle 3D (`LayerCB::needs_models`), `ps` pour tout le reste, qui ne + /// paie pas ainsi les registres de ces modèles. + unsafe fn draw_layer(&self, cb: &LayerCB) { + self.upload_cb(cb); + self.ctx.PSSetShader(if cb.needs_models() { &self.ps_models } else { &self.ps }, None); + self.ctx.Draw(4, 0); + } + /// Calque vidéo NV12. pub unsafe fn draw_video( &self, @@ -1032,16 +1218,14 @@ impl Compositor { srv_y: &ID3D11ShaderResourceView, srv_uv: &ID3D11ShaderResourceView, ) { - self.upload_cb(cb); self.ctx .PSSetShaderResources(0, Some(&[Some(srv_y.clone()), Some(srv_uv.clone())])); - self.ctx.Draw(4, 0); + self.draw_layer(cb); } /// Calque couleur pleine (fond). pub unsafe fn draw_solid(&self, cb: &LayerCB) { - self.upload_cb(cb); - self.ctx.Draw(4, 0); + self.draw_layer(cb); } /// Fond wallpaper image (cover-fit). `path` = chemin absolu (résolu côté app). Décodé et @@ -1103,13 +1287,14 @@ impl Compositor { programme_t: f32, ) -> Result<()> { let full = [0.0, 0.0, 1.0, 1.0]; - self.draw_image_in(path, full, [0.0, 0.0], 0.0, output_aspect, motion, programme_t) + self.draw_image_in(path, full, [0.0, 0.0], 0.0, output_aspect, motion, programme_t, 0.0) } /// `draw_image_bg` pour un rect quelconque — la bulle webcam s'en sert avec ses coins /// arrondis. `output_aspect` est le ratio du RECT visé, pas celui de la sortie : le crop /// « cover » se calcule contre la zone qu'on remplit. `motion` anime l'image au temps - /// programme `programme_t` ; la bulle passe `WallpaperMotion::None`. + /// programme `programme_t` ; la bulle passe `WallpaperMotion::None`. `transparency` = + /// `layer_fx.x` (0 = opaque): a camera layer's background box fades with the layer. #[allow(clippy::too_many_arguments)] unsafe fn draw_image_in( &self, @@ -1120,6 +1305,7 @@ impl Compositor { output_aspect: f32, motion: WallpaperMotion, programme_t: f32, + transparency: f32, ) -> Result<()> { let (srv, iw, ih) = self.cached_image(path)?; let ai = iw as f32 / ih as f32; @@ -1135,7 +1321,8 @@ impl Compositor { (0.0, (1.0 - vis) * 0.5, 1.0, 1.0 - (1.0 - vis) * 0.5) }; let (anim, mb) = crate::frame_geometry::wallpaper_motion_slots(motion, programme_t, ao); - self.upload_cb(&LayerCB { + self.ctx.PSSetShaderResources(2, Some(&[Some(srv)])); + self.draw_layer(&LayerCB { dst, src: [u0, v0, u1, v1], quad_px, @@ -1143,10 +1330,9 @@ impl Compositor { mode: 6.0, fx: [0.0, 0.0, anim[0], anim[1]], mb, + layer_fx: [transparency, 0.0, 0.0, 0.0], ..Default::default() }); - self.ctx.PSSetShaderResources(2, Some(&[Some(srv)])); - self.ctx.Draw(4, 0); Ok(()) } @@ -1162,20 +1348,25 @@ impl Compositor { /// /// `quad_px` / `radius_px` sont ceux de la bulle : le fond doit épouser ses coins arrondis, /// sinon un rectangle déborde derrière la caméra. + /// + /// `transparency` = the camera layer's `layer_fx.x`: the box fades with the camera. unsafe fn draw_webcam_bg( &self, bg: Option<&SceneBackground>, dst: [f32; 4], quad_px: [f32; 2], radius_px: f32, + transparency: f32, ) { const BLACK: [f32; 4] = [0.0, 0.0, 0.0, 1.0]; + let layer_fx = [transparency, 0.0, 0.0, 0.0]; let solid = |color: [f32; 4]| LayerCB { dst, quad_px, radius_px, mode: 1.0, color, + layer_fx, ..Default::default() }; match bg { @@ -1192,6 +1383,7 @@ impl Compositor { quad_px, radius_px, fx: [dir[0], dir[1], 0.0, 0.0], + layer_fx, ..crate::frame_geometry::gradient_layer(stops, offsets, BLACK) }); } @@ -1201,7 +1393,9 @@ impl Compositor { let aspect = if quad_px[1] > 0.0 { quad_px[0] / quad_px[1] } else { 1.0 }; let still = WallpaperMotion::None; if let Err(e) = - self.draw_image_in(path, dst, quad_px, radius_px, aspect, still, 0.0) + self.draw_image_in( + path, dst, quad_px, radius_px, aspect, still, 0.0, transparency, + ) { eprintln!("[compositor] fond webcam \"{}\" : {:#}", path, e); self.draw_solid(&solid(BLACK)); @@ -1661,7 +1855,7 @@ impl Compositor { /// Curseur custom (dot+ring) centré en `center` (0..1 sortie), taille `size_px`, opacité `a`. /// `clip` = rect "Clip to canvas" en espace sortie [x,y,w,h] ; passer un rect englobant tout /// (ex. [-1,-1,3,3]) pour désactiver l'effet. - unsafe fn draw_cursor(&self, center: [f32; 2], size_px: f32, a: f32, clip: [f32; 4]) { + unsafe fn draw_cursor(&self, center: [f32; 2], size_px: f32, a: f32, clip: [f32; 4]) -> [f32; 4] { let w = size_px / self.rw(); let h = size_px / self.rh(); let dst = [center[0] - w * 0.5, center[1] - h * 0.5, w, h]; @@ -1673,6 +1867,7 @@ impl Compositor { fx: clip, ..Default::default() }); + dst } /// Curseur thème (sprite PNG, ex. arrow.png) dont le PIVOT `hotspot` (fraction 0..1 de @@ -1690,7 +1885,7 @@ impl Compositor { sprite: &SceneCursorSprite, clip: [f32; 4], model: Option, - ) -> Result<()> { + ) -> Result> { let path = sprite.path.as_str(); let (srv, iw, ih) = self.cached_image(path)?; // Sans champ de distance, repli sur le sprite plat plutôt que sur le curseur math. @@ -1699,16 +1894,16 @@ impl Compositor { match self.cursor_sdf(path) { Ok((sdf, shape)) => { let shape = crate::frame_geometry::model_shape(sprite, shape); - if let Some(cb) = crate::frame_geometry::cursor_model_cb( + let Some(cb) = crate::frame_geometry::cursor_model_cb( placement, size_px, pose, shape, a, clip, - ) { - self.upload_cb(&cb); - self.ctx.PSSetShaderResources(2, Some(&[Some(srv)])); - self.ctx.PSSetShaderResources(4, Some(&[Some(sdf)])); - self.ctx.Draw(4, 0); - self.ctx.PSSetShaderResources(4, Some(&[None])); - } - return Ok(()); + ) else { + return Ok(None); + }; + self.ctx.PSSetShaderResources(2, Some(&[Some(srv)])); + self.ctx.PSSetShaderResources(4, Some(&[Some(sdf)])); + self.draw_layer(&cb); + self.ctx.PSSetShaderResources(4, Some(&[None])); + return Ok(Some(cb.dst)); } Err(e) => eprintln!("[curseur] champ de \"{path}\" : {e:#}"), } @@ -1723,10 +1918,9 @@ impl Compositor { clip, [self.rw(), self.rh()], ); - self.upload_cb(&cb); self.ctx.PSSetShaderResources(2, Some(&[Some(srv)])); - self.ctx.Draw(4, 0); - Ok(()) + self.draw_layer(&cb); + Ok(Some(cb.dst)) } /// Sprite de l'état courant (`cursor_type`, ex. `"text"`), à défaut celui de la flèche, @@ -1746,17 +1940,17 @@ impl Compositor { a: f32, clip: [f32; 4], model: Option, - ) { + ) -> Option<[f32; 4]> { let sprite = cursor_type.and_then(|t| sprites.get(t)).or_else(|| sprites.get("arrow")); if let Some(sprite) = sprite { - if self.draw_cursor_sprite(placement, size_px, a, sprite, clip, model).is_ok() { - return; + if let Ok(dst) = self.draw_cursor_sprite(placement, size_px, a, sprite, clip, model) { + return dst; } } // Le repli math reste droit même sur un plan incliné : il ne devrait plus apparaître // maintenant que l'art par défaut existe, et lui donner sa propre passe de warp pour // un cas de secours ne se justifie pas. - self.draw_cursor(placement.upright_center(), size_px, a, clip); + Some(self.draw_cursor(placement.upright_center(), size_px, a, clip)) } /// Ombre portée (§7 E4) sous un quad `dst` (normalisé) de taille `size_px`. @@ -1850,6 +2044,11 @@ impl Compositor { }); } + /// Le métrage dans la dernière image composée : ses coins et son warp (`FootageQuad`). + pub fn footage_quad(&self) -> Option { + self.footage.get() + } + /// Compose une frame animée (§6/§8) : fond flouté + screen zoomé (padding, coins, ombre) /// + webcam crop carré (coins, ombre), placements interpolés A↔B par la timeline. pub unsafe fn compose_frame( @@ -1893,8 +2092,10 @@ impl Compositor { timeline_t_override: *self.timeline_t_override.borrow(), programme_time: *self.programme_time.borrow(), }); + self.footage.set(Some(g.footage_quad([self.rw(), self.rh()]))); + // The extra cameras' copies, made before anything is drawn like the two above. + let extra_cams = self.extra_camera_srvs(&g.camera_layers); let scene_preset = g.scene_preset.clone(); - let mb_taps = g.mb_taps; let mb_amount = g.mb_amount; let source_t = g.source_t; let _padding_scale = g.padding_scale; @@ -1905,7 +2106,6 @@ impl Compositor { let s_radius = g.s_radius; let frame_min_px = g.frame_min_px; let w_dst = g.w_dst; - let w_dst_prev = g.w_dst_prev; let w_px = g.w_px; let w_radius = g.w_radius; let shape_fade = g.shape_fade; @@ -1915,8 +2115,6 @@ impl Compositor { let dof = g.depth_of_field_on(self.cpu_backend); let dof_srv = if dof { Some(self.fill_dof_pyramid(&sy, &suv, stw, sth)?) } else { None }; - self.begin([0.0, 0.0, 0.0, 1.0]); - // --- fond --- // Parité web (frameRenderer.blurredBackgroundLayer) : le fond est le WALLPAPER sélectionné // (image/couleur/gradient) et « Blur BG » floute CE wallpaper, PAS la vidéo. Le natif @@ -1925,86 +2123,120 @@ impl Compositor { // ensuite ; pour une couleur plate le flou est un no-op visuel). Côté fixture/bench // (pas de scène) on garde le fond screen-flouté, dont le coût est mesuré (C4). let scene_bg = self.scene.borrow().as_ref().map(|s| (s.background.clone(), s.effects.blur)); - if let Some((bg, blur_wallpaper)) = scene_bg { - match bg { - SceneBackground::Color { color } => { - let c = parse_hex(&color).unwrap_or(lp.bg_color); - self.draw_solid(&LayerCB { - dst: [0.0, 0.0, 1.0, 1.0], - mode: 1.0, - color: c, - ..Default::default() - }); - } - SceneBackground::Gradient { angle_deg, stops, offsets, motion } => { - // angle CSS → direction unitaire (espace sortie, y vers le bas) : - // 0° = vers le haut, 90° = vers la droite. - let a = angle_deg.to_radians(); - let dir = [a.sin(), -a.cos()]; - let (anim, mb) = crate::frame_geometry::wallpaper_motion_slots( - motion, - g.programme_t, - self.rw() / self.rh(), - ); - self.draw_solid(&LayerCB { - dst: [0.0, 0.0, 1.0, 1.0], - fx: [dir[0], dir[1], anim[0], anim[1]], - mb, - ..crate::frame_geometry::gradient_layer(&stops, &offsets, lp.bg_color) - }); - } - SceneBackground::Image { path, motion } => { - // image bg (cover-fit, mise en cache) ; fallback couleur si chargement échoue - // (loggé — un fallback silencieux masquerait un chemin cassé, cf. le panic - // borrow qu'on a déjà eu : toute panne doit être visible/traçable). - let aspect = self.rw() / self.rh(); - if let Err(e) = self.draw_image_bg(&path, aspect, motion, g.programme_t) { - eprintln!("[compositor] wallpaper image \"{}\" : {:#}", path, e); + // Le fond de la frame d'avant, s'il est celui qu'on demande (`BackgroundKey`), est recopié + // au lieu d'être redessiné. Côté scène seulement : sans scène, le fond est la vidéo floutée. + let bg_size = [self.rw(), self.rh()]; + let bg_cached = scene_ref.as_ref().is_some_and(|s| { + self.bg_cache + .borrow() + .as_ref() + .is_some_and(|c| { + c.key.matches(Some(&s.background), s.effects.blur, lp.bg_color, bg_size) + }) + }); + if bg_cached { + self.bind_compose_state(); + if let Some(c) = self.bg_cache.borrow().as_ref() { + self.ctx.CopyResource(&self.rt, &c.tex); + } + } else { + self.begin([0.0, 0.0, 0.0, 1.0]); + // Faux si l'image du fond n'a pas pu se charger : son repli n'est pas gardé. + let mut bg_drawn = true; + if let Some((bg, blur_wallpaper)) = scene_bg { + match bg { + SceneBackground::Color { color } => { + let c = parse_hex(&color).unwrap_or(lp.bg_color); self.draw_solid(&LayerCB { dst: [0.0, 0.0, 1.0, 1.0], mode: 1.0, - color: lp.bg_color, + color: c, ..Default::default() }); } + SceneBackground::Gradient { angle_deg, stops, offsets, motion } => { + // angle CSS → direction unitaire (espace sortie, y vers le bas) : + // 0° = vers le haut, 90° = vers la droite. + let a = angle_deg.to_radians(); + let dir = [a.sin(), -a.cos()]; + let (anim, mb) = crate::frame_geometry::wallpaper_motion_slots( + motion, + g.programme_t, + self.rw() / self.rh(), + ); + self.draw_solid(&LayerCB { + dst: [0.0, 0.0, 1.0, 1.0], + fx: [dir[0], dir[1], anim[0], anim[1]], + mb, + ..crate::frame_geometry::gradient_layer(&stops, &offsets, lp.bg_color) + }); + } + SceneBackground::Image { path, motion } => { + // image bg (cover-fit, mise en cache) ; fallback couleur si chargement échoue + // (loggé — un fallback silencieux masquerait un chemin cassé, cf. le panic + // borrow qu'on a déjà eu : toute panne doit être visible/traçable). + let aspect = self.rw() / self.rh(); + if let Err(e) = self.draw_image_bg(&path, aspect, motion, g.programme_t) { + eprintln!("[compositor] wallpaper image \"{}\" : {:#}", path, e); + bg_drawn = false; + self.draw_solid(&LayerCB { + dst: [0.0, 0.0, 1.0, 1.0], + mode: 1.0, + color: lp.bg_color, + ..Default::default() + }); + } + } } - } - // « Blur BG » (parité web blurredBackgroundLayer) : floute CE wallpaper qu'on vient - // de dessiner (dual-Kawase, déjà utilisé pour le fond fixture ci-dessous). No-op - // visuel sur une couleur plate, effet réel sur gradient/image. - if blur_wallpaper > 0.0 { - self.blur_bg(blur_wallpaper); + // « Blur BG » (parité web blurredBackgroundLayer) : floute CE wallpaper qu'on vient + // de dessiner (dual-Kawase, déjà utilisé pour le fond fixture ci-dessous). No-op + // visuel sur une couleur plate, effet réel sur gradient/image. + if blur_wallpaper > 0.0 { + self.blur_bg(blur_wallpaper); + self.bind_compose_state(); + } + } else if cfg.bg_blur > 0.0 { + let over = 0.06; + self.draw_video( + &LayerCB { + dst: [-over, -over, 1.0 + 2.0 * over, 1.0 + 2.0 * over], + src: [0.0, 0.0, u_max, v_max], + quad_px: [self.rw(), self.rh()], + mode: 0.0, + color: [1.0, 1.0, 1.0, 1.0], + ..Default::default() + }, + &sy, + &suv, + ); + self.blur_bg(cfg.bg_blur); self.bind_compose_state(); - } - } else if cfg.bg_blur > 0.0 { - let over = 0.06; - self.draw_video( - &LayerCB { - dst: [-over, -over, 1.0 + 2.0 * over, 1.0 + 2.0 * over], - src: [0.0, 0.0, u_max, v_max], - quad_px: [self.rw(), self.rh()], - mode: 0.0, - color: [1.0, 1.0, 1.0, 1.0], + self.draw_solid(&LayerCB { + dst: [0.0, 0.0, 1.0, 1.0], + mode: 1.0, + color: [0.0, 0.0, 0.0, 0.35], ..Default::default() - }, - &sy, - &suv, - ); - self.blur_bg(cfg.bg_blur); - self.bind_compose_state(); - self.draw_solid(&LayerCB { - dst: [0.0, 0.0, 1.0, 1.0], - mode: 1.0, - color: [0.0, 0.0, 0.0, 0.35], - ..Default::default() - }); - } else { - self.draw_solid(&LayerCB { - dst: [0.0, 0.0, 1.0, 1.0], - mode: 1.0, - color: lp.bg_color, - ..Default::default() + }); + } else { + self.draw_solid(&LayerCB { + dst: [0.0, 0.0, 1.0, 1.0], + mode: 1.0, + color: lp.bg_color, + ..Default::default() + }); + } + // Pas de cache pour un fond animé (`of` rend `None`). + let key = scene_ref.as_ref().filter(|_| bg_drawn).and_then(|s| { + crate::frame_geometry::BackgroundKey::of( + Some(&s.background), + s.effects.blur, + lp.bg_color, + bg_size, + ) }); + if let Some(key) = key { + self.keep_background(key)?; + } } // --- screen : crop du clip actif, puis zoom appliqué dans ce rect source (§8) --- @@ -2138,6 +2370,11 @@ impl Compositor { self.ctx.PSSetShaderResources(5, Some(&[None])); } + // --- flous de confidentialité : sur le métrage, AVANT le curseur. Le curseur reste net + // par-dessus, et le cristal de Prism Glow, qui réfracte l'image composée, n'y voit que des + // pixels déjà floutés. Les autres annotations restent le calque le plus haut (plus bas). + self.draw_annotations(scene_ref.as_ref(), source_t, s_ann, &g, true); + // --- curseur custom : suit le mapping src/dst (zoom+layout), click bounce, // et flou de mouvement (parité `compositor_macos.rs` et `compositor_linux.rs`) --- if let Some(track) = cursor_ref.as_ref() { @@ -2161,6 +2398,13 @@ impl Compositor { .map(|s| s.cursor.cursor_sprites.clone()) .unwrap_or_default(); let cursor_type = plan.cursor_type.as_deref(); + // Le cristal de Prism Glow réfracte l'image telle qu'elle est composée à cet + // instant, flous compris : sa copie en t5, une fois pour toutes les copies de la + // traînée (`ann_copy` est libre, les flous l'ont déjà lue). + if plan.glass { + self.ctx.CopySubresourceRegion(&self.ann_copy, 0, 0, 0, 0, &self.rt, 0, None); + self.ctx.PSSetShaderResources(5, Some(&[Some(self.ann_copy_srv.clone())])); + } // L'impact des clics (mode 16, sans texture), posé sur l'écran SOUS le curseur. for cb in &plan.impacts { self.draw_solid(cb); @@ -2180,11 +2424,14 @@ impl Compositor { // buffer ISOLÉ (transparent), pas directement sur la scène déjà composée. self.ctx.ClearRenderTargetView(&self.accum_rtv, &[0.0, 0.0, 0.0, 0.0]); self.ctx.OMSetRenderTargets(Some(&[Some(self.accum_rtv.clone())]), None); + // L'union des quads peints dans `accum` (x0, y0, x1, y1 en fractions de la + // sortie) : la seule région que la recopie doit relire. + let mut bounds = [f32::MAX, f32::MAX, f32::MIN, f32::MIN]; for k in 0..plan.taps { let f = k as f32 / (plan.taps - 1) as f32; let w = crate::frame_geometry::cursor_tap_weight(k, plan.taps); self.ctx.OMSetBlendState(&self.blend_add, Some(&[w, w, w, w]), 0xffffffff); - self.draw_cur_themed( + let drawn = self.draw_cur_themed( &cursor_sprites, cursor_type, plan.prev_placement.lerp(plan.placement, f), @@ -2193,6 +2440,14 @@ impl Compositor { plan.clip, plan.model, ); + if let Some(d) = drawn { + bounds = [ + bounds[0].min(d[0]), + bounds[1].min(d[1]), + bounds[2].max(d[0] + d[2]), + bounds[3].max(d[1] + d[3]), + ]; + } } // composite le buffer accumulé sur la scène (blend "over" normal, prémultiplié). self.ctx.OMSetRenderTargets(Some(&[Some(self.rtv.clone())]), None); @@ -2210,12 +2465,30 @@ impl Compositor { }; self.ctx.RSSetViewports(Some(&[vp])); self.ctx.OMSetBlendState(&self.blend, None, 0xffffffff); - self.ctx.Draw(3, 0); + // Rien n'a été peint dans `accum` hors des quads de la traînée : la recopie + // s'y borne au lieu de repasser sur toute la sortie (0,5 ms par frame 1080p + // mesurée sur Linux, Radeon 610M). + let (rw, rh) = (self.rw(), self.rh()); + let rect = RECT { + left: (bounds[0] * rw).floor().clamp(0.0, rw) as i32, + top: (bounds[1] * rh).floor().clamp(0.0, rh) as i32, + right: (bounds[2] * rw).ceil().clamp(0.0, rw) as i32, + bottom: (bounds[3] * rh).ceil().clamp(0.0, rh) as i32, + }; + if rect.right > rect.left && rect.bottom > rect.top { + self.ctx.RSSetState(&self.rs_scissor); + self.ctx.RSSetScissorRects(Some(&[rect])); + self.ctx.Draw(3, 0); + self.ctx.RSSetState(None); + } self.ctx.PSSetShaderResources(0, Some(&[None])); // restaure l'état de composition standard (VS/PS/topologie quad-strip) pour // le dessin de la webcam qui suit juste après. self.bind_compose_state(); } + if plan.glass { + self.ctx.PSSetShaderResources(5, Some(&[None])); + } } } @@ -2233,99 +2506,144 @@ impl Compositor { // // Le center-crop carré de square/circle en est un cas particulier (boîte 1:1) — il n'a // plus besoin d'être traité à part. - let [su0, sv0, su1, sv1] = crate::frame_geometry::webcam_source_rect( - [wcw, wch], - [wtw as f32, wth as f32], - scene_ref - .as_ref() - .and_then(|scene| scene.layout.webcam_crop), - w_px[0] / w_px[1].max(0.0001), + // Camera 0's crop, mirror and desk turn as camera settings, plus its homography + // (`Scene::camera(0)`): with one, `src` is not read and the crop is dropped. + let cam0 = g.camera0_settings(scene_ref.as_ref()); + // L'ombre portée appartient à la bulle flottante PiP : elle se retire avec elle + // (`shape_fade`), pour qu'au plein écran plus rien n'encadre la caméra. C'est une + // ombre légère NON paramétrable — indépendante du slider Shadow, qui ne pilote plus + // que l'écran (`WEBCAM_SHADOW_OPACITY`, pas `shadow_scale`) — et propre au PiP : les + // blocs side-by-side / top-bottom soudent la caméra à l'écran et n'en portent aucune + // (parité `preset.shadow` web, `null` hors PiP dans `compositeLayout.ts`). + let webcam_is_block = matches!( + scene_preset.as_deref(), + Some("dual-frame") | Some("vertical-stack"), ); - // miroir = échanger les bornes u du rect source (flip horizontal). - let (u0, u1) = if lp.webcam_mirror { (su1, su0) } else { (su0, su1) }; - if lp.has_webcam { - // L'ombre portée appartient à la bulle flottante PiP : elle se retire avec elle - // (`shape_fade`), pour qu'au plein écran plus rien n'encadre la caméra. C'est une - // ombre légère NON paramétrable — indépendante du slider Shadow, qui ne pilote plus - // que l'écran (`WEBCAM_SHADOW_OPACITY`, pas `shadow_scale`) — et propre au PiP : les - // blocs side-by-side / top-bottom soudent la caméra à l'écran et n'en portent aucune - // (parité `preset.shadow` web, `null` hors PiP dans `compositeLayout.ts`). - let webcam_is_block = matches!( - scene_preset.as_deref(), - Some("dual-frame") | Some("vertical-stack"), - ); - // L'ombre appartient à la bulle PiP. En détourage il n'y a plus de bulle — une - // ombre portée par un rectangle invisible se lit comme un artefact. - let is_cutout = matches!( - scene_ref.as_ref().and_then(|s| s.webcam_effect.as_ref()), - Some(e) if e.shader_code() == 1.0 - ) && self.webcam_mask.borrow().is_some(); - if cfg.shadow && !webcam_is_block && !is_cutout && shape_fade > 0.0 { - let strength = WEBCAM_SHADOW_OPACITY * shape_fade; - self.draw_shadow( - w_dst, - w_px, - w_radius, - WEBCAM_SHADOW_SPREAD_FRAC * frame_min_px, - [0.0, WEBCAM_SHADOW_OFFSET_FRAC * frame_min_px], - strength, - ); - } - // Effet d'arrière-plan : le mode vient de la scène, le masque par pixel de - // l'inférence. Les DEUX sont requis — un mode sans masque rendrait la webcam - // invisible en détourage, donc tant que rien n'a été segmenté on dessine la piste - // telle quelle. C'est aussi ce qui rend le premier lancement gracieux. - let mask = self.webcam_mask.borrow(); - let effect = scene_ref - .as_ref() - .and_then(|s| s.webcam_effect.as_ref()) - .filter(|_| mask.is_some()) - .map(|e| (e.shader_code(), e)) - .filter(|(code, _)| *code > 0.0); - - // Fond personnalisé : on PEINT le fond dans la bulle, puis on y découpe la caméra - // par-dessus — le mélange alpha donne `lerp(fond, caméra, personne)`, soit exactement - // ce que la branche « mode 3 » du shader calculait, mais pour les TROIS sortes de - // fond. Le shader ne sait peindre qu'une couleur plate sous le masque ; dégradés et - // images y tombaient sur du noir, et le défaut EST une image. - let (effect_code, blur_intensity) = match effect { - Some((code, e)) if code > 2.5 => { - self.draw_webcam_bg(e.background.as_ref(), w_dst, w_px, w_radius); - (1.0, 0.0) - } - Some((code, e)) => (code, e.blur_intensity.clamp(0.0, 1.0)), - None => (0.0, 0.0), - }; - + // L'ombre appartient à la bulle PiP. En détourage il n'y a plus de bulle — une + // ombre portée par un rectangle invisible se lit comme un artefact. + let is_cutout = matches!( + scene_ref.as_ref().and_then(|s| s.webcam_effect.as_ref()), + Some(e) if e.shader_code() == 1.0 + ) && self.webcam_mask.borrow().is_some(); + // Effet d'arrière-plan : le mode vient de la scène, le masque par pixel de + // l'inférence. Les DEUX sont requis — un mode sans masque rendrait la webcam + // invisible en détourage, donc tant que rien n'a été segmenté on dessine la piste + // telle quelle. C'est aussi ce qui rend le premier lancement gracieux. + let mask = self.webcam_mask.borrow(); + let effect = scene_ref + .as_ref() + .and_then(|s| s.webcam_effect.as_ref()) + .filter(|_| mask.is_some()) + .map(|e| (e.shader_code(), e)) + .filter(|(code, _)| *code > 0.0); + // Fond personnalisé : on PEINT le fond dans la bulle, puis on y découpe la caméra + // par-dessus — le mélange alpha donne `lerp(fond, caméra, personne)`, soit exactement + // ce que la branche « mode 3 » du shader calculait, mais pour les TROIS sortes de + // fond. Le shader ne sait peindre qu'une couleur plate sous le masque ; dégradés et + // images y tombaient sur du noir, et le défaut EST une image. + // `custom_bg` = Some(background) in that mode; it is painted between shadow and camera. + let (effect_code, blur_intensity, custom_bg) = match effect { + Some((code, e)) if code > 2.5 => (1.0, 0.0, Some(e.background.as_ref())), + Some((code, e)) => (code, e.blur_intensity.clamp(0.0, 1.0), None), + None => (0.0, 0.0, None), + }; + // Camera 0's video draw: the mask, bound to slot 3 for this draw only. `draw_video` ne + // lie que les slots 0-1, donc le masque posé ici tient pour l'appel qui suit. Il est + // délié juste après pour ne pas fuir sur les calques d'annotation, qui utilisent eux + // aussi le slot 2 et au-delà. + let draw_camera0 = |cb: &LayerCB| { if let Some(m) = mask.as_ref() { - // `draw_video` ne lie que les slots 0-1, donc le masque posé ici tient pour - // l'appel qui suit. Il est délié juste après pour ne pas fuir sur les calques - // d'annotation, qui utilisent eux aussi le slot 2 et au-delà. self.ctx.PSSetShaderResources(3, Some(&[Some(m.srv.clone())])); } - self.draw_video( - &LayerCB { - dst: w_dst, - src: [u0, sv0, u1, sv1], - quad_px: w_px, - radius_px: w_radius, - mode: 0.0, - // `color.a` porte l'alpha du découpage (`color.a * personne`) ; le RGB n'est - // plus lu, le fond ayant déjà été peint sous la caméra. - color: [0.0, 0.0, 0.0, 1.0], - fx: [w_valid[0], w_valid[1], effect_code, blur_intensity], - src_prev: [u0, sv0, u1, sv1], // src fixe (pas de zoom webcam) - dst_prev: w_dst_prev, - mb: [mb_taps, mb_amount, 1.0, 0.0], - ..Default::default() - }, - &wy, - &wuv, - ); + self.draw_video(cb, &wy, &wuv); if mask.is_some() { self.ctx.PSSetShaderResources(3, Some(&[None])); } + }; + + if g.camera_layers.is_empty() { + // Without layout regions: camera 0 from `w_dst`, as before the layers. + let [u0, v0, u1, v1] = crate::frame_geometry::camera_source_rect( + Some(&cam0), + [wcw, wch], + [wtw as f32, wth as f32], + w_px[0] / w_px[1].max(0.0001), + ); + if lp.has_webcam { + if cfg.shadow && !webcam_is_block && !is_cutout && shape_fade > 0.0 { + let strength = WEBCAM_SHADOW_OPACITY * shape_fade; + self.draw_shadow( + w_dst, + w_px, + w_radius, + WEBCAM_SHADOW_SPREAD_FRAC * frame_min_px, + [0.0, WEBCAM_SHADOW_OFFSET_FRAC * frame_min_px], + strength, + ); + } + if let Some(bg) = custom_bg { + self.draw_webcam_bg(bg, w_dst, w_px, w_radius, 0.0); + } + let cb = g.webcam_video_cb([u0, v0, u1, v1], w_valid, effect_code, blur_intensity); + draw_camera0(&crate::frame_geometry::with_camera_homography(cb, Some(&cam0))); + } + } else { + // Every planned layer, in order: shadow, then (camera 0) the custom background, + // then the video. Camera 0 keeps its frame, effects and mask; the extra cameras + // draw from `set_extra_camera_frames` without effects, and a missing one is skipped. + let render = [self.rw(), self.rh()]; + let cam0_base = g.webcam_video_cb([0.0; 4], w_valid, effect_code, blur_intensity); + for plan in &g.camera_layers { + if plan.camera == 0 { + if !lp.has_webcam { + continue; + } + let video = crate::frame_geometry::camera_layer_cb( + plan, + Some(&cam0), + [wcw, wch], + [wtw as f32, wth as f32], + render, + &cam0_base, + ); + if let Some(shadow) = + g.camera_layer_shadow(plan, &video, render, cfg.shadow, is_cutout) + { + self.draw_solid(&shadow); + } + if let Some(bg) = custom_bg { + let t = video.layer_fx[0]; + self.draw_webcam_bg(bg, video.dst, video.quad_px, video.radius_px, t); + } + draw_camera0(&video); + } else { + let Some(cam) = extra_cams.get(plan.camera - 1).and_then(|c| c.as_ref()) + else { + continue; + }; + let settings = scene_ref.as_ref().and_then(|s| s.camera(plan.camera)); + let valid = [ + cam.visible[0] / cam.tex[0].max(1.0), + cam.visible[1] / cam.tex[1].max(1.0), + ]; + let video = crate::frame_geometry::camera_layer_cb( + plan, + settings, + cam.visible, + cam.tex, + render, + &crate::frame_geometry::extra_camera_base_cb(valid), + ); + if let Some(shadow) = + g.camera_layer_shadow(plan, &video, render, cfg.shadow, false) + { + self.draw_solid(&shadow); + } + self.draw_video(&video, &cam.y, &cam.uv); + } + } } + drop(mask); // --- annotations : calque le plus haut, comme dans le DOM de la preview (le calque y est // monté après la vidéo). Ancrées sur `s_ann`, le rect ÉCRAN SANS ZOOM — c'est le conteneur @@ -2334,12 +2652,13 @@ impl Compositor { // la transform, donc les annotations restent en place pendant que le contenu zoome dessous. // Ce fut `s_dst` tant que le zoom vivait dans la coupe source ; depuis l'issue #179 il vit // dans la BOÎTE, et `s_dst` emmenait annotations et sous-titres avec lui. - // Exception : le flou de confidentialité suit le contenu (`FrameGeometry::privacy_mask`), - // d'où la géométrie entière passée en plus de `s_ann`. + // Les flous de confidentialité, eux, sont passés avant le curseur (plus haut) : ils suivent + // le contenu (`FrameGeometry::privacy_mask`), d'où la géométrie entière passée en plus de + // `s_ann`. // `source_t`, la même base de temps que les zoom/speed regions : le temps SOURCE du clip, // pas le compteur de frames. C'est ce qui garde une annotation alignée sur l'image quand // une speed region répète ou saute des frames. - self.draw_annotations(scene_ref.as_ref(), source_t, s_ann, &g); + self.draw_annotations(scene_ref.as_ref(), source_t, s_ann, &g, false); Ok(()) } @@ -2351,19 +2670,23 @@ impl Compositor { /// sous-titres sous un zoom (issue #179, puis #397 sur Linux). L'arithmétique elle-même vit /// dans `frame_geometry::annotation_dst_in`, partagée par les trois backends. Le flou, lui, /// se place par `g.privacy_mask` : un masque doit rester sur ce qu'il cache. + /// + /// `privacy` : les flous de confidentialité seuls, dessinés sur le métrage avant le curseur ; + /// sinon toutes les autres annotations, le calque le plus haut. unsafe fn draw_annotations( &self, scene: Option<&Scene>, t: f32, s_ann: [f32; 4], g: &crate::frame_geometry::FrameGeometry, + privacy: bool, ) { let Some(scene) = scene else { return }; if scene.annotations.is_empty() { return; } let visible = |a: &crate::scene::SceneAnnotation| { - t >= a.start_sec as f32 && t < a.end_sec as f32 + t >= a.start_sec as f32 && t < a.end_sec as f32 && (a.kind == "blur") == privacy }; // Une seule recopie du render target pour TOUTES les annotations flou de la frame — leur // lecture doit voir l'image composée sans les flous eux-mêmes, sinon deux zones qui se @@ -2534,6 +2857,13 @@ impl Compositor { if text.content.trim().is_empty() { continue; } + // The desk label shows only while the camera is covered: skip the rest of + // its section before rasterizing anything. + if crate::text_anim::is_desk_cover(text.animation.as_deref()) + && g.webcam_cover <= 0.0 + { + continue; + } // `font_size_rel` est une fraction de la HAUTEUR DE LA BOÎTE D'ANCRAGE — rect // écran, ou cadre de sortie pour un sous-titre (cf. le contrat et // `annotationScale.ts`) : on la ramène en pixels de sortie ici, avec le même @@ -2579,10 +2909,13 @@ impl Compositor { // dispose ici) : dans une région accélérée, elle défile donc au rythme du // clip. À vitesse 1 — le cas de toutes les annotations existantes — c'est // exactement le timing de l'aperçu DOM. - let anim = crate::text_anim::text_animation_state( + // The desk label is the exception: its opacity is the camera cover, which + // runs on the screen clock (`annotation_text_state`). + let anim = crate::text_anim::annotation_text_state( text.animation.as_deref(), (t - annotation.start_sec as f32) * 1000.0, ((annotation.end_sec - annotation.start_sec) * 1000.0) as f32, + g.webcam_cover, ); // Les décalages sont donnés à la hauteur de référence : on les ramène à la // sortie, comme la taille de police, pour que l'animation ait la même @@ -2917,6 +3250,12 @@ impl Compositor { Ok((rw, rh, out)) } + /// Le RT de la dernière composition, à `render_size()`, pour qui le copie ailleurs sans + /// passer par la RAM : l'anneau de textures partagées de la preview (`shared_frames`). + pub fn render_target(&self) -> &ID3D11Texture2D { + &self.rt + } + /// Comme `rgb_to_nv12`, mais vers `target_w`×`target_h` : si la cible diffère de la taille /// de rendu, le RT composé est d'abord redimensionné (bilinéaire, `ps_tex`/`sampler` déjà /// utilisés partout ailleurs dans le fichier, cf. `blit_resized`). Le RT suivant la sortie, @@ -3162,6 +3501,9 @@ impl Compositor { /// (p.ex. après un export) pour ne pas retenir indéfiniment des textures de pool. pub fn clear_srv_cache(&self) { self.srv_cache.borrow_mut().clear(); + // The extra cameras' frames belong to the decoders just closed: forget them too, so + // a stale pointer is never read before the next `set_extra_camera_frames`. + self.extra_camera_frames.set([0; crate::camera_layers::MAX_EXTRA_CAMERAS]); } } @@ -3249,6 +3591,62 @@ mod tests { ); } + /// Un NV12 sans données, à la taille voulue : tout ce que `nv12_srvs` lit d'une frame. + fn nv12_frame(gpu: &crate::d3d::Gpu, w: u32, h: u32) -> (Box, ID3D11Texture2D) { + let desc = D3D11_TEXTURE2D_DESC { + Width: w, + Height: h, + MipLevels: 1, + ArraySize: 1, + Format: DXGI_FORMAT_NV12, + SampleDesc: DXGI_SAMPLE_DESC { Count: 1, Quality: 0 }, + Usage: D3D11_USAGE_DEFAULT, + BindFlags: D3D11_BIND_SHADER_RESOURCE.0 as u32, + CPUAccessFlags: 0, + MiscFlags: 0, + }; + unsafe { + let mut tex: Option = None; + gpu.device.CreateTexture2D(&desc, None, Some(&mut tex)).expect("texture NV12"); + let tex = tex.expect("texture NV12"); + let mut frame: Box = Box::new(std::mem::zeroed()); + frame.data[0] = tex.as_raw() as *mut u8; + frame.data[1] = std::ptr::null_mut(); + (frame, tex) + } + } + + /// Un décodeur remplacé sans `clear_srv_cache` (changement de résolution en cours de flux, + /// `CpuFrames::ensure_tex`) peut donner à une texture NEUVE l'adresse d'une ancienne. La + /// copie privée gardée pour l'ancienne est alors trop petite, `CopySubresourceRegion` la + /// saute sans erreur et l'image gèle. On ne peut pas choisir l'adresse d'une texture ; + /// l'entrée de la petite est donc replacée sous la clé de la grande, l'état exact où le + /// cache se retrouve après un tel recyclage. + #[test] + fn a_larger_source_at_a_cached_address_gets_a_copy_of_its_own_size() { + let Ok(gpu) = crate::d3d::Gpu::create_auto(false) else { + eprintln!("pas de device D3D11 — test sauté"); + return; + }; + let comp = Compositor::new_sized(&gpu, 320, 180).expect("compositeur"); + let (small, _small_tex) = nv12_frame(&gpu, 64, 64); + let (large, _large_tex) = nv12_frame(&gpu, 128, 96); + let (small_key, large_key) = (small.data[0] as usize, large.data[0] as usize); + + unsafe { comp.nv12_srvs(&*small) }.expect("source de 64x64"); + { + let mut cache = comp.srv_cache.borrow_mut(); + let stale = cache.remove(&small_key).expect("entrée de la petite source"); + cache.insert(large_key, stale); + } + unsafe { comp.nv12_srvs(&*large) }.expect("source de 128x96"); + + let (dst, ..) = comp.srv_cache.borrow().get(&large_key).cloned().expect("entrée remplacée"); + let mut dd = D3D11_TEXTURE2D_DESC::default(); + unsafe { dst.GetDesc(&mut dd) }; + assert_eq!((dd.Width, dd.Height), (128, 96), "la copie a gardé la taille de l'ancienne texture"); + } + /// Un cran de padding en format Auto change la taille de rendu de la preview. `resized` /// réalloue les cibles et garde le reste : le masque webcam, et la boîte aux lettres où le /// worker de segmentation dépose les suivants. diff --git a/crates/compositor/src/cpu_frames_windows.rs b/crates/compositor/src/cpu_frames_windows.rs index 790f23a7fc..ea0e921fbe 100644 --- a/crates/compositor/src/cpu_frames_windows.rs +++ b/crates/compositor/src/cpu_frames_windows.rs @@ -41,8 +41,9 @@ pub(crate) struct CpuFrames { /// NV12 en mémoire système : la cible de swscale, la source de l'upload. nv12: *mut AVFrame, /// La texture NV12 échantillonnée par les shaders. UNE seule, réécrite à chaque - /// frame — le `srv_cache` du compositeur (clé `(ptr, slice)`) n'a donc qu'une entrée - /// et ne recrée jamais de SRV, contrairement au pool tournant de D3D11VA. + /// frame — le `srv_cache` du compositeur (clé : le pointeur de texture) n'a donc qu'une + /// entrée et ne recrée jamais de SRV, contrairement au pool tournant de D3D11VA. Quand + /// `ensure_tex` la réalloue à une autre taille, `nv12_srvs` voit l'écart et refait la copie. // ponytail: une seule texture = le CPU peut attendre que le GPU ait fini de lire la // frame précédente. Sur WARP tout est CPU et le pilote sérialise déjà ; si un backend // GPU réutilise ce chemin un jour et que le Map bloque, double-bufferiser ici. diff --git a/crates/compositor/src/extra_cameras.rs b/crates/compositor/src/extra_cameras.rs new file mode 100644 index 0000000000..0b2833c843 --- /dev/null +++ b/crates/compositor/src/extra_cameras.rs @@ -0,0 +1,241 @@ +//! The extra cameras (k >= 1) a camera layout region draws, shared by the live preview +//! (`live.rs`) and the export walk (`timeline_walk.rs`). +//! +//! Index convention: camera 0 is the clip's `webcam`; camera k >= 1 is +//! `additional_cameras[k - 1]`. Only the cameras a non-empty layout region shows are opened, +//! and each is decoded only near those regions (`extra_camera_active`). Unlike camera 0, an +//! extra camera never shortens a clip and never holds its last picture: past its end its +//! layer disappears. + +use crate::camera_layers::MAX_EXTRA_CAMERAS; +use crate::ffi::AVFrame; +use crate::live::PREFETCH_LEAD_SEC; +use crate::pipeline::Decoder; +use crate::scene::{SceneCameraLayoutRegion, SceneClipCamera}; +use crate::timeline_walk::{frame_step, FrameStep, NextFrameTime}; +use anyhow::Result; +use std::collections::{HashMap, HashSet}; + +/// An extra camera of an export clip: its file and its offset +/// (camera source time = screen source time - `offset_sec`). +pub type ClipCamera = SceneClipCamera; + +/// What a clip opens for its extra cameras: index k-1 = camera k, `None` = not opened (no +/// layout region of the clip shows it, or the clip has no file for it). No extra camera at all +/// is the empty list. +pub(crate) type ExtraCameraKeys = Vec>; + +/// Re-seek an extra camera instead of decoding forward when its target is further ahead than +/// this (it sat idle between two regions). +pub(crate) const EXTRA_RESEEK_SEC: f64 = 1.0; + +/// Camera indices >= 1 that the non-empty `regions` draw, sorted, without duplicates. +pub(crate) fn cameras_in_regions(regions: &[SceneCameraLayoutRegion]) -> Vec { + let mut cameras: Vec = regions + .iter() + .filter(|r| r.end_sec > r.start_sec) + .flat_map(|r| r.layers.iter().map(|l| l.camera)) + .filter(|camera| (1..=MAX_EXTRA_CAMERAS).contains(camera)) + .collect(); + cameras.sort_unstable(); + cameras.dedup(); + cameras +} + +pub(crate) fn has_camera_file(camera: Option<&SceneClipCamera>) -> bool { + camera.is_some_and(|c| !c.path.trim().is_empty()) +} + +/// The slots to open for `cameras` from a clip's camera files (`sources[k-1]` = camera k). +pub(crate) fn extra_camera_keys(cameras: &[usize], sources: &[SceneClipCamera]) -> ExtraCameraKeys { + let mut keys: ExtraCameraKeys = (1..=MAX_EXTRA_CAMERAS) + .map(|k| { + let source = sources.get(k - 1); + if cameras.contains(&k) && has_camera_file(source) { + source.cloned() + } else { + None + } + }) + .collect(); + while keys.last().is_some_and(|k| k.is_none()) { + keys.pop(); + } + keys +} + +/// Is extra camera `camera` near one of its layout regions at screen source time `t` — from +/// `PREFETCH_LEAD_SEC` before the region to its end? Outside, its decoder stays idle. +pub(crate) fn extra_camera_active(regions: &[SceneCameraLayoutRegion], camera: usize, t: f64) -> bool { + regions.iter().any(|r| { + r.end_sec > r.start_sec + && r.layers.iter().any(|l| l.camera == camera) + && (r.start_sec - PREFETCH_LEAD_SEC..=r.end_sec).contains(&t) + }) +} + +/// A camera's source time at screen source time `screen_t` (`camera = screen - offset`), +/// never before the file's start. Camera 0 (`live.rs`) and the extra cameras share it. +pub(crate) fn camera_source_time(screen_t: f64, offset_sec: f64) -> f64 { + (screen_t - offset_sec).max(0.0) +} + +/// An extra camera's open result: a file that will not open is `None` with one warning line, +/// and its layer is skipped — never drawn from another source. +pub(crate) fn opened_or_skipped(path: &str, opened: Result) -> Option { + match opened { + Ok(dec) => Some(dec), + Err(e) => { + eprintln!("WARNING: extra camera unreadable ({path}): {e:#}. Its layer will not be drawn."); + None + } + } +} + +/// Opens `path` into `decs` once. A file that will not open is remembered in `unreadable` +/// and never tried again, so its layer is skipped for the whole export. `true` when `decs` +/// holds a decoder for `path`. +pub(crate) fn open_once( + decs: &mut HashMap, + unreadable: &mut HashSet, + path: &str, + open: impl FnOnce(&str) -> Result, +) -> bool { + if decs.contains_key(path) { + return true; + } + if unreadable.contains(path) { + return false; + } + match opened_or_skipped(path, open(path)) { + Some(dec) => { + decs.insert(path.to_string(), dec); + true + } + None => { + unreadable.insert(path.to_string()); + false + } + } +} + +/// The list `set_extra_camera_frames` takes: null for an empty slot, else `frame` of the camera +/// (itself null when it has nothing to show). +pub(crate) fn extra_frame_list( + extra: &[Option], + frame: impl Fn(&T) -> *const AVFrame, +) -> Vec<*const AVFrame> { + extra.iter().map(|slot| slot.as_ref().map_or(std::ptr::null(), &frame)).collect() +} + +/// Advances `dec` to `target` (camera source time) with the webcam's hold semantics +/// (`frame_step`); re-seeks after an idle stretch or a jump back. Null past the camera's last +/// frame. `ended_at` is the camera source time past which the file has no frame left: the +/// decoder is not asked again until the target moves back before it. +pub(crate) unsafe fn step_extra_camera( + dec: &mut Decoder, + ended_at: &mut Option, + target: f64, +) -> Result<*const AVFrame> { + if ended_at.is_some_and(|end| target >= end) { + return Ok(std::ptr::null()); + } + let frame_dur = 1.0 / dec.fps().max(1.0); + let cur = dec.cur_frame(); + let far = cur.is_null() || { + let t = dec.cur_time_sec(); + target < t - frame_dur * 0.5 || target > t + EXTRA_RESEEK_SEC + }; + let frame = if far { + dec.seek_to(target)? + } else { + let mut frame = cur; + let mut guard = 0u32; + loop { + let next = dec.peek_next_time_sec()?; + // Unlike camera 0, an extra camera does not hold its last picture: past its + // end (one frame's worth of slack) its layer disappears. + if matches!(next, NextFrameTime::Eof) && target > dec.cur_time_sec() + frame_dur { + frame = std::ptr::null_mut(); + break; + } + match frame_step(next, 0.0, target) { + FrameStep::Commit => frame = dec.commit_peek()?, + FrameStep::CommitAndStop => { + frame = dec.commit_peek()?; + break; + } + FrameStep::Hold => break, + } + guard += 1; + if guard > 1000 { + break; + } + } + frame + }; + Ok(settle(ended_at, frame, target)) +} + +/// Seeks `dec` to `target` (camera source time). Null past the camera's last frame. +pub(crate) unsafe fn seek_extra_camera( + dec: &mut Decoder, + ended_at: &mut Option, + target: f64, +) -> Result<*const AVFrame> { + if ended_at.is_some_and(|end| target >= end) { + return Ok(std::ptr::null()); + } + let frame = dec.seek_to(target)?; + Ok(settle(ended_at, frame, target)) +} + +fn settle(ended_at: &mut Option, frame: *mut AVFrame, target: f64) -> *const AVFrame { + *ended_at = if frame.is_null() { Some(target) } else { None }; + frame +} + +#[cfg(test)] +mod tests { + use super::*; + use anyhow::anyhow; + + #[test] + fn export_skips_an_extra_camera_that_will_not_open() { + let mut decs: HashMap = HashMap::new(); + let mut unreadable = HashSet::new(); + let mut attempts = 0; + let mut refuse = |_: &str| -> Result { + attempts += 1; + Err(anyhow!("no such file")) + }; + assert!(!open_once(&mut decs, &mut unreadable, "/cam2.mp4", &mut refuse)); + // A later clip of the same export does not try the file again. + assert!(!open_once(&mut decs, &mut unreadable, "/cam2.mp4", &mut refuse)); + assert_eq!(attempts, 1); + assert!(decs.is_empty()); + + // A readable camera opens once and is reused by the next clip. + let mut opens = 0; + let mut accept = |_: &str| -> Result { + opens += 1; + Ok(7) + }; + assert!(open_once(&mut decs, &mut unreadable, "/cam3.mp4", &mut accept)); + assert!(open_once(&mut decs, &mut unreadable, "/cam3.mp4", &mut accept)); + assert_eq!(opens, 1); + assert_eq!(decs.get("/cam3.mp4"), Some(&7)); + } + + #[test] + fn extra_offsets_move_into_the_screen_clock() { + // camera + offset = screen: at offset 2 s, a camera frame at 0.5 s is due at 2.5 s + // of screen time, not before. + let held = camera_source_time(2.4, 2.0); + let due = camera_source_time(2.5, 2.0); + assert_eq!(frame_step(NextFrameTime::At(0.5), 0.0, held), FrameStep::Hold); + assert_eq!(frame_step(NextFrameTime::At(0.5), 0.0, due), FrameStep::Commit); + // Before the camera started, its source time clamps to its first frame. + assert_eq!(camera_source_time(1.0, 2.0), 0.0); + } +} diff --git a/crates/compositor/src/frame_geometry.rs b/crates/compositor/src/frame_geometry.rs index c788d92b85..9cba80d82d 100644 --- a/crates/compositor/src/frame_geometry.rs +++ b/crates/compositor/src/frame_geometry.rs @@ -27,7 +27,7 @@ use crate::config::Cfg; use crate::scene::{Scene, SceneCrop}; -/// Constant buffer d'un calque : **176 octets**, un par draw. +/// Constant buffer d'un calque : **256 octets**, un par draw. /// /// C'est le contrat partagé par les trois côtés — `cbuffer Layer` dans `shaders.hlsl`, /// `struct Layer` dans `shaders.metal` et `vk_shaders/layer.wgsl`, et ce struct. Ils doivent @@ -35,13 +35,15 @@ use crate::scene::{Scene, SceneCrop}; /// produit un shader qui lit `color` là où on a écrit `fx`. /// /// `align(16)` vient de la version macOS ; sous `repr(C)` seul, les offsets sont déjà -/// 0/16/32/40/44/48/64/80/96/112/128/144/160 des deux côtés — l'alignement Rust ne change que +/// 0/16/32/40/44/48/64/80/96/112/128/144/160/176 des deux côtés — l'alignement Rust ne change que /// l'adresse du struct, pas son contenu, et Windows le `copy_nonoverlapping` dans un /// constant buffer mappé où l'alignement source est sans effet. Les deux formes étaient /// donc compatibles ; les unifier évite qu'elles cessent de l'être. /// /// (Le commentaire d'origine annonçait « 64 octets ». Il n'a jamais été juste : dix champs, -/// trente-deux `f32`. Les trois derniers, le flou de mouvement de l'écran incliné, en font 176.) +/// trente-deux `f32`. Les trois suivants, le flou de mouvement de l'écran incliné, en font 176, et `cover`, le voile de la vue bureau, 192.) +/// (The camera layers' homography `persp` and transparency `layer_fx` make it 256: offsets 192 +/// and 240.) #[repr(C, align(16))] #[derive(Clone, Copy, Default)] pub struct LayerCB { @@ -62,6 +64,28 @@ pub struct LayerCB { pub trail_a: [f32; 4], pub trail_b: [f32; 4], pub trail_mb: [f32; 4], + /// Desk-view cover of the webcam layer: x = strength 0..1, y = blur radius (quad px), + /// z = dim factor, w unused. Zero everywhere else. + pub cover: [f32; 4], + /// Camera (mode 0) only: the rows of the homography H that maps a point of the layer's + /// `dst` quad (0..1, top-left origin) to the camera frame (0..1 of its valid part): + /// `[h0, h1, h2, 0], [h3, h4, h5, 0], [h6, h7, h8, 0]`. Read only when `layer_fx.y` is 1; + /// `src` is then ignored. Every other mode ignores it. + pub persp: [[f32; 4]; 3], + /// x = transparency 0..1 (0 = opaque), applied to the output of every mode; y = 1 when + /// `persp` applies; z, w = 0. All zero = the layer draws exactly as before the lanes. + pub layer_fx: [f32; 4], +} + +impl LayerCB { + /// Le calque est l'un des modèles 3D — le curseur modelé (mode 15), l'impact de son clic + /// (16), l'appareil et son ombre (17) — que seule la variante « modèles » du shader de calque + /// compile. Dans le même shader que les autres, leurs branches fixaient l'occupation de TOUS + /// les calques (168 VGPR sur RADV) ; les trois backends décident donc du shader d'un calque + /// par ce test, au moment de le dessiner (cf. le haut de `vk_shaders/layer.wgsl`). + pub fn needs_models(&self) -> bool { + self.mode > 14.5 && self.mode < 17.5 + } } pub const OUT_W: u32 = 1920; @@ -319,6 +343,33 @@ pub(crate) fn cover_crop_uv(visible: [f32; 2], tex: [f32; 2], box_ar: f32) -> (f (u0, v0, u1, v1) } +/// How the webcam texture is laid onto its quad this frame. The backends swap the source +/// rect's u bounds for `flip_u` and its v bounds for `flip_v` — 180° is both — and drop the +/// webcam crop for `full_frame`. Derived per frame because a Full Camera region can turn +/// the camera for a desk shot. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct WebcamOrientation { + pub flip_u: bool, + pub flip_v: bool, + pub full_frame: bool, +} + +pub fn webcam_orientation( + region: Option<&crate::scene::SceneCameraFullscreenRegion>, + layout_mirror: bool, +) -> WebcamOrientation { + let Some(r) = region else { + return WebcamOrientation { flip_u: layout_mirror, flip_v: false, full_frame: false }; + }; + let turned = r.rotation == 180; + let mirror = r.mirror.unwrap_or(layout_mirror); + WebcamOrientation { + flip_u: mirror ^ turned, + flip_v: turned, + full_frame: turned && r.full_frame, + } +} + /// Camera equivalent of the screen crop pipeline: apply the user crop first, then a centred /// cover-crop inside that authored window so arbitrary layout slots never stretch the image. pub(crate) fn webcam_source_rect( @@ -335,6 +386,140 @@ pub(crate) fn webcam_source_rect( box_ar, ) } + +/// The camera's perspective correction, when it has one whose nine values are all finite. A +/// matrix with a NaN or an infinity would draw garbage (or nothing), so it counts as none. +pub(crate) fn camera_homography(camera: Option<&crate::scene::SceneCamera>) -> Option<[f32; 9]> { + camera + .and_then(|c| c.homography) + .filter(|h| h.iter().all(|v| v.is_finite())) +} + +/// The source rect a camera is drawn from: its crop, then a cover-crop to `box_ar`, then its +/// rotation (180° = both bounds swapped) and mirror (u bounds swapped), `flip_u = mirror ^ +/// turned` as `webcam_orientation` does. A camera without settings is drawn upright and +/// unmirrored. With a homography, crop, rotation and mirror are all ignored: the corner order +/// already defines the picture (the shader then does not read `src` at all). +pub(crate) fn camera_source_rect( + camera: Option<&crate::scene::SceneCamera>, + visible: [f32; 2], + tex: [f32; 2], + box_ar: f32, +) -> [f32; 4] { + let corrected = camera_homography(camera).is_some(); + let settings = camera.filter(|_| !corrected); + let [u0, v0, u1, v1] = webcam_source_rect(visible, tex, settings.and_then(|c| c.crop), box_ar); + let turned = settings.is_some_and(|c| c.rotation == 180); + let mirror = settings.and_then(|c| c.mirror).unwrap_or(false); + let (u0, u1) = if mirror ^ turned { (u1, u0) } else { (u0, u1) }; + let (v0, v1) = if turned { (v1, v0) } else { (v0, v1) }; + [u0, v0, u1, v1] +} + +/// The window of a corrected picture (`aspect` = its width/height) a box of ratio `box_ar` +/// shows, as `[u0, v0, du, dv]` of the picture's unit square: centred and cover-fitted, so the +/// picture is cropped, never stretched — what `cover_uv_rect` does for an uncorrected camera, +/// whose `src` the shader stops reading once a homography is set. The whole picture without a +/// usable aspect or box. +pub(crate) fn corrected_cover_window(aspect: Option, box_ar: f32) -> [f32; 4] { + let Some(aspect) = aspect.filter(|a| a.is_finite() && *a > 0.0) else { + return [0.0, 0.0, 1.0, 1.0]; + }; + if !(box_ar.is_finite() && box_ar > 0.0) { + return [0.0, 0.0, 1.0, 1.0]; + } + if box_ar > aspect { + let dv = aspect / box_ar; + [0.0, (1.0 - dv) * 0.5, 1.0, dv] + } else { + let du = box_ar / aspect; + [(1.0 - du) * 0.5, 0.0, du, 1.0] + } +} + +/// `cb` with the camera's homography in `persp` and `layer_fx.y = 1`; unchanged without one. +/// The matrix is composed with `corrected_cover_window` for the box `cb.quad_px`, so a box +/// whose ratio is not the camera's `aspect` (a frame-filling layer, a half of side-by-side) +/// crops the corrected picture instead of stretching it. +pub(crate) fn with_camera_homography( + mut cb: LayerCB, + camera: Option<&crate::scene::SceneCamera>, +) -> LayerCB { + if let Some(h) = camera_homography(camera) { + let box_ar = cb.quad_px[0] / cb.quad_px[1]; + let [u0, v0, du, dv] = corrected_cover_window(camera.and_then(|c| c.aspect), box_ar); + // H * C, with C = [[du, 0, u0], [0, dv, v0], [0, 0, 1]] taking box uv to picture uv. + let h: [f32; 9] = std::array::from_fn(|i| { + let (r, c) = (i / 3, i % 3); + match c { + 0 => h[r * 3] * du, + 1 => h[r * 3 + 1] * dv, + _ => h[r * 3] * u0 + h[r * 3 + 1] * v0 + h[r * 3 + 2], + } + }); + cb.persp = [ + [h[0], h[1], h[2], 0.0], + [h[3], h[4], h[5], 0.0], + [h[6], h[7], h[8], 0.0], + ]; + cb.layer_fx[1] = 1.0; + } + cb +} + +/// The video draw of an extra camera before `camera_layer_cb` places it: mode 0, no background +/// effect (`fx.zw = 0`), `valid` = the valid fraction of its decoder texture, no motion trail. +pub(crate) fn extra_camera_base_cb(valid: [f32; 2]) -> LayerCB { + LayerCB { + mode: 0.0, + color: [0.0, 0.0, 0.0, 1.0], + fx: [valid[0], valid[1], 0.0, 0.0], + mb: [1.0, 0.0, 1.0, 0.0], + ..Default::default() + } +} + +/// The video draw of one planned camera layer: `base` (camera 0's `webcam_video_cb`, or +/// `extra_camera_base_cb` for the others) moved to the plan's rect, with its corners, its +/// source rect (`camera_source_rect`), its homography and its transparency (`1 - opacity`). +/// Planned layers have no motion trail: `dst_prev = dst` and one tap. The exception is camera +/// 0's untouched default layer (the plan's rect is `base.dst`, fully opaque — what a frame +/// outside every layout region plans): it keeps `base`'s trail, as on today's path. +pub(crate) fn camera_layer_cb( + plan: &crate::camera_layers::CameraLayerPlan, + camera: Option<&crate::scene::SceneCamera>, + visible_px: [f32; 2], + tex_px: [f32; 2], + render: [f32; 2], + base: &LayerCB, +) -> LayerCB { + let quad_px = [plan.dst[2] * render[0], plan.dst[3] * render[1]]; + let min_px = quad_px[0].min(quad_px[1]); + let src = camera_source_rect(camera, visible_px, tex_px, quad_px[0] / quad_px[1].max(0.0001)); + let cover = base.cover[0]; + let untouched = plan.dst == base.dst && plan.opacity >= 1.0; + let (dst_prev, mb) = if untouched { + (base.dst_prev, base.mb) + } else { + (plan.dst, [1.0, 0.0, base.mb[2], base.mb[3]]) + }; + with_camera_homography( + LayerCB { + dst: plan.dst, + src, + quad_px, + radius_px: plan.radius_frac * min_px, + src_prev: src, + dst_prev, + mb, + cover: [cover, 0.04 * min_px * cover, base.cover[2], base.cover[3]], + persp: [[0.0; 4]; 3], + layer_fx: [1.0 - plan.opacity.clamp(0.0, 1.0), 0.0, 0.0, 0.0], + ..*base + }, + camera, + ) +} /// Rétrécit un rect SOURCE déjà exprimé en UV (`[u0, v0, u1, v1]`) autour de son /// centre pour qu'il porte le ratio `box_ar` une fois rapporté aux pixels de la /// texture. C'est la forme générale de `object-fit: cover`, et LA primitive qui @@ -1590,6 +1775,54 @@ fn same_source_path(a: &str, b: &str) -> bool { /// shader un temps borné, là où un `f32` de plusieurs heures perdrait la finesse du bruit. pub const WALLPAPER_MOTION_PERIOD_S: f32 = 120.0; +/// Ce qui décide à lui seul du fond composé — sa description, la force de son flou, la couleur +/// de repli, la taille de rendu —, gardé avec sa copie (`bg_cache`) : tant que le fond demandé +/// lui est égal, le backend recopie le fond de la frame d'avant au lieu de le redessiner, ce qui +/// coûtait plus que l'écran lui-même (1,65 ms par frame 1080p sur une Radeon 610M, un wallpaper +/// 4K échantillonné sans mips). +/// +/// La comparaison de chaque frame (`matches`) ne construit ni ne copie rien : une image en data +/// URI pèse plusieurs Mo. Seul `of`, au moment de garder un nouveau fond, la clone. +#[derive(Clone, PartialEq)] +pub struct BackgroundKey { + background: Option, + blur: f32, + fallback: [f32; 4], + render_px: [f32; 2], +} + +impl BackgroundKey { + /// La clé d'un fond fixe ; `None` pour un fond animé, qui change à chaque frame. + pub fn of( + background: Option<&crate::scene::SceneBackground>, + blur: f32, + fallback: [f32; 4], + render_px: [f32; 2], + ) -> Option { + use crate::scene::{SceneBackground as B, WallpaperMotion}; + if let Some(B::Gradient { motion, .. } | B::Image { motion, .. }) = background { + if *motion != WallpaperMotion::None { + return None; + } + } + Some(BackgroundKey { background: background.cloned(), blur, fallback, render_px }) + } + + /// Le fond demandé est celui-ci. + pub fn matches( + &self, + background: Option<&crate::scene::SceneBackground>, + blur: f32, + fallback: [f32; 4], + render_px: [f32; 2], + ) -> bool { + self.background.as_ref() == background + && self.blur == blur + && self.fallback == fallback + && self.render_px == render_px + } +} + /// Emplacements libres des modes 5 (dégradé) et 6 (image) pour le fond animé : `fx.zw` = (temps /// programme replié, indice du mouvement), `mb.x` = aspect w/h du rect rempli (les nappes de /// l'aurore restent rondes). @@ -1849,6 +2082,14 @@ pub struct FrameGeometry { pub w_px: [f32; 2], pub w_radius: f32, pub shape_fade: f32, + /// Orientation of the webcam this frame (mirror, desk-shot turn, crop bypass). + pub webcam: WebcamOrientation, + /// Desk-view cover strength of the webcam this frame, 0..1 (`camera_fullscreen_cover_at`). + pub webcam_cover: f32, + /// Every camera layer to draw this frame, in draw order (`camera_layers::camera_layers_at`). + /// Empty when the scene has no camera layout regions: the backends then draw camera 0 from + /// `w_dst` as before. Otherwise camera 0 is in here too, and its `dst`/opacity come from it. + pub camera_layers: Vec, /// Cadre autour de l'écran : chrome de fenêtre plat (mode 14) ou appareil modelé (mode 17). /// `None` : aucun, et le rendu est celui d'avant le cadre, à l'octet. `Some` : `s_dst` est /// déjà la boîte rétrécie, et `s_radius` le rayon des coins de l'écran — des seuls coins BAS @@ -2087,6 +2328,109 @@ impl FrameGeometry { self.tilt_trail(render_px).filter(|_| !self.screen_trail(render_px)) } + /// The camera-0 video draw (mode 0), the same on every backend. `src` = the source rect with + /// the mirror / desk turn already applied as swapped bounds (it is also `src_prev`: only + /// `dst_prev` carries the motion), `valid` = the valid fraction of the decoder texture, + /// `effect_code` / `blur_intensity` = the background effect sent to the shader. + pub fn webcam_video_cb( + &self, + src: [f32; 4], + valid: [f32; 2], + effect_code: f32, + blur_intensity: f32, + ) -> LayerCB { + LayerCB { + dst: self.w_dst, + src, + quad_px: self.w_px, + radius_px: self.w_radius, + mode: 0.0, + // `color.a` carries the cutout alpha (`color.a * person`); the RGB is not read, the + // background has already been painted under the camera. + color: [0.0, 0.0, 0.0, 1.0], + fx: [valid[0], valid[1], effect_code, blur_intensity], + src_prev: src, + dst_prev: self.w_dst_prev, + mb: [self.mb_taps, self.mb_amount, 1.0, 0.0], + cover: [ + self.webcam_cover, + 0.04 * self.w_px[0].min(self.w_px[1]) * self.webcam_cover, + 0.35, + 0.0, + ], + ..Default::default() + } + } + + /// Camera 0's settings as a `SceneCamera`, for `camera_source_rect` / `camera_layer_cb`: + /// its orientation is the project mirror and the desk-view turn (`self.webcam`, expressed so + /// that `mirror ^ turned` gives back `flip_u`), its crop is the layout's (none for a + /// full-frame desk shot), and only its homography and the corrected picture's aspect (so a + /// box of another ratio cover-crops it, `with_camera_homography`) come from + /// `Scene::camera(0)`. + pub(crate) fn camera0_settings(&self, scene: Option<&Scene>) -> crate::scene::SceneCamera { + let turned = self.webcam.flip_v; + let settings = scene.and_then(|s| s.camera(0)); + crate::scene::SceneCamera { + index: 0, + rotation: if turned { 180 } else { 0 }, + mirror: Some(self.webcam.flip_u ^ turned), + crop: if self.webcam.full_frame { + None + } else { + scene.and_then(|s| s.layout.webcam_crop) + }, + homography: settings.and_then(|c| c.homography), + aspect: settings.and_then(|c| c.aspect), + } + } + + /// The drop shadow under a planned camera layer, `video` being its `camera_layer_cb`. Same + /// constants as camera 0's PiP shadow, and it fades with the layer (`video.layer_fx.x`). + /// None when `shadows_on` is off or the layer fills the frame. Camera 0 also keeps today's + /// rules: no shadow in the block presets nor in cutout (`camera0_cutout`), and it leaves + /// with the bubble (`shape_fade`). + pub(crate) fn camera_layer_shadow( + &self, + plan: &crate::camera_layers::CameraLayerPlan, + video: &LayerCB, + render: [f32; 2], + shadows_on: bool, + camera0_cutout: bool, + ) -> Option { + let strength = if plan.camera == 0 { + let block = matches!( + self.scene_preset.as_deref(), + Some("dual-frame") | Some("vertical-stack") + ); + if block || camera0_cutout { + return None; + } + WEBCAM_SHADOW_OPACITY * self.shape_fade + } else { + WEBCAM_SHADOW_OPACITY + }; + if !shadows_on || plan.fills_frame || strength <= 0.0 { + return None; + } + let spread = WEBCAM_SHADOW_SPREAD_FRAC * self.frame_min_px; + let offset = WEBCAM_SHADOW_OFFSET_FRAC * self.frame_min_px; + let (sx, sy) = (spread / render[0].max(1.0), spread / render[1].max(1.0)); + let oy = offset / render[1].max(1.0); + let dst = video.dst; + Some(LayerCB { + dst: [dst[0] - sx, dst[1] - sy + oy, dst[2] + 2.0 * sx, dst[3] + 2.0 * sy], + quad_px: [video.quad_px[0] + 2.0 * spread, video.quad_px[1] + 2.0 * spread], + radius_px: video.radius_px, + mode: 2.0, + color: [0.0, 0.0, 0.0, strength], + fx: [spread, 0.0, 0.0, 0.0], + mb: [0.0, 1.0, 1.0, 0.0], + layer_fx: [video.layer_fx[0], 0.0, 0.0, 0.0], + ..Default::default() + }) + } + /// Le calque du mode 18 : le rendu isolé de l'écran cadré (t2), recomposé le long de sa /// trajectoire. `src` = la coupe (le métrage relu directement là où le rendu isolé s'arrête /// au bord de la sortie), `quad_px` et `radius_px` = l'écran et ses coins : ce repli ne lit le @@ -2658,10 +3002,6 @@ impl FrameGeometry { [centre[0] + px, centre[1] + py] }; let pts = [at(x0, y0), at(x1, y0), at(x1, y1), at(x0, y1)]; - // Le mode 8 étale aussi chaque pixel vers là où le plan était une frame plus tôt - // (`tilt_trail`) : une copie traînée du secret sort du quad courant. Le masque couvre alors - // la boîte englobante du secret aux deux frames, en rect droit, comme le chemin droit. - // Même warp que le shader sur les coins d'avant (le drapeau projectif est celui du plan). // La perspective grossit localement le côté proche : la force suit l'arête la plus // agrandie par rapport au rect droit zoomé, pour que nulle part le masque ne soit plus // fin, rapporté au contenu, qu'au repos. @@ -2672,28 +3012,29 @@ impl FrameGeometry { .max(len(pts[0], pts[3]) / eh) .max(len(pts[1], pts[2]) / eh); let strength = (zoom_k * tilt_k).max(1.0); - // Sans mouvement, la traînée EST le quad (au bit près) : rien de plus à couvrir. - if let Some(trail) = self.tilt_trail(render_px).filter(|t| t.corners != quad.corners) { - let before = crate::regions::TiltedQuad { corners: trail.corners, ..quad }; - let then = |fx: f32, fy: f32| { - let (px, py) = before.point_px(fx, fy); - [centre[0] + px, centre[1] + py] - }; - let (mut lo, mut hi) = ([f32::MAX; 2], [f32::MIN; 2]); - for [px, py] in pts.into_iter().chain([then(x0, y0), then(x1, y0), then(x1, y1), then(x0, y1)]) { - (lo, hi) = ([lo[0].min(px), lo[1].min(py)], [hi[0].max(px), hi[1].max(py)]); + // Le mode 8 étale aussi chaque pixel vers là où le plan était une frame plus tôt + // (`tilt_trail`) : une copie traînée du secret sort du quad courant. Le masque garde la + // perspective du quad courant, élargi de la plus grande avance d'un de ses coins : chaque + // coin d'avant tombe dedans, donc tout le secret d'avant aussi (un quad convexe contient + // l'enveloppe de ses coins), et la traînée, qui va de l'un à l'autre. Même warp que le + // shader sur les coins d'avant (le drapeau projectif est celui du plan). Sans mouvement, + // la traînée EST le quad (au bit près) : rien de plus à couvrir. + let trail = self.tilt_trail(render_px).filter(|t| t.corners != quad.corners); + let pts = match trail { + Some(trail) => { + let before = crate::regions::TiltedQuad { corners: trail.corners, ..quad }; + let then = |fx: f32, fy: f32| { + let (px, py) = before.point_px(fx, fy); + [centre[0] + px, centre[1] + py] + }; + let prev = [then(x0, y0), then(x1, y0), then(x1, y1), then(x0, y1)]; + let reach = (0..4) + .map(|i| (prev[i][0] - pts[i][0]).hypot(prev[i][1] - pts[i][1])) + .fold(0.0f32, f32::max); + grow_quad(pts, reach + 0.5) } - let rect = - [lo[0] / rw, lo[1] / rh, (hi[0] - lo[0]).max(1.0) / rw, (hi[1] - lo[1]).max(1.0) / rh]; - let rect = match self.screen_mask { - Some(m) => m.clip(rect)?, - None => rect, - }; - let mut mask = PrivacyMask::upright(rect, render_px, strength); - // Un ovale inscrit dans la boîte élargie ne couvrirait plus l'ovale courant. - mask.oval_ok = false; - return Some(mask); - } + None => pts, + }; let (mut min_x, mut min_y) = (f32::MAX, f32::MAX); let (mut max_x, mut max_y) = (f32::MIN, f32::MIN); for [px, py] in pts { @@ -2721,9 +3062,65 @@ impl FrameGeometry { warp: Some(local), projective: quad.projective, strength, - oval_ok: true, + // Un ovale inscrit dans le quad élargi ne couvrirait plus sûrement l'ovale d'avant. + oval_ok: trail.is_none(), }) } + + /// Où tombe le métrage dans l'image (`FootageQuad`) : là où `privacy_mask` pose un masque, + /// donc là où l'éditeur doit poser le gimbal d'un flou pour qu'il couvre ce qu'il cache. + pub fn footage_quad(&self, render_px: [f32; 2]) -> FootageQuad { + let [rw, rh] = render_px; + let s_px = [self.s_dst[2] * rw, self.s_dst[3] * rh]; + if let Some(quad) = self.screen_tilt(s_px).filter(|_| self.tilted() && rw > 0.0 && rh > 0.0) { + let centre = self.screen_center_px(render_px); + let at = |fx: f32, fy: f32| { + let (px, py) = quad.point_px(fx, fy); + [(centre[0] + px) / rw, (centre[1] + py) / rh] + }; + return FootageQuad { + corners: [at(0.0, 0.0), at(1.0, 0.0), at(1.0, 1.0), at(0.0, 1.0)], + projective: quad.projective, + }; + } + let [x, y, w, h] = self.s_dst; + FootageQuad { corners: [[x, y], [x + w, y], [x + w, y + h], [x, y + h]], projective: false } + } +} + +/// Le métrage dans l'image : ses coins TL, TR, BR, BL en fractions de la sortie, et la +/// correspondance d'un point `(u, v)` du métrage à l'image, bilinéaire entre eux ou, sous la +/// caméra réelle, l'homographie qu'ils définissent (`TiltedQuad::point_px`). +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct FootageQuad { + pub corners: [[f32; 2]; 4], + pub projective: bool, +} + +/// Le quad convexe `pts` (TL, TR, BR, BL) dont chaque côté recule de `d` px vers l'extérieur : +/// ses côtés restent parallèles aux siens, donc sa perspective aussi, et il contient tout point +/// à moins de `d` de lui. +fn grow_quad(pts: [[f32; 2]; 4], d: f32) -> [[f32; 2]; 4] { + // L'extérieur d'un côté est d'un bord ou de l'autre selon le sens de parcours. + let area: f32 = (0..4) + .map(|i| { + let (a, b) = (pts[i], pts[(i + 1) % 4]); + a[0] * b[1] - b[0] * a[1] + }) + .sum(); + let side = area.signum(); + let normal = |i: usize| { + let (a, b) = (pts[i], pts[(i + 1) % 4]); + let (ex, ey) = (b[0] - a[0], b[1] - a[1]); + let len = ex.hypot(ey).max(1e-6); + [side * ey / len, -side * ex / len] + }; + std::array::from_fn(|i| { + // Le coin glisse sur la bissectrice jusqu'à la rencontre des deux côtés reculés. + let (na, nb) = (normal((i + 3) % 4), normal(i)); + let k = d / (1.0 + na[0] * nb[0] + na[1] * nb[1]).max(0.2); + [pts[i][0] + (na[0] + nb[0]) * k, pts[i][1] + (na[1] + nb[1]) * k] + }) } /// Marge ajoutée autour d'un masque de confidentialité, en px de la boîte écran AU REPOS : un @@ -2873,6 +3270,9 @@ pub fn plan_frame(input: &FrameGeometryInput) -> FrameGeometry { let zoom_regions = scene.map(|s| &s.zoom_regions).unwrap_or(&empty_zoom); let cam_regions = scene.map(|s| &s.camera_fullscreen_regions).unwrap_or(&empty_cam); + // The layout regions, for the seams a Full Camera region shares with them. + let empty_layouts: Vec = Vec::new(); + let layouts = scene.map(|s| &s.camera_layout_regions).unwrap_or(&empty_layouts); let webcam_reactive = scene.map(|s| s.layout.webcam_reactive_zoom).unwrap_or(false); let source_t = input.timeline_t_override.unwrap_or(frame / FPS); // Les transitions se mesurent à l'écran (`ScreenClock`), la frame précédente aussi : une @@ -2891,6 +3291,8 @@ pub fn plan_frame(input: &FrameGeometryInput) -> FrameGeometry { let mut zoom_tilt = 0.0f32; let mut zoom_camera = 0.0f32; let mut zoom_camera_prev = 0.0f32; + let mut zoom_strength = 0.0f32; + let mut zoom_strength_prev = 0.0f32; let mut zoom_aim = [0.5f32; 2]; let mut zoom_orbit = [0.5f32; 2]; let (mut zoom_aim_prev, mut zoom_orbit_prev) = ([0.5f32; 2], [0.5f32; 2]); @@ -2919,6 +3321,7 @@ pub fn plan_frame(input: &FrameGeometryInput) -> FrameGeometry { zoom_rotation = zs.rotation; zoom_tilt = zs.tilt; zoom_camera = zs.camera; + zoom_strength = zs.strength; zoom_aim = zs.aim; zoom_orbit = zs.orbit; // Même `CameraFrame` qu'à la frame courante : la caméra réelle d'avant (poids, visée, @@ -2934,24 +3337,38 @@ pub fn plan_frame(input: &FrameGeometryInput) -> FrameGeometry { pp.focus = zs_p.focus; zoom_rotation_prev = zs_p.rotation; zoom_camera_prev = zs_p.camera; + zoom_strength_prev = zs_p.strength; zoom_aim_prev = zs_p.aim; zoom_orbit_prev = zs_p.orbit; } // Full Camera ignore le rétrécissement réactif de la webcam (design web : mélanger // "rétrécit pour le zoom" et "grandit en plein cadre" dans la même frame n'a pas de sens). let cam_progress = - crate::regions::camera_fullscreen_progress_at(cam_regions, source_t, &clock); + crate::regions::camera_fullscreen_progress_at(cam_regions, source_t, &clock, layouts); let cam_progress_prev = - crate::regions::camera_fullscreen_progress_at(cam_regions, source_t_prev, &clock); + crate::regions::camera_fullscreen_progress_at( + cam_regions, + source_t_prev, + &clock, + layouts, + ); let shape_fade = - crate::regions::camera_fullscreen_shape_at(cam_regions, source_t, &clock); - // rétrécissement réactif : la webcam rétrécit pendant un zoom actif, de la MOITIÉ de ce - // que prendrait 1/zoom : ×0,78 au zoom par défaut (1,8), ×0,6 au zoom maximal (5). En - // 1/zoom elle tombait à ×0,56 au zoom par défaut et un visage ne se lisait plus. Ignoré - // pendant Full Camera (voir ci-dessus). - let reactive_scale = |zoom: f32, progress: f32| -> f32 { - if webcam_reactive && progress <= 0.0 && zoom.is_finite() && zoom > 0.0 { - ((1.0 + 1.0 / zoom) / 2.0).min(1.0) + crate::regions::camera_fullscreen_shape_at(cam_regions, source_t, &clock, layouts); + let webcam = webcam_orientation( + crate::regions::camera_fullscreen_region_at(cam_regions, source_t, &clock, layouts), + lp.webcam_mirror, + ); + let webcam_cover = + crate::regions::camera_fullscreen_cover_at(cam_regions, source_t, &clock, layouts); + // rétrécissement réactif : la webcam garde 70 % de sa taille pendant un zoom actif, quel + // que soit son niveau (elle suivait 1/zoom, et rétrécissait donc d'autant plus que le zoom + // était profond : ×0,6 au zoom maximal). L'enveloppe est celle de la région : elle descend + // avec le ease-in du zoom, tient 0,7 sur le span, remonte avec le ease-out. Une région à + // échelle 1 ne zoome rien : elle laisse la caméra à sa taille. Ignoré pendant Full Camera + // (voir ci-dessus). + let reactive_scale = |strength: f32, zoom: f32, progress: f32| -> f32 { + if webcam_reactive && progress <= 0.0 && zoom > 1.0 && strength.is_finite() { + (1.0 - 0.3 * strength.clamp(0.0, 1.0)).clamp(0.7, 1.0) } else { 1.0 } @@ -2965,8 +3382,9 @@ pub fn plan_frame(input: &FrameGeometryInput) -> FrameGeometry { // Seul `reactive_scale` (rétrécissement pendant un zoom, une valeur ANIMÉE par frame que // le rect statique de l'app ne capture pas) doit encore s'appliquer dans ce cas. let base_size_scale = if app_webcam_rect.is_some() { 1.0 } else { lp.webcam_size_scale }; - let webcam_size_scale = base_size_scale * reactive_scale(p.zoom, cam_progress); - let webcam_size_scale_prev = base_size_scale * reactive_scale(pp.zoom, cam_progress_prev); + let webcam_size_scale = base_size_scale * reactive_scale(zoom_strength, p.zoom, cam_progress); + let webcam_size_scale_prev = + base_size_scale * reactive_scale(zoom_strength_prev, pp.zoom, cam_progress_prev); // padding : échelle globale du layout autour du centre du cadre (parité web frameRenderer : // paddingScale = 1 - padding*0.4 → padding 0 = plein cadre). S'applique à TOUS les presets : @@ -3354,6 +3772,30 @@ pub fn plan_frame(input: &FrameGeometryInput) -> FrameGeometry { _ => 0.12, }, }; + // Camera layout regions: every camera layer of this frame, camera 0 included, on the + // same time and clock as Full Camera. Camera 0's default is the layer just planned + // above, so a region glides from and back to exactly where `w_dst` puts it. Without + // regions nothing is planned, and the backends keep drawing camera 0 from `w_dst`. + let camera_layers = match scene { + Some(s) if !s.camera_layout_regions.is_empty() => { + let default_cam0 = lp.has_webcam.then(|| crate::camera_layers::CameraLayerPlan { + camera: 0, + dst: w_dst, + radius_frac: w_radius / w_px[0].min(w_px[1]).max(1.0), + shape: lp.webcam_shape, + opacity: 1.0, + fills_frame: cam_progress >= 1.0, + }); + crate::camera_layers::camera_layers_at( + &s.camera_layout_regions, + &s.camera_fullscreen_regions, + source_t, + &clock, + default_cam0, + ) + } + _ => Vec::new(), + }; FrameGeometry { scene_preset, @@ -3385,6 +3827,9 @@ pub fn plan_frame(input: &FrameGeometryInput) -> FrameGeometry { w_px, w_radius, shape_fade, + webcam, + webcam_cover, + camera_layers, window_frame, screen_mask, } @@ -3419,6 +3864,10 @@ pub struct CursorPlan { /// L'impact des clics récents sur l'écran (mode 16), à dessiner SOUS le curseur. Vide sans /// curseur modélisé, sans clic récent ou à `clickBounce` nul. pub impacts: Vec, + /// Le curseur modélisé est le cristal de Prism Glow : il réfracte l'image déjà composée, le + /// métrage et ses flous de confidentialité, que le backend lui copie juste avant de le + /// dessiner. Seuls des pixels floutés passent ainsi à travers le verre. + pub glass: bool, } impl CursorPlan { @@ -3720,6 +4169,7 @@ pub fn plan_cursor(g: &FrameGeometry, input: &CursorPlanInput) -> Option Option, x: f32, y: f32| { let anchor = anchor.map(|a| format!(r#","webcamAnchor":"{a}""#)).unwrap_or_default(); let layout = format!( @@ -6490,19 +6985,56 @@ mod tests { plan_frame(&golden_input(&scene, &cfg)).w_dst }; for (anchor, x, y, want) in [ - (Some("bottom-right"), 0.77, 0.77, [0.82, 0.82]), + (Some("bottom-right"), 0.77, 0.77, [0.83, 0.83]), (Some("top-left"), 0.03, 0.03, [0.03, 0.03]), - (Some("bottom"), 0.4, 0.77, [0.425, 0.82]), - (Some("left"), 0.03, 0.4, [0.03, 0.425]), - (None, 0.77, 0.77, [0.795, 0.795]), + (Some("bottom"), 0.4, 0.77, [0.43, 0.83]), + (Some("left"), 0.03, 0.4, [0.03, 0.43]), + (None, 0.77, 0.77, [0.80, 0.80]), ] { let w = camera_at(anchor, x, y); let got = [w[0], w[1], w[2], w[3]]; - let want = [want[0], want[1], 0.15, 0.15]; + let want = [want[0], want[1], 0.14, 0.14]; assert!(got.iter().zip(want).all(|(g, w)| (g - w).abs() < 1e-5), "{anchor:?} : {got:?} au lieu de {want:?}"); } } + /// Le rétrécissement réactif vise 70 % QUEL QUE SOIT le niveau de zoom : la caméra garde la + /// même taille à 1,25×, 2× et 5× (elle suivait 1/zoom, et rétrécissait d'autant plus que le + /// zoom était profond). + #[test] + fn the_camera_shrink_is_the_same_at_every_zoom_level() { + let cfg = crate::config::all().pop().expect("au moins une config"); + let camera_at = |scale: &str| { + let layout = r#""webcamReactiveZoom":true,"webcamAnchor":"bottom-right","webcamRect":{"x":0.77,"y":0.77,"width":0.2,"height":0.2}}"#; + let json = zoomed_golden_scene_json() + .replace(r#""webcamReactiveZoom":false}"#, layout) + .replace(r#""scale":2.0"#, scale); + let scene = Scene::from_json(&json).expect("scène"); + plan_frame(&golden_input(&scene, &cfg)).w_dst + }; + for scale in [r#""scale":1.25"#, r#""scale":2.0"#, r#""scale":5.0"#] { + let w = camera_at(scale); + let got = [w[0], w[1], w[2], w[3]]; + let want = [0.83, 0.83, 0.14, 0.14]; + assert!(got.iter().zip(want).all(|(g, w)| (g - w).abs() < 1e-5), "{scale} : {got:?} au lieu de {want:?}"); + } + } + + /// Une région de zoom à échelle 1 ne zoome rien : elle ne rétrécit pas la caméra. + #[test] + fn a_one_x_zoom_region_keeps_the_camera_at_its_size() { + let cfg = crate::config::all().pop().expect("au moins une config"); + let layout = r#""webcamReactiveZoom":true,"webcamAnchor":"bottom-right","webcamRect":{"x":0.77,"y":0.77,"width":0.2,"height":0.2}}"#; + let json = zoomed_golden_scene_json() + .replace(r#""webcamReactiveZoom":false}"#, layout) + .replace(r#""scale":2.0"#, r#""scale":1.0"#); + let scene = Scene::from_json(&json).expect("scène"); + let w = plan_frame(&golden_input(&scene, &cfg)).w_dst; + let got = [w[0], w[1], w[2], w[3]]; + let want = [0.77, 0.77, 0.2, 0.2]; + assert!(got.iter().zip(want).all(|(g, w)| (g - w).abs() < 1e-5), "{got:?} au lieu de {want:?}"); + } + /// La vitesse se mesure dans la coupe VISIBLE, zoom compris : sous un x2, le même geste /// traverse deux fois plus d'écran et penche donc plus le plan (hors saturation). #[test] @@ -6839,6 +7371,87 @@ mod tests { } } + /// Le point `p` est dans le quad convexe `q` (TL, TR, BR, BL, px de sortie). + fn in_convex_quad(q: [[f32; 2]; 4], p: [f32; 2]) -> bool { + let side = |i: usize| { + let (a, b) = (q[i], q[(i + 1) % 4]); + (b[0] - a[0]) * (p[1] - a[1]) - (b[1] - a[1]) * (p[0] - a[0]) + }; + (0..4).all(|i| side(i) >= -1e-3) || (0..4).all(|i| side(i) <= 1e-3) + } + + /// Sous l'orbite, la caméra suit le pointeur et le flou de mouvement étale le contenu vers + /// sa position de la frame d'avant : le masque garde la perspective du contenu (un quad warpé + /// aux côtés parallèles aux siens, pas un rect droit) et couvre le secret aux deux frames. + #[test] + fn a_moving_tilted_mask_keeps_its_perspective() { + let cfg = crate::config::all().pop().expect("au moins une config"); + let json = zoomed_golden_scene_json() + .replace(r#""rotation":"none""#, r#""rotation":"orbit","focusMode":"auto""#); + let scene = Scene::from_json(&json).expect("scène"); + // Le pointeur file vers la droite : l'orbite le suit, la caméra bouge à chaque frame. + let track: &'static crate::cursor::CursorTrack = Box::leak(Box::new(crate::cursor::CursorTrack::new( + (0..=90).map(|i| (i as f32 / 30.0, 0.05 + 0.5 * i as f32 / 90.0, 0.3)).collect(), + vec![], + vec![], + ))); + let g = plan_frame(&FrameGeometryInput { cursor: Some(track), ..golden_input(&scene, &cfg) }); + let render = [1170.0, 658.0]; + let quad = g.screen_tilt([g.s_dst[2] * render[0], g.s_dst[3] * render[1]]).expect("orbite"); + let trail = g.tilt_trail(render).expect("garde : flou de mouvement actif sous l'orbite"); + assert_ne!(trail.corners, quad.corners, "garde : la caméra doit bouger d'une frame à l'autre"); + let a = blur_annotation(""); + let m = g.privacy_mask(&a, render).expect("masque"); + let warp = m.warp.expect("le masque d'un écran incliné qui bouge reste un quad warpé"); + let origin = [m.dst[0] * render[0], m.dst[1] * render[1]]; + let mask = warp.map(|[x, y]| [origin[0] + x, origin[1] + y]); + let centre = [(g.s_dst[0] + g.s_dst[2] * 0.5) * render[0], (g.s_dst[1] + g.s_dst[3] * 0.5) * render[1]]; + let before = crate::regions::TiltedQuad { corners: trail.corners, ..quad }; + for q in [quad, before] { + for (fx, fy) in [(a.x, a.y), (a.x + a.w, a.y), (a.x + a.w, a.y + a.h), (a.x, a.y + a.h)] { + let (px, py) = q.point_px(fx, fy); + let p = [centre[0] + px, centre[1] + py]; + assert!(in_convex_quad(mask, p), "{p:?} hors du masque {mask:?}"); + } + } + let (tl, tr) = (quad.point_px(a.x, a.y), quad.point_px(a.x + a.w, a.y)); + let content = (tr.1 - tl.1) / (tr.0 - tl.0); + assert!(content.abs() > 1e-3, "garde : l'orbite doit incliner le haut du secret ({content})"); + let top = (mask[1][1] - mask[0][1]) / (mask[1][0] - mask[0][0]); + assert!((top - content).abs() < 1e-3, "le haut du masque ({top}) doit suivre celui du contenu ({content})"); + assert!(!m.oval_ok, "un masque élargi à la trace doit refuser l'ovale"); + } + + /// Le métrage que l'éditeur lit avec l'image : le rect zoomé à plat, le quad incliné du mode 8 + /// sous un préset, l'homographie de ses coins sous la caméra réelle. + #[test] + fn the_footage_quad_is_where_masks_land() { + let cfg = crate::config::all().pop().expect("au moins une config"); + let render = [1170.0, 658.0]; + let flat = plan_frame(&golden_input(&zoomed_golden_scene(), &cfg)); + let [x, y, w, h] = flat.s_dst; + assert_eq!( + flat.footage_quad(render), + FootageQuad { corners: [[x, y], [x + w, y], [x + w, y + h], [x, y + h]], projective: false } + ); + let tilted = plan_frame(&golden_input(&tilted_golden_scene(), &cfg)); + let q = tilted.footage_quad(render); + let quad = tilted.screen_tilt([tilted.s_dst[2] * render[0], tilted.s_dst[3] * render[1]]).expect("iso"); + let centre = [ + (tilted.s_dst[0] + tilted.s_dst[2] * 0.5) * render[0], + (tilted.s_dst[1] + tilted.s_dst[3] * 0.5) * render[1], + ]; + for (k, (cx, cy)) in quad.corners.iter().enumerate() { + let want = [(centre[0] + cx) / render[0], (centre[1] + cy) / render[1]]; + assert!((q.corners[k][0] - want[0]).abs() < 1e-5 && (q.corners[k][1] - want[1]).abs() < 1e-5, "coin {k}"); + } + assert!(!q.projective); + let json = zoomed_golden_scene_json() + .replace(r#""rotation":"none""#, r#""rotation":"orbit","focusMode":"auto""#); + let orbit = plan_frame(&golden_input(&Scene::from_json(&json).expect("scène"), &cfg)); + assert!(orbit.footage_quad(render).projective, "la caméra réelle warpe par homographie"); + } + /// Une entrée mesurée sur le cadre de sortie n'est pas concernée par le zoom. #[test] fn a_privacy_mask_in_frame_space_ignores_the_zoom() { @@ -7016,7 +7629,7 @@ mod tests { #[test] fn layer_cb_matches_the_shader_constant_buffer() { use std::mem::{align_of, offset_of, size_of}; - assert_eq!(size_of::(), 176); + assert_eq!(size_of::(), 256); assert_eq!(align_of::(), 16); for (name, got, want) in [ ("dst", offset_of!(LayerCB, dst), 0), @@ -7032,11 +7645,527 @@ mod tests { ("trail_a", offset_of!(LayerCB, trail_a), 128), ("trail_b", offset_of!(LayerCB, trail_b), 144), ("trail_mb", offset_of!(LayerCB, trail_mb), 160), + ("cover", offset_of!(LayerCB, cover), 176), + ("persp", offset_of!(LayerCB, persp), 192), + ("layer_fx", offset_of!(LayerCB, layer_fx), 240), ] { assert_eq!(got, want, "offset de `{name}`"); } } + /// The camera-0 draw as every backend builds it today leaves both new lanes at zero: no + /// homography (`layer_fx.y = 0`) and opaque (`layer_fx.x = 0`), so the shaders take exactly + /// the path they took before the lanes existed. The other fields are the ones the three + /// backends used to spell out inline. + #[test] + fn the_webcam_draw_leaves_the_homography_and_transparency_lanes_at_zero() { + let cfg = crate::config::all().pop().expect("cfg"); + let scene = golden_scene(); + let g = plan_frame(&golden_input(&scene, &cfg)); + let src = [0.1, 0.2, 0.9, 0.8]; + let cb = g.webcam_video_cb(src, [1.0, 0.75], 2.0, 0.4); + assert_eq!(cb.persp, [[0.0; 4]; 3]); + assert_eq!(cb.layer_fx, [0.0; 4]); + assert_eq!(cb.dst, g.w_dst); + assert_eq!(cb.src, src); + assert_eq!(cb.src_prev, src); + assert_eq!(cb.quad_px, g.w_px); + assert_eq!(cb.radius_px, g.w_radius); + assert_eq!(cb.mode, 0.0); + assert_eq!(cb.color, [0.0, 0.0, 0.0, 1.0]); + assert_eq!(cb.fx, [1.0, 0.75, 2.0, 0.4]); + assert_eq!(cb.dst_prev, g.w_dst_prev); + assert_eq!(cb.mb, [g.mb_taps, g.mb_amount, 1.0, 0.0]); + let min_px = g.w_px[0].min(g.w_px[1]); + assert_eq!(cb.cover, [g.webcam_cover, 0.04 * min_px * g.webcam_cover, 0.35, 0.0]); + } + + fn extra_camera(index: usize) -> crate::scene::SceneCamera { + crate::scene::SceneCamera { + index, + rotation: 0, + mirror: None, + crop: None, + homography: None, + aspect: None, + } + } + + fn planned(camera: usize, opacity: f32) -> crate::camera_layers::CameraLayerPlan { + crate::camera_layers::CameraLayerPlan { + camera, + dst: [0.5, 0.25, 0.25, 0.5], + radius_frac: 0.2, + shape: 0, + opacity, + fills_frame: false, + } + } + + /// One planned layer's video draw: placed at the plan's rect with its corners, no trail, + /// the transparency is `1 - opacity`, and the camera's homography replaces its crop. + #[test] + fn camera_layer_cb_places_the_layer_and_carries_its_lanes() { + let render = [1920.0, 1080.0]; + let (visible, tex) = ([1280.0, 720.0], [1280.0, 736.0]); + let base = extra_camera_base_cb([1.0, 720.0 / 736.0]); + let plan = planned(1, 0.25); + let plain = camera_layer_cb(&plan, None, visible, tex, render, &base); + assert_eq!(plain.dst, plan.dst); + assert_eq!(plain.dst_prev, plan.dst); + assert_eq!(plain.quad_px, [480.0, 540.0]); + assert_eq!(plain.radius_px, 0.2 * 480.0); + assert_eq!(plain.layer_fx, [0.75, 0.0, 0.0, 0.0]); + assert_eq!(plain.persp, [[0.0; 4]; 3]); + assert_eq!(plain.src_prev, plain.src); + assert_eq!(plain.mb[0], 1.0); + assert_eq!(plain.fx, base.fx); + + // A crop moves the source rect; with a homography as well, the crop is ignored. + let crop = SceneCrop { x: 0.5, y: 0.0, width: 0.5, height: 1.0 }; + let cropped = extra_camera(1); + let cropped = crate::scene::SceneCamera { crop: Some(crop), ..cropped }; + let with_crop = camera_layer_cb(&plan, Some(&cropped), visible, tex, render, &base); + assert_ne!(with_crop.src, plain.src); + let h = [-1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0]; + let corrected = crate::scene::SceneCamera { homography: Some(h), ..cropped.clone() }; + let warped = camera_layer_cb(&plan, Some(&corrected), visible, tex, render, &base); + assert_eq!(warped.layer_fx[1], 1.0); + assert_eq!(warped.layer_fx[0], 0.75); + let rows = [[-1.0, 0.0, 1.0, 0.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0]]; + assert_eq!(warped.persp, rows); + assert_eq!(warped.src, plain.src); + + // Mirror swaps the u bounds only; 180° swaps both, and mirror on top of it un-swaps u. + let mirrored = crate::scene::SceneCamera { mirror: Some(true), ..extra_camera(1) }; + let m = camera_layer_cb(&plan, Some(&mirrored), visible, tex, render, &base).src; + assert_eq!(m, [plain.src[2], plain.src[1], plain.src[0], plain.src[3]]); + let turned = crate::scene::SceneCamera { rotation: 180, ..extra_camera(1) }; + let t = camera_layer_cb(&plan, Some(&turned), visible, tex, render, &base).src; + assert_eq!(t, [plain.src[2], plain.src[3], plain.src[0], plain.src[1]]); + let both = crate::scene::SceneCamera { mirror: Some(true), ..turned.clone() }; + let b = camera_layer_cb(&plan, Some(&both), visible, tex, render, &base).src; + assert_eq!(b, [plain.src[0], plain.src[3], plain.src[2], plain.src[1]]); + } + + /// Camera 0's default layer, planned outside every layout region at its own rect and fully + /// opaque, keeps its motion trail; moved or faded, it draws without one (R8). + #[test] + fn camera_0s_untouched_layer_keeps_its_trail() { + let render = [1920.0, 1080.0]; + let base = LayerCB { + dst: [0.7, 0.7, 0.2, 0.2], + dst_prev: [0.65, 0.7, 0.2, 0.2], + mb: [8.0, 0.5, 1.0, 0.0], + ..Default::default() + }; + let plan = crate::camera_layers::CameraLayerPlan { + camera: 0, + dst: base.dst, + radius_frac: 0.2, + shape: 0, + opacity: 1.0, + fills_frame: false, + }; + let kept = camera_layer_cb(&plan, None, [64.0; 2], [64.0; 2], render, &base); + assert_eq!(kept.dst_prev, base.dst_prev); + assert_eq!(kept.mb, base.mb); + + let moved = crate::camera_layers::CameraLayerPlan { dst: [0.1, 0.1, 0.2, 0.2], ..plan }; + let m = camera_layer_cb(&moved, None, [64.0; 2], [64.0; 2], render, &base); + assert_eq!(m.dst_prev, moved.dst); + assert_eq!(m.mb, [1.0, 0.0, 1.0, 0.0]); + + let faded = crate::camera_layers::CameraLayerPlan { opacity: 0.5, ..plan }; + let f = camera_layer_cb(&faded, None, [64.0; 2], [64.0; 2], render, &base); + assert_eq!(f.dst_prev, faded.dst); + assert_eq!(f.mb, [1.0, 0.0, 1.0, 0.0]); + } + + /// A corrected camera in a box of another ratio is cover-cropped, not stretched: the box's + /// local uv lands on a centred window of the corrected picture with the box's own ratio. + #[test] + fn a_corrected_camera_is_cover_fitted_into_its_box() { + let identity = [1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0]; + let cam = crate::scene::SceneCamera { + homography: Some(identity), + aspect: Some(16.0 / 9.0), + ..extra_camera(2) + }; + let row = |cb: &LayerCB, r: usize| [cb.persp[r][0], cb.persp[r][1], cb.persp[r][2]]; + let near = |a: [f32; 3], b: [f32; 3]| a.iter().zip(b).all(|(x, y)| (x - y).abs() < 1e-5); + + // A box narrower than 16:9 (1752x1080, a 1.62 export): the full height, a centred + // slice of the width with the box's ratio. + let narrow = LayerCB { quad_px: [1752.0, 1080.0], ..Default::default() }; + let cb = with_camera_homography(narrow, Some(&cam)); + let du = (1752.0 / 1080.0) / (16.0 / 9.0); + assert!(near(row(&cb, 0), [du, 0.0, (1.0 - du) * 0.5]), "{:?}", cb.persp); + assert!(near(row(&cb, 1), [0.0, 1.0, 0.0]), "{:?}", cb.persp); + assert!(near(row(&cb, 2), [0.0, 0.0, 1.0]), "{:?}", cb.persp); + + // A wider box (a 16:9 picture in a 21:9 box): full width, a centred band of height. + let wide = LayerCB { quad_px: [2100.0, 900.0], ..Default::default() }; + let cb = with_camera_homography(wide, Some(&cam)); + let dv = (16.0 / 9.0) / (2100.0 / 900.0); + assert!(near(row(&cb, 0), [1.0, 0.0, 0.0]), "{:?}", cb.persp); + assert!(near(row(&cb, 1), [0.0, dv, (1.0 - dv) * 0.5]), "{:?}", cb.persp); + + // The box the template gives a PiP of this camera has its aspect: nothing is cropped. + let pip = LayerCB { quad_px: [422.4, 237.6], ..Default::default() }; + let cb = with_camera_homography(pip, Some(&cam)); + assert!(near(row(&cb, 0), [1.0, 0.0, 0.0]) && near(row(&cb, 1), [0.0, 1.0, 0.0])); + + // A real (projective) matrix is composed on the right: H * C, so H's third row picks + // up the window too. + let h = [0.8, 0.1, 0.05, -0.02, 0.9, 0.04, 0.1, -0.2, 1.0]; + let cam = crate::scene::SceneCamera { homography: Some(h), ..cam }; + let cb = with_camera_homography(narrow, Some(&cam)); + let u0 = (1.0 - du) * 0.5; + assert!(near(row(&cb, 2), [0.1 * du, -0.2, 0.1 * u0 + 1.0]), "{:?}", cb.persp); + assert_eq!(cb.layer_fx[1], 1.0); + } + + /// Without an aspect, or with a degenerate box, the whole corrected picture is shown. + #[test] + fn the_cover_window_falls_back_to_the_whole_picture() { + let whole = [0.0, 0.0, 1.0, 1.0]; + assert_eq!(corrected_cover_window(None, 1.5), whole); + assert_eq!(corrected_cover_window(Some(0.0), 1.5), whole); + assert_eq!(corrected_cover_window(Some(1.5), f32::NAN), whole); + assert_eq!(corrected_cover_window(Some(1.5), f32::INFINITY), whole); + assert_eq!(corrected_cover_window(Some(1.5), 0.0), whole); + assert_eq!(corrected_cover_window(Some(1.5), 1.5), whole); + } + + /// A homography with a NaN or an infinity in it counts as none. + #[test] + fn a_non_finite_homography_is_ignored() { + let mut h = [1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0]; + let ok = crate::scene::SceneCamera { homography: Some(h), ..extra_camera(1) }; + assert_eq!(camera_homography(Some(&ok)), Some(h)); + for bad in [f32::NAN, f32::INFINITY, f32::NEG_INFINITY] { + h[7] = bad; + let cam = crate::scene::SceneCamera { homography: Some(h), ..extra_camera(1) }; + assert_eq!(camera_homography(Some(&cam)), None); + let cb = with_camera_homography(LayerCB::default(), Some(&cam)); + assert_eq!(cb.layer_fx, [0.0; 4]); + assert_eq!(cb.persp, [[0.0; 4]; 3]); + } + } + + /// Camera 0's settings give back exactly today's source rect: layout crop, project mirror + /// and desk turn, through `camera_source_rect`. + #[test] + fn camera_0_settings_reproduce_todays_source_rect() { + let crop = SceneCrop { x: 0.1, y: 0.2, width: 0.6, height: 0.7 }; + let (visible, tex, box_ar) = ([1280.0, 720.0], [1280.0, 736.0], 1.3); + for (flip_u, flip_v, full_frame) in [ + (false, false, false), + (true, false, false), + (false, true, false), + (true, true, false), + (false, true, true), + ] { + let mut g = plan_frame(&golden_input(&golden_scene(), &crate::config::all()[0])); + g.webcam = WebcamOrientation { flip_u, flip_v, full_frame }; + let mut scene = golden_scene(); + scene.layout.webcam_crop = Some(crop); + let [cu0, cv0, cu1, cv1] = webcam_source_rect( + visible, + tex, + if full_frame { None } else { Some(crop) }, + box_ar, + ); + let (u0, u1) = if flip_u { (cu1, cu0) } else { (cu0, cu1) }; + let (v0, v1) = if flip_v { (cv1, cv0) } else { (cv0, cv1) }; + let cam0 = g.camera0_settings(Some(&scene)); + assert!(cam0.homography.is_none()); + assert_eq!(camera_source_rect(Some(&cam0), visible, tex, box_ar), [u0, v0, u1, v1]); + } + } + + /// Camera 0 with a perspective is cover-fitted into a box of another ratio, on both paths: + /// its default draw in a Full Camera section (no layout region, the box fills the frame) + /// and a planned frame-filling layer. Its PiP box, which has the corrected aspect, shows + /// the whole corrected picture. + #[test] + fn camera_0_with_a_perspective_is_cover_fitted_on_both_paths() { + let cfg = crate::config::all().pop().expect("cfg"); + let identity = "[1,0,0,0,1,0,0,0,1]"; + let aspect = 4.0_f32 / 3.0; + let cameras = format!(r#""cameras":[{{"index":0,"homography":{identity},"aspect":{aspect}}}]"#); + let zoom = r#""zoomRegions":[{"clipIndex":0,"startSec":0.0,"endSec":5.0,"scale":2.0,"focusX":0.5,"focusY":0.3,"rotation":"none"}]"#; + let near = |a: f32, b: f32| (a - b).abs() < 1e-4; + let render = [1170.0_f32, 658.0]; + let box_ar = render[0] / render[1]; + // A 4:3 picture in a wider box: the full width, a centred band of the box's ratio. + let dv = aspect / box_ar; + let check = |cb: &LayerCB| { + assert_eq!(cb.layer_fx[1], 1.0); + assert!(near(cb.persp[0][0], 1.0) && near(cb.persp[0][2], 0.0), "{:?}", cb.persp); + assert!(near(cb.persp[1][1], dv), "{:?}", cb.persp); + assert!(near(cb.persp[1][2], (1.0 - dv) * 0.5), "{:?}", cb.persp); + }; + + // No layout region, inside a Full Camera section: the default box fills the frame. + let json = zoomed_golden_scene_json().replace( + zoom, + &format!( + r#""zoomRegions":[],{cameras},"cameraFullscreenRegions":[{{"clipIndex":0,"startSec":0.0,"endSec":9.0}}]"# + ), + ); + let scene = Scene::from_json(&json).expect("scene"); + let g = plan_frame(&FrameGeometryInput { + timeline_t_override: Some(4.0), + ..golden_input(&scene, &cfg) + }); + assert!(g.camera_layers.is_empty()); + assert!(near(g.w_px[0], render[0]) && near(g.w_px[1], render[1]), "{:?}", g.w_px); + let cam0 = g.camera0_settings(Some(&scene)); + assert_eq!(cam0.aspect, Some(aspect)); + check(&with_camera_homography( + g.webcam_video_cb([0.0, 0.0, 1.0, 1.0], [1.0, 1.0], 0.0, 0.0), + Some(&cam0), + )); + + // A planned layer that fills the frame. + let json = zoomed_golden_scene_json().replace( + zoom, + &format!( + r#""zoomRegions":[],{cameras},"cameraLayoutRegions":[{{"clipIndex":0,"startSec":0.0,"endSec":4.0,"layers":[ + {{"camera":0,"rect":{{"x":0,"y":0,"width":1,"height":1}},"fillsFrame":true}}]}}]"# + ), + ); + let scene = Scene::from_json(&json).expect("scene"); + let g = plan_frame(&golden_input(&scene, &cfg)); + assert_eq!(g.camera_layers.len(), 1); + let cam0 = g.camera0_settings(Some(&scene)); + let base = g.webcam_video_cb([0.0; 4], [1.0, 1.0], 0.0, 0.0); + let plan = g.camera_layers[0]; + check(&camera_layer_cb(&plan, Some(&cam0), [1280.0, 720.0], [1280.0, 720.0], render, &base)); + + // The PiP box the app gives camera 0 has the corrected aspect: nothing is cropped. + let pip = LayerCB { quad_px: [400.0, 300.0], ..Default::default() }; + let cb = with_camera_homography(pip, Some(&cam0)); + assert!(near(cb.persp[0][0], 1.0) && near(cb.persp[1][1], 1.0), "{:?}", cb.persp); + } + + /// The shadow of a planned layer fades with it, and a frame-filling layer has none. + #[test] + fn a_planned_layers_shadow_fades_with_it() { + let cfg = crate::config::all().pop().expect("cfg"); + let g = plan_frame(&golden_input(&golden_scene(), &cfg)); + let render = [1920.0, 1080.0]; + let plan = planned(2, 0.4); + let base = extra_camera_base_cb([1.0; 2]); + let video = camera_layer_cb(&plan, None, [64.0; 2], [64.0; 2], render, &base); + let shadow = g.camera_layer_shadow(&plan, &video, render, true, false).expect("shadow"); + assert_eq!(shadow.mode, 2.0); + assert_eq!(shadow.layer_fx[0], video.layer_fx[0]); + assert_eq!(shadow.color[3], WEBCAM_SHADOW_OPACITY); + assert!(g.camera_layer_shadow(&plan, &video, render, false, false).is_none()); + let full = crate::camera_layers::CameraLayerPlan { fills_frame: true, ..plan }; + assert!(g.camera_layer_shadow(&full, &video, render, true, false).is_none()); + // Cutout removes camera 0's bubble, not the others'. + assert!(g.camera_layer_shadow(&plan, &video, render, true, true).is_some()); + let cam0 = crate::camera_layers::CameraLayerPlan { camera: 0, ..plan }; + assert!(g.camera_layer_shadow(&cam0, &video, render, true, true).is_none()); + } + + /// A turned Full Camera section from 1 s to 9 s: the plan carries the cover strength, full in + /// the hold after the start and zero in the steady part. + #[test] + fn the_frame_plan_carries_the_cover() { + let cfg = crate::config::all().pop().expect("cfg"); + let json = zoomed_golden_scene_json().replace( + r#""zoomRegions":[{"clipIndex":0,"startSec":0.0,"endSec":5.0,"scale":2.0,"focusX":0.5,"focusY":0.3,"rotation":"none"}]"#, + r#""zoomRegions":[],"cameraFullscreenRegions":[{"clipIndex":0,"startSec":1.0,"endSec":9.0,"rotation":180,"fullFrame":true}]"#, + ); + let scene = Scene::from_json(&json).expect("scene"); + let cover_at = |t: f32| { + plan_frame(&FrameGeometryInput { timeline_t_override: Some(t), ..golden_input(&scene, &cfg) }) + .webcam_cover + }; + assert_eq!(cover_at(1.5), 1.0); + assert_eq!(cover_at(5.0), 0.0); + } + + /// A scene without layout regions plans no camera layers, and camera 0 lands exactly where + /// it did: the new (empty) keys change nothing. + #[test] + fn without_layout_regions_the_plan_has_no_camera_layers() { + let cfg = crate::config::all().pop().expect("cfg"); + let before = plan_frame(&golden_input(&golden_scene(), &cfg)); + let json = zoomed_golden_scene_json().replace( + r#""zoomRegions":[{"clipIndex":0,"startSec":0.0,"endSec":5.0,"scale":2.0,"focusX":0.5,"focusY":0.3,"rotation":"none"}]"#, + r#""zoomRegions":[],"cameras":[],"cameraLayoutRegions":[]"#, + ); + let scene = Scene::from_json(&json).expect("scene"); + let after = plan_frame(&golden_input(&scene, &cfg)); + assert!(before.camera_layers.is_empty()); + assert!(after.camera_layers.is_empty()); + assert_eq!(after.w_dst, before.w_dst); + assert_eq!(after.w_dst_prev, before.w_dst_prev); + assert_eq!(after.w_px, before.w_px); + assert_eq!(after.w_radius, before.w_radius); + assert_eq!(after.shape_fade, before.shape_fade); + assert_eq!(after.webcam, before.webcam); + assert_eq!(after.webcam_cover, before.webcam_cover); + } + + /// Inside a layout region camera 0 is one of the planned layers, at the region's rect. + #[test] + fn with_a_layout_region_camera_0_comes_from_the_plan() { + let cfg = crate::config::all().pop().expect("cfg"); + let json = zoomed_golden_scene_json().replace( + r#""zoomRegions":[{"clipIndex":0,"startSec":0.0,"endSec":5.0,"scale":2.0,"focusX":0.5,"focusY":0.3,"rotation":"none"}]"#, + r#""zoomRegions":[],"cameraLayoutRegions":[{"clipIndex":0,"startSec":0.0,"endSec":4.0,"layers":[ + {"camera":0,"rect":{"x":0.1,"y":0.2,"width":0.3,"height":0.4},"radiusFrac":0.1,"shape":"rectangle"}]}]"#, + ); + let scene = Scene::from_json(&json).expect("scene"); + // `golden_input` samples t = 1.5 s: past the lead-in, well before the lead-out. + let g = plan_frame(&golden_input(&scene, &cfg)); + assert_eq!(g.camera_layers.len(), 1); + let cam0 = g.camera_layers[0]; + assert_eq!(cam0.camera, 0); + assert_eq!(cam0.dst, [0.1, 0.2, 0.3, 0.4]); + assert_eq!(cam0.opacity, 1.0); + assert_eq!(cam0.shape, webcam_shape_code("rectangle")); + assert!(!cam0.fills_frame); + // Outside the region camera 0 is the default PiP, where `w_dst` puts it. + let outside = plan_frame(&FrameGeometryInput { + timeline_t_override: Some(6.0), + ..golden_input(&scene, &cfg) + }); + assert_eq!(outside.camera_layers.len(), 1); + assert_eq!(outside.camera_layers[0].dst, outside.w_dst); + let r = outside.w_radius / outside.w_px[0].min(outside.w_px[1]).max(1.0); + assert_eq!(outside.camera_layers[0].radius_frac, r); + } + + /// The desk label's opacity at the compositor caller boundary, against the cover of the same + /// frame plan: equal at every sampled frame, and the label (spanned over its whole section by + /// the app) is on screen wherever the cover is not 0. Three sections: the review's 2.6 s one + /// at 1×, where the label's old fade (from its own 1.15 s window) was half gone while the + /// cover still held 1; a long one; and a section inside a 2× speed region, whose cover runs + /// on the screen clock. + #[test] + fn the_desk_label_fades_with_the_cover() { + use crate::text_anim::{annotation_text_state, DESK_COVER_ANIMATION}; + let cfg = crate::config::all().pop().expect("cfg"); + let zoom = r#""zoomRegions":[{"clipIndex":0,"startSec":0.0,"endSec":5.0,"scale":2.0,"focusX":0.5,"focusY":0.3,"rotation":"none"}]"#; + let section = |start: f32, end: f32, speed: Option<&str>| { + let regions = format!( + r#""zoomRegions":[],"cameraFullscreenRegions":[{{"clipIndex":0,"startSec":{start},"endSec":{end},"rotation":180,"fullFrame":true}}]{}"#, + speed.map(|s| format!(",{s}")).unwrap_or_default() + ); + Scene::from_json(&zoomed_golden_scene_json().replace(zoom, ®ions)).expect("scene") + }; + // (label, cover) at source time `t` of a label spanning [start, end]. + let sample = |scene: &Scene, start: f32, end: f32, t: f32| { + let g = plan_frame(&FrameGeometryInput { + timeline_t_override: Some(t), + ..golden_input(scene, &cfg) + }); + let label = annotation_text_state( + Some(DESK_COVER_ANIMATION), + (t - start) * 1000.0, + (end - start) * 1000.0, + g.webcam_cover, + ); + (label.opacity, g.webcam_cover) + }; + let speed = r#""speedRegions":[{"clipIndex":0,"startSec":1.0,"endSec":9.0,"speed":2.0}]"#; + let cases = [ + ("2.6 s at 1x", section(1.0, 3.6, None), 1.0f32, 3.6f32), + ("long at 1x", section(1.0, 9.0, None), 1.0, 9.0), + ("2.6 s on screen inside 2x", section(2.0, 7.2, Some(speed)), 2.0, 7.2), + ]; + for (name, scene, start, end) in &cases { + let mut t = start - 0.1; + while t < end + 0.1 { + let (label, cover) = sample(scene, *start, *end, t); + assert_eq!(label, cover, "{name}: t = {t}"); + if cover > 0.0 { + assert!(t >= *start && t < *end, "{name}: cover {cover} outside the label at {t}"); + } + t += 1.0 / 120.0; + } + } + // The review's frames: the cover still holds 1 at 0.9 s and 1.0 s (the old label was at + // ~0.5 and below). The 0.28 s left between the holds is two 0.14 s fades meeting at + // +1.1575 s, where label and cover touch 0 together before rising into the end hold. + let (_, short, s0, s1) = &cases[0]; + for at in [0.5, 0.9, 1.0] { + assert_eq!(sample(short, *s0, *s1, s0 + at), (1.0, 1.0), "at +{at} s"); + } + let (mid, _) = sample(short, *s0, *s1, s0 + 1.09); + assert!(mid > 0.0 && mid < 1.0, "mid-fade {mid}"); + let (low, low_cover) = sample(short, *s0, *s1, s0 + 1.1575); + assert!(low < 1e-3 && low == low_cover, "junction {low} vs {low_cover}"); + assert_eq!(sample(short, *s0, *s1, s1 - 1.25), (1.0, 1.0), "end hold"); + // Inside 2x the same 2.6 s of screen time takes 5.2 s of source: every screen instant + // matches the 1x section's, which the source-time label could not do. + let (_, fast, f0, f1) = &cases[2]; + for k in 0..=26 { + let screen = k as f32 * 0.1; + let slow = sample(short, *s0, *s1, s0 + screen); + let quick = sample(fast, *f0, *f1, f0 + 2.0 * screen); + assert!((slow.0 - quick.0).abs() < 1e-3, "screen +{screen} s: {slow:?} vs {quick:?}"); + } + } + + /// A turned Full Camera section that meets a layout region: the cover follows the seam rule + /// (no hold before the layout region, only its own fade), and the label, spanned over the + /// whole section by the app, follows the cover there frame for frame without any seam + /// logic of its own. + #[test] + fn the_desk_label_follows_the_cover_across_a_seam_with_a_layout_region() { + use crate::regions::DESK_COVER_FADE_S; + use crate::text_anim::{annotation_text_state, DESK_COVER_ANIMATION}; + let cfg = crate::config::all().pop().expect("cfg"); + let zoom = r#""zoomRegions":[{"clipIndex":0,"startSec":0.0,"endSec":5.0,"scale":2.0,"focusX":0.5,"focusY":0.3,"rotation":"none"}]"#; + let full = r#""zoomRegions":[],"cameraFullscreenRegions":[{"clipIndex":0,"startSec":1.0,"endSec":6.0,"rotation":180,"fullFrame":true}]"#; + let layout = r#","cameraLayoutRegions":[{"clipIndex":0,"startSec":6.0,"endSec":9.0,"layers":[ + {"camera":0,"rect":{"x":0.1,"y":0.2,"width":0.3,"height":0.4},"radiusFrac":0.1,"shape":"rectangle"}]}]"#; + let json = zoomed_golden_scene_json(); + let seam = + Scene::from_json(&json.replace(zoom, &format!("{full}{layout}"))).expect("scene"); + let lone = Scene::from_json(&json.replace(zoom, full)).expect("scene"); + let (start, end) = (1.0f32, 6.0f32); + let sample = |scene: &Scene, t: f32| { + let g = plan_frame(&FrameGeometryInput { + timeline_t_override: Some(t), + ..golden_input(scene, &cfg) + }); + let label = annotation_text_state( + Some(DESK_COVER_ANIMATION), + (t - start) * 1000.0, + (end - start) * 1000.0, + g.webcam_cover, + ); + (label.opacity, g.webcam_cover) + }; + let mut t = start - 0.1; + while t < 9.1 { + let (label, cover) = sample(&seam, t); + assert_eq!(label, cover, "t = {t}"); + if cover > 0.0 { + assert!(t >= start && t < end, "cover {cover} outside the label at {t}"); + } + t += 1.0 / 120.0; + } + // Before the seam the cover (and so the label) is only its fade: half way up at + // end - fade/2 and sharp just before the fade, where a lone section still holds 1. + let (mid, _) = sample(&seam, end - DESK_COVER_FADE_S / 2.0); + assert!((mid - 0.5).abs() < 1e-3, "half way up at the seam: {mid}"); + let before_fade = end - DESK_COVER_FADE_S - 0.05; + assert_eq!(sample(&seam, before_fade), (0.0, 0.0), "sharp before the seam's fade"); + assert_eq!(sample(&lone, before_fade), (1.0, 1.0), "a lone section still holds"); + assert!(sample(&seam, end - 0.001).0 > 0.99, "covered at the seam"); + } + /// Le pivot doit rester collé à `center` quand le sprite grandit — c'est exactement ce qui /// était cassé (ancrage centré en dur : la pointe s'éloignait proportionnellement à la /// taille). On dessine la même flèche à deux tailles et on vérifie que le point désigné @@ -7444,6 +8573,9 @@ mod tests { w_px: [0.0, 0.0], w_radius: 0.0, shape_fade: 0.0, + webcam: WebcamOrientation::default(), + webcam_cover: 0.0, + camera_layers: Vec::new(), window_frame: None, screen_mask: None, } @@ -8508,4 +9640,117 @@ mod tests { println!("left ×2 : {:.3} côté proche, {:.3} côté lointain", near / at_focus, far / at_focus); assert!(near > 1.04 * at_focus && far < 0.96 * at_focus, "{near} {at_focus} {far}"); } + + use crate::scene::SceneCameraFullscreenRegion; + + fn cam_region(rotation: u16, mirror: Option) -> SceneCameraFullscreenRegion { + SceneCameraFullscreenRegion { + clip_index: None, + start_sec: 0.0, + end_sec: 1.0, + rotation, + mirror, + full_frame: rotation == 180, + } + } + + #[test] + fn without_a_region_the_webcam_is_laid_as_before() { + for m in [false, true] { + assert_eq!( + webcam_orientation(None, m), + WebcamOrientation { flip_u: m, flip_v: false, full_frame: false } + ); + } + } + + /// 180° is both axes swapped; a mirror on top cancels the horizontal one. + #[test] + fn a_turned_region_swaps_both_axes_and_a_mirror_cancels_one() { + let turned = cam_region(180, Some(false)); + assert_eq!( + webcam_orientation(Some(&turned), true), + WebcamOrientation { flip_u: true, flip_v: true, full_frame: true } + ); + let turned_mirrored = cam_region(180, Some(true)); + assert_eq!( + webcam_orientation(Some(&turned_mirrored), false), + WebcamOrientation { flip_u: false, flip_v: true, full_frame: true } + ); + } + + #[test] + fn a_plain_region_keeps_the_layout_mirror_and_an_unknown_rotation_is_none() { + assert!(webcam_orientation(Some(&cam_region(0, None)), true).flip_u); + let odd = cam_region(90, None); + assert_eq!( + webcam_orientation(Some(&odd), false), + WebcamOrientation { flip_u: false, flip_v: false, full_frame: false } + ); + } + + /// `layer.wgsl` is only compiled on Linux, in CI, where a name or syntax error breaks every + /// draw. Parse and validate it on every host through wgpu's own naga, and pin the desk-view + /// cover: the fragment entry point must call the radius-taking blur kernel, and the kernel + /// must keep its taps inside the picture's valid area. + /// + /// The file does not declare `LAYER_MODELS`: `compositor_linux::layer_source` prefixes it, + /// once per pipeline. Both variants are checked here the same way, the one without the 3D + /// models first, since that is the pipeline the webcam (mode 0) is drawn with. + #[test] + fn layer_wgsl_validates_and_its_fragment_applies_the_desk_view_cover() { + use wgpu::naga; + fn calls(block: &naga::Block, target: naga::Handle) -> usize { + block + .iter() + .map(|st| match st { + naga::Statement::Call { function, .. } => usize::from(*function == target), + naga::Statement::Block(b) => calls(b, target), + naga::Statement::If { accept, reject, .. } => calls(accept, target) + calls(reject, target), + naga::Statement::Loop { body, continuing, .. } => calls(body, target) + calls(continuing, target), + naga::Statement::Switch { cases, .. } => cases.iter().map(|c| calls(&c.body, target)).sum(), + _ => 0, + }) + .sum() + } + + for models in [false, true] { + // The same prefix as `compositor_linux::layer_source`, which only builds on Linux. + let source = + format!("const LAYER_MODELS: bool = {models};\n{}", include_str!("vk_shaders/layer.wgsl")); + let module = naga::front::wgsl::parse_str(&source) + .unwrap_or_else(|e| panic!("layer.wgsl (LAYER_MODELS = {models}) parses: {e:?}")); + naga::valid::Validator::new(naga::valid::ValidationFlags::all(), naga::valid::Capabilities::all()) + .validate(&module) + .unwrap_or_else(|e| panic!("layer.wgsl (LAYER_MODELS = {models}) validates: {e:?}")); + let (kernel, f) = module + .functions + .iter() + .find(|(_, f)| f.name.as_deref() == Some("blur_webcam_radius")) + .expect("blur_webcam_radius exists"); + assert_eq!(f.arguments.len(), 5, "uv, max_r_px, qpx, local_px, valid"); + // The taps are clamped to the valid part of an aligned decoder texture, half a texel + // in: the kernel reads the texture size to know what half a texel is. + assert!( + f.expressions.iter().any(|(_, e)| matches!( + e, + naga::Expression::ImageQuery { query: naga::ImageQuery::Size { .. }, .. } + )), + "blur_webcam_radius reads the texture size for its half-texel clamp" + ); + // `fs_main` only applies the layer's transparency to `fs_layer`, which draws it. + let fs = module.entry_points.iter().find(|e| e.name == "fs_main").expect("fs_main"); + let (body, f) = module + .functions + .iter() + .find(|(_, f)| f.name.as_deref() == Some("fs_layer")) + .expect("fs_layer exists"); + assert_eq!(calls(&fs.function.body, body), 1, "fs_main draws through fs_layer"); + assert_eq!( + calls(&f.body, kernel), + 1, + "fs_layer blurs the covered camera once (LAYER_MODELS = {models})" + ); + } + } } diff --git a/crates/compositor/src/gif_export.rs b/crates/compositor/src/gif_export.rs index 8215704c29..7ab0c1b6c7 100644 --- a/crates/compositor/src/gif_export.rs +++ b/crates/compositor/src/gif_export.rs @@ -256,6 +256,7 @@ fn export_gif_inner( let mut screen_decs: HashMap = HashMap::new(); let mut webcam_decs: HashMap = HashMap::new(); + let mut extra_decs: HashMap = HashMap::new(); screen_decs.insert(clips[0].screen.clone(), unsafe { Decoder::open_for_export(&clips[0].screen, gpu)? }); @@ -270,6 +271,7 @@ fn export_gif_inner( &scene, &mut screen_decs, &mut webcam_decs, + &mut extra_decs, &mut |frame_index| { control.check()?; // CPU readback of the staged RT (RGBA8 tightly-packed, diff --git a/crates/compositor/src/lib.rs b/crates/compositor/src/lib.rs index e26480cc63..67c233e4b2 100644 --- a/crates/compositor/src/lib.rs +++ b/crates/compositor/src/lib.rs @@ -30,6 +30,7 @@ pub mod audio; pub mod audio_jobs; pub mod camera; +pub mod camera_layers; pub mod config; pub mod cursor; pub mod cursor_sdf; @@ -43,6 +44,7 @@ pub mod regions; // cibles) et le shim C. Seul Linux l'appelle aujourd'hui, parce que c'est la // seule plateforme dont la capture passe par `MediaRecorder`, mais rien dedans // n'est spécifique à Linux. +pub mod prism_mesh; pub mod remux; pub mod scene; pub mod sculpt; @@ -50,9 +52,12 @@ pub mod sculpt; // `segmentation` ses deux entrées échouent proprement, ce qui garde le reste du crate // indépendant du choix de packaging d'ONNX Runtime. pub mod segmentation; +// Livraison de la preview par textures partagées (Windows) ; la tenue des cases est portable. +pub mod shared_frames; pub mod text_anim; pub mod text_fonts; pub mod text_plate; +pub(crate) mod extra_cameras; pub(crate) mod timeline_walk; // GPU backend : Windows → d3d_windows, macOS → d3d_macos. Ré-exporté sous le nom `d3d` diff --git a/crates/compositor/src/live.rs b/crates/compositor/src/live.rs index ebcb463887..d8e2c4e87c 100644 --- a/crates/compositor/src/live.rs +++ b/crates/compositor/src/live.rs @@ -23,13 +23,21 @@ //! le napi — le `gen` est l'identité de la frame (cf. `LatestFrame`). use crate::compositor::{Compositor, LiveParams}; +use crate::ffi::AVFrame; use crate::regions::{speed_at, ProgrammeClock}; -use crate::scene::Scene; +use crate::scene::{Scene, SceneCameraLayoutRegion, SceneClipCamera}; use crate::config::{self, Cfg}; use crate::cursor::CursorTrack; -use crate::d3d::Gpu; +use crate::d3d::{Backend, Gpu}; use crate::frame_geometry::webcam_is_real; use crate::pipeline::Decoder; +#[cfg(windows)] +use crate::shared_frames::SharedRing; +use crate::shared_frames::{SharedFrame, SlotBook}; +use crate::extra_cameras::{ + camera_source_time, cameras_in_regions, extra_camera_active, extra_camera_keys, extra_frame_list, + has_camera_file, opened_or_skipped, seek_extra_camera, step_extra_camera, ExtraCameraKeys, +}; use crate::timeline_walk::{frame_step, FrameStep, NextFrameTime}; use anyhow::Result; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; @@ -49,10 +57,6 @@ fn parse_hex_color(s: &str) -> Option<[f32; 4]> { Some([r, g, b, 1.0]) } -fn webcam_seek_time(screen_source_time_sec: f64, webcam_offset_sec: f64) -> f64 { - (screen_source_time_sec - webcam_offset_sec).max(0.0) -} - /// Décodeurs déjà ouverts ET positionnés au bon playhead pour un clip à venir — le résultat /// d'un préchargement en tâche de fond (voir `open_and_seek_clip`/`maybe_start_prefetch` /// dans `render_thread`). Appliquer ceci à un `Player` (`apply_prefetched`) ne fait plus @@ -69,6 +73,9 @@ struct PrefetchedClip { /// des paires ouvertes plusieurs bascules plus tôt. Cf. `open_webcam_or_stand_in`. webcam_decoder_is_real: bool, webcam_offset_sec: f64, + /// The clip's extra cameras (index k-1 = camera k) and what they were opened for. + extra: Vec>, + extra_keys: ExtraCameraKeys, idx: u32, /// Piste curseur du clip à venir, préchargée ici pour la même raison que les décodeurs : /// sans ça, la bascule à la frontière restait synchrone sur CE point précis (lecture + @@ -106,9 +113,13 @@ struct PrefetchedClip { /// `should_draw_webcam`). /// /// ponytail: on garde le remplaçant plutôt que de passer `wdec` en `Option`, ce qui -/// toucherait 22 sites dont le pool de décodeurs et la boucle de composition `unsafe`. À faire -/// si quelqu'un mesure que le décodeur inutile coûte (VRAM des pools D3D11VA, une ouverture -/// par clip) — l'avertissement ci-dessous dit enfin à quelle fréquence le cas visible arrive. +/// toucherait 22 sites dont le pool de décodeurs et la boucle de composition `unsafe`. Il est +/// ouvert mais n'est plus jamais avancé ni recherché : la composition reçoit la frame écran à +/// sa place (`Player::webcam_frame`), l'image même qu'il aurait décodée. Le décoder à côté de +/// l'écran coûtait la moitié du débit de décodage, sans caméra, c'est-à-dire le cas courant : +/// une source 4K lue à 2× plafonnait à ~46 frames/s pour 97 que le décodeur seul tient, et la +/// preview prenait des secondes de retard dans les régions de vitesse. Reste à faire si la +/// VRAM des pools D3D11VA ou l'ouverture par clip se mesurent. unsafe fn open_webcam_or_stand_in( screen_path: &str, webcam_path: &str, @@ -142,27 +153,139 @@ fn should_draw_webcam(webcam_path: &str, screen_path: &str, decoder_is_real: boo webcam_is_real(webcam_path, screen_path) && decoder_is_real } +/// An extra camera (k >= 1) of the active clip, decoded only near the layout regions that show +/// it (`extra_camera_active`). +struct ExtraCamera { + dec: Decoder, + offset_sec: f64, + /// The last step or seek left a frame to draw. Read by `Player::recompose`. + shown: bool, + /// Camera source time past which the file has no frame left (`step_extra_camera`). + ended_at: Option, +} + +impl ExtraCamera { + /// `step_extra_camera` toward `target` (camera source time). + unsafe fn step_to(&mut self, target: f64) -> Result<*const AVFrame> { + let frame = step_extra_camera(&mut self.dec, &mut self.ended_at, target)?; + self.shown = !frame.is_null(); + Ok(frame) + } + + /// `seek_extra_camera` to `target` (camera source time). + unsafe fn seek(&mut self, target: f64) -> Result<*const AVFrame> { + let frame = seek_extra_camera(&mut self.dec, &mut self.ended_at, target)?; + self.shown = !frame.is_null(); + Ok(frame) + } +} + +/// The extra cameras clip `clip_index` opens: those its own layout regions +/// (`Scene::for_clip_window`) draw and that the clip has a file for. +fn extra_cameras_for_clip(scene: &Scene, clip_index: usize) -> Vec { + let Some(clip) = scene.clips.get(clip_index) else { + return Vec::new(); + }; + // The common case — no layout region anywhere — skips the windowing copy of the scene. + if scene.camera_layout_regions.is_empty() || clip.additional_cameras.is_empty() { + return Vec::new(); + } + cameras_in_regions(&scene_for_clip(scene, clip_index).camera_layout_regions) + .into_iter() + .filter(|&k| has_camera_file(clip.additional_cameras.get(k - 1))) + .collect() +} + +/// `extra_camera_keys` for clip `clip_index` of `scene`, from the clip's own camera files. +fn scene_extra_camera_keys(scene: &Scene, clip_index: usize) -> ExtraCameraKeys { + let sources = scene.clips.get(clip_index).map(|c| c.additional_cameras.as_slice()).unwrap_or(&[]); + extra_camera_keys(&extra_cameras_for_clip(scene, clip_index), sources) +} + +/// The windowed layout regions of clip `clip_index` (what `camera_layers_at` plans from). +fn clip_layout_regions(scene: &Scene, clip_index: usize) -> Vec { + if scene.camera_layout_regions.is_empty() { + return Vec::new(); + } + scene_for_clip(scene, clip_index).camera_layout_regions +} + +fn same_extra_key(a: &Option, b: &Option) -> bool { + match (a, b) { + (None, None) => true, + (Some(a), Some(b)) => a.path == b.path && (a.offset_sec - b.offset_sec).abs() < 1e-9, + _ => false, + } +} + +fn same_extra_keys(a: &[Option], b: &[Option]) -> bool { + a.len() == b.len() && a.iter().zip(b).all(|(a, b)| same_extra_key(a, b)) +} + +/// Same spans and same cameras in each: what decides when an extra camera is active +/// (`extra_camera_active`). Rects and shapes do not matter to the decoders. +fn same_region_timing(a: &[SceneCameraLayoutRegion], b: &[SceneCameraLayoutRegion]) -> bool { + a.len() == b.len() + && a.iter().zip(b).all(|(a, b)| { + a.clip_index == b.clip_index + && a.start_sec == b.start_sec + && a.end_sec == b.end_sec + && a.layers.len() == b.layers.len() + && a.layers.iter().zip(&b.layers).all(|(x, y)| x.camera == y.camera) + }) +} + +/// Whether `set_extra_cameras` seeks the extra cameras right away. Playing on (`steady`: +/// playing, and the screen position did not jump) with the same files and the same region +/// timing, the next playback step (`advance_extras(.., false)`) moves them on: seeking every +/// camera on each scene push would stall the preview while it plays. +fn extras_need_seek(keys_changed: bool, timing_changed: bool, steady: bool) -> bool { + keys_changed || timing_changed || !steady +} + +/// Opens the extra cameras `keys` asks for, without seeking them: they are positioned when +/// they become active (`Player::advance_extras`). +unsafe fn open_extra_cameras(keys: &[Option], gpu: &Gpu) -> Vec> { + keys.iter() + .map(|key| { + let key = key.as_ref()?; + let dec = opened_or_skipped(&key.path, Decoder::open(&key.path, gpu))?; + Some(ExtraCamera { dec, offset_sec: key.offset_sec, shown: false, ended_at: None }) + }) + .collect() +} + unsafe fn open_and_seek_clip( screen_path: &str, webcam_path: &str, webcam_offset_sec: f64, + extra_keys: &[Option], source_time_sec: f64, gpu: &Gpu, ) -> Result { let source_time_sec = source_time_sec.max(0.0); let mut sdec = Decoder::open(screen_path, gpu)?; let (mut wdec, webcam_decoder_is_real) = open_webcam_or_stand_in(screen_path, webcam_path, gpu)?; - let sf = sdec.seek_to(source_time_sec)?; - let mut wf = wdec.seek_to(webcam_seek_time(source_time_sec, webcam_offset_sec))?; - if wf.is_null() { - wf = wdec.seek_to(0.0)?; + let extra = open_extra_cameras(extra_keys, gpu); + let sf = sdec.seek_to_or_last(source_time_sec)?; + if webcam_decoder_is_real && wdec.seek_to(camera_source_time(source_time_sec, webcam_offset_sec))?.is_null() { + wdec.seek_to(0.0)?; } if sf.is_null() { anyhow::bail!("clip préchargé vide au temps source {source_time_sec:.3}s (screen=\"{screen_path}\")"); } let idx = (source_time_sec * sdec.fps()).round().max(0.0) as u32; let cursor_track = CursorTrack::load(&format!("{screen_path}.cursor.json"), 0.0, 24.0 * 3600.0).ok(); - Ok(PrefetchedClip { sdec, wdec, webcam_decoder_is_real, webcam_offset_sec, idx, cursor_track }) + Ok(PrefetchedClip { + sdec, + wdec, + webcam_decoder_is_real, + webcam_offset_sec, + extra, + extra_keys: extra_keys.to_vec(), + idx, + cursor_track, + }) } /// Nombre de paires de décodeurs INACTIVES gardées ouvertes en plus de la paire active. @@ -174,11 +297,29 @@ unsafe fn open_and_seek_clip( /// les timelines 2-4 clips, à baisser si la VRAM serre. const DECODER_POOL_CAP: usize = 3; -/// Une paire de décodeurs mise de côté, prête à être réactivée sans réouverture. -struct PooledClip { +/// What a set of open decoders was opened for: the screen, camera 0 and the extra cameras. +/// A pooled set is reused only for the same key, extra cameras included — a pooled pair +/// opened without camera 2 must not stand in for a clip whose layout shows it. +#[derive(Clone)] +struct ClipKey { screen_path: String, webcam_path: String, webcam_offset_sec: f64, + extras: ExtraCameraKeys, +} + +impl ClipKey { + fn matches(&self, other: &ClipKey) -> bool { + self.screen_path == other.screen_path + && self.webcam_path == other.webcam_path + && (self.webcam_offset_sec - other.webcam_offset_sec).abs() < 1e-9 + && same_extra_keys(&self.extras, &other.extras) + } +} + +/// Une paire de décodeurs mise de côté, prête à être réactivée sans réouverture. +struct PooledClip { + key: ClipKey, clip: PrefetchedClip, } @@ -189,14 +330,19 @@ struct PooledClip { unsafe fn seek_pair( sdec: &mut Decoder, wdec: &mut Decoder, + webcam_decoder_is_real: bool, source_time_sec: f64, webcam_offset_sec: f64, ) -> Result { - let sf = sdec.seek_to(source_time_sec)?; + let sf = sdec.seek_to_or_last(source_time_sec)?; if sf.is_null() { return Ok(false); } - let mut wf = wdec.seek_to(webcam_seek_time(source_time_sec, webcam_offset_sec))?; + // Le remplaçant n'est jamais lu (`Player::webcam_frame`) : rien à positionner. + if !webcam_decoder_is_real { + return Ok(true); + } + let mut wf = wdec.seek_to(camera_source_time(source_time_sec, webcam_offset_sec))?; if wf.is_null() { wf = wdec.seek_to(0.0)?; } @@ -215,50 +361,41 @@ unsafe fn seek_pair( unsafe fn swap_clip_pooled( player: &mut Player, pool: &mut Vec, - request: &ActiveClipRequest, - active_screen: &str, - active_webcam: &str, - active_webcam_offset_sec: f64, + request: &ClipKey, + source_time_sec: f64, + active: &ClipKey, ) -> Result<()> { let timing = std::env::var("OPENSCREEN_CLIPSWITCH_TIMING").is_ok(); let t0 = std::time::Instant::now(); - let t = request.source_time_sec.max(0.0); - let matches = |p: &PooledClip| { - p.screen_path == request.screen_path - && p.webcam_path == request.webcam_path - && (p.webcam_offset_sec - request.webcam_offset_sec).abs() < 1e-9 - }; + let t = source_time_sec.max(0.0); let mut hit = false; - let incoming: PrefetchedClip = match pool.iter().position(&matches) { + let incoming: PrefetchedClip = match pool.iter().position(|p| p.key.matches(request)) { Some(i) => { let mut pooled = pool.remove(i); // Reseek les décodeurs poolés AVANT de les installer. Échec → on les jette et on // ouvre à neuf (chemin connu sûr), jamais une frame vide. - if seek_pair(&mut pooled.clip.sdec, &mut pooled.clip.wdec, t, request.webcam_offset_sec)? { + if seek_pair( + &mut pooled.clip.sdec, + &mut pooled.clip.wdec, + pooled.clip.webcam_decoder_is_real, + t, + request.webcam_offset_sec, + )? { pooled.clip.idx = (t * pooled.clip.sdec.fps()).round().max(0.0) as u32; hit = true; pooled.clip } else { drop(pooled); - player.open_clip(&request.screen_path, &request.webcam_path, request.webcam_offset_sec, t)? + player.open_clip(request, t)? } } - None => player.open_clip(&request.screen_path, &request.webcam_path, request.webcam_offset_sec, t)?, + None => player.open_clip(request, t)?, }; let outgoing = player.swap_active(incoming); // Met la paire quittée en pool : dédup par clé (jamais deux entrées d'un même média), puis // éviction LRU (le plus ancien, en tête, part en premier). - pool.retain(|p| { - !(p.screen_path == active_screen - && p.webcam_path == active_webcam - && (p.webcam_offset_sec - active_webcam_offset_sec).abs() < 1e-9) - }); - pool.push(PooledClip { - screen_path: active_screen.to_string(), - webcam_path: active_webcam.to_string(), - webcam_offset_sec: active_webcam_offset_sec, - clip: outgoing, - }); + pool.retain(|p| !p.key.matches(active)); + pool.push(PooledClip { key: active.clone(), clip: outgoing }); while pool.len() > DECODER_POOL_CAP { pool.remove(0); } @@ -285,6 +422,12 @@ pub struct Player { /// (`swap_active`), lu par la boucle de rendu pour décider de dessiner la vignette. webcam_decoder_is_real: bool, webcam_offset_sec: f64, + /// The active clip's extra cameras (index k-1 = camera k), what they were opened for, and + /// the clip's windowed layout regions that decide when each one is decoded. All empty + /// without layout regions: no decoder, nothing to advance. + extra: Vec>, + extra_keys: ExtraCameraKeys, + camera_regions: Vec, has_current_frame: bool, use_current_on_next_step: bool, idx: u32, @@ -307,6 +450,9 @@ impl Player { }, webcam_decoder_is_real, webcam_offset_sec: 0.0, + extra: Vec::new(), + extra_keys: Vec::new(), + camera_regions: Vec::new(), has_current_frame: false, use_current_on_next_step: false, idx: 0, @@ -325,10 +471,17 @@ impl Player { screen_path: &str, webcam_path: &str, webcam_offset_sec: f64, + extra_keys: &[Option], source_time_sec: f64, ) -> Result<()> { - let prefetched = - open_and_seek_clip(screen_path, webcam_path, webcam_offset_sec, source_time_sec, &self.gpu)?; + let prefetched = open_and_seek_clip( + screen_path, + webcam_path, + webcam_offset_sec, + extra_keys, + source_time_sec, + &self.gpu, + )?; self.apply_prefetched(prefetched); Ok(()) } @@ -359,7 +512,13 @@ impl Player { // Les DEUX flux doivent avoir une frame : `compose_frame` les échantillonne tous les // deux sans condition, un seul manquant suffit à le faire échouer (d'où le `false` que // `seek_pair` peut rendre → l'appelant retombe sur l'ouverture complète). - if !seek_pair(&mut self.sdec, &mut self.wdec, source_time_sec, self.webcam_offset_sec)? { + if !seek_pair( + &mut self.sdec, + &mut self.wdec, + self.webcam_decoder_is_real, + source_time_sec, + self.webcam_offset_sec, + )? { return Ok(false); } self.idx = (source_time_sec * self.sdec.fps()).round().max(0.0) as u32; @@ -389,6 +548,8 @@ impl Player { // recalculer), l'entrante impose la sienne au player. webcam_decoder_is_real: self.webcam_decoder_is_real, webcam_offset_sec: self.webcam_offset_sec, + extra: std::mem::replace(&mut self.extra, incoming.extra), + extra_keys: std::mem::replace(&mut self.extra_keys, incoming.extra_keys), idx: self.idx, // Le curseur est re-dérivé du chemin à la réactivation ; inutile de le trimballer. cursor_track: None, @@ -403,14 +564,107 @@ impl Player { /// Ouvre une nouvelle paire de décodeurs positionnée à `source_time_sec`, SANS l'installer /// (l'appelant l'échange via `swap_active`). Réutilise le device D3D11 du player. - unsafe fn open_clip( - &self, - screen: &str, - webcam: &str, - webcam_offset_sec: f64, - source_time_sec: f64, - ) -> Result { - open_and_seek_clip(screen, webcam, webcam_offset_sec, source_time_sec, &self.gpu) + unsafe fn open_clip(&self, key: &ClipKey, source_time_sec: f64) -> Result { + open_and_seek_clip( + &key.screen_path, + &key.webcam_path, + key.webcam_offset_sec, + &key.extras, + source_time_sec, + &self.gpu, + ) + } + + /// What the active extra cameras were opened for (the extra half of the pool key). + fn extra_keys(&self) -> &[Option] { + &self.extra_keys + } + + /// Installs the active clip's windowed layout regions and the extra cameras they need + /// (`keys`): slots whose key is unchanged keep their decoder, the others are closed and + /// (re)opened — a file that will not open leaves its slot empty, once. Then positions every + /// camera that is near one of its regions at the current screen time, so a paused + /// recompose shows it at once — unless nothing that matters to the decoders changed while + /// playback goes on (`steady`, see `extras_need_seek`). `true` when a decoder was closed: + /// the caller must then clear the compositor's SRV cache (which also forgets the extra + /// frame pointers). + pub(crate) unsafe fn set_extra_cameras( + &mut self, + regions: Vec, + keys: ExtraCameraKeys, + steady: bool, + ) -> bool { + let timing_changed = !same_region_timing(&self.camera_regions, ®ions); + let keys_changed = !same_extra_keys(&self.extra_keys, &keys); + self.camera_regions = regions; + let mut closed = false; + if keys_changed { + let mut old = std::mem::take(&mut self.extra); + let mut extra = Vec::with_capacity(keys.len()); + for (k, key) in keys.iter().enumerate() { + let old_cam = old.get_mut(k).and_then(Option::take); + let old_key = self.extra_keys.get(k).cloned().flatten(); + if same_extra_key(&old_key, key) { + extra.push(old_cam); + continue; + } + closed |= old_cam.is_some(); + drop(old_cam); + extra.extend(open_extra_cameras(std::slice::from_ref(key), &self.gpu)); + } + closed |= old.iter().any(Option::is_some); + self.extra = extra; + self.extra_keys = keys; + } + if !extras_need_seek(keys_changed, timing_changed, steady) { + return closed; + } + let t = self.sdec.cur_time_sec(); + closed | self.advance_extras(t, true).1 + } + + /// The extra cameras' frames at screen source time `screen_t`, each camera stepped + /// (`seek == false`) or sought toward its own source time only while it is near one of + /// its regions; an idle camera is left alone and gives null. A camera whose decoder fails + /// is closed with one warning — its layer disappears, the preview goes on. The `bool` says + /// a decoder was closed (see `set_extra_cameras`). + unsafe fn advance_extras(&mut self, screen_t: f64, seek: bool) -> (Vec<*const AVFrame>, bool) { + let mut closed = false; + let mut frames = Vec::with_capacity(self.extra.len()); + for (k, slot) in self.extra.iter_mut().enumerate() { + let camera = k + 1; + let frame = match slot { + Some(cam) if extra_camera_active(&self.camera_regions, camera, screen_t) => { + let target = camera_source_time(screen_t, cam.offset_sec); + let result = if seek { cam.seek(target) } else { cam.step_to(target) }; + match result { + Ok(frame) => frame, + Err(e) => { + eprintln!("WARNING: extra camera {camera} stopped decoding: {e:#}. Its layer will not be drawn."); + *slot = None; + closed = true; + std::ptr::null() + } + } + } + Some(cam) => { + cam.shown = false; + std::ptr::null() + } + None => std::ptr::null(), + }; + frames.push(frame); + } + (frames, closed) + } + + /// Hands the extra cameras' frames to `comp` for the next `compose_frame`. Called before + /// every compose: the compositor keeps the pointers until the next call. + unsafe fn hand_extra_frames(comp: &Compositor, frames: &[*const AVFrame], closed: bool) { + if closed { + comp.clear_srv_cache(); + } + comp.set_extra_camera_frames(frames); } /// Le décodeur webcam ACTIF est-il la vraie caméra ? `false` quand c'est le remplaçant @@ -421,6 +675,17 @@ impl Player { self.webcam_decoder_is_real } + /// La frame webcam à composer avec la frame écran `screen` : celle de la caméra, ou sans + /// caméra la frame écran elle-même, que le remplaçant aurait décodée à l'identique et que + /// la composition ne dessine pas (`should_draw_webcam`). + unsafe fn webcam_frame(&self, screen: *mut crate::ffi::AVFrame) -> *mut crate::ffi::AVFrame { + if self.webcam_decoder_is_real { + self.wdec.cur_frame() + } else { + screen + } + } + /// Temps source courant du décodeur écran — utilisé par `render_thread` pour détecter le /// franchissement de la fin de fenêtre du clip actif pendant la lecture libre, et pour /// calculer la cible de `step` en lecture libre. `pub` (pas `pub(crate)`) : le harnais @@ -429,6 +694,12 @@ impl Player { self.sdec.cur_time_sec() } + /// Temps source courant du décodeur webcam, le remplaçant compris : sans caméra il ne doit + /// jamais bouger (`tests/no_camera_stand_in.rs`). + pub unsafe fn webcam_time_sec(&self) -> f64 { + self.wdec.cur_time_sec() + } + /// Recalcule l'horloge programme pour le clip `clip_index` de `scene` (la scène COMPLÈTE, /// pas la fenêtre d'un clip : il faut la durée de sortie des clips précédents). /// @@ -509,7 +780,10 @@ impl Player { } let target_webcam_t = (self.sdec.cur_time_sec() - self.webcam_offset_sec).max(0.0); - let wf = if use_current { + let wf = if !self.webcam_decoder_is_real { + // Pas de caméra : le remplaçant n'est pas décodé (cf. `open_webcam_or_stand_in`). + sf + } else if use_current { self.wdec.cur_frame() } else { let cur = self.wdec.cur_frame(); @@ -557,6 +831,8 @@ impl Player { self.has_current_frame = true; self.sync_time(comp); + let (extra, closed) = self.advance_extras(self.sdec.cur_time_sec(), false); + Self::hand_extra_frames(comp, &extra, closed); comp.compose_frame(sf, wf, self.idx as f32, cfg)?; self.idx = self.idx.wrapping_add(1); Ok(true) @@ -582,11 +858,15 @@ impl Player { return Ok(false); } let sf = self.sdec.cur_frame(); - let wf = self.wdec.cur_frame(); + let wf = self.webcam_frame(sf); if sf.is_null() || wf.is_null() { return Ok(false); } self.sync_time(comp); + let extra = extra_frame_list(&self.extra, |cam| { + if cam.shown { cam.dec.cur_frame() as *const AVFrame } else { std::ptr::null() } + }); + comp.set_extra_camera_frames(&extra); let f = self.idx.saturating_sub(1); comp.compose_frame(sf, wf, f as f32, cfg)?; Ok(true) @@ -598,10 +878,15 @@ impl Player { /// aucun raccourci keyframe pour les seeks avant lointains (lent ET, combiné au bug de /// `set_time`, incorrect au-delà de 6s sur un enregistrement réel). pub unsafe fn present_frame(&mut self, comp: &Compositor, cfg: &Cfg, target_sec: f64) -> Result { - let sf = self.sdec.seek_to(target_sec)?; - let wf = self - .wdec - .seek_to(webcam_seek_time(target_sec, self.webcam_offset_sec))?; + let sf = self.sdec.seek_to_or_last(target_sec)?; + // La caméra aussi, au-delà de sa dernière image comprise : sans frame webcam, la frame + // ne serait pas composée. Sans caméra, la frame écran tient sa place. + let wf = if self.webcam_decoder_is_real { + self.wdec + .seek_to_or_last(camera_source_time(target_sec, self.webcam_offset_sec))? + } else { + sf + }; if sf.is_null() || wf.is_null() { self.has_current_frame = false; return Ok(false); @@ -612,6 +897,8 @@ impl Player { // "idx" ne sert plus qu'au fallback fixture (jamais lu si une scène est posée) — dérivé // du temps réel pour rester cohérent si jamais consulté. self.idx = (target_sec * self.sdec.fps()).round().max(0.0) as u32; + let (extra, closed) = self.advance_extras(target_sec, true); + Self::hand_extra_frames(comp, &extra, closed); comp.compose_frame(sf, wf, self.idx as f32, cfg)?; Ok(true) } @@ -690,6 +977,9 @@ struct ActiveClipRequest { screen_path: String, webcam_path: String, webcam_offset_sec: f64, + /// Cameras 2-4 of the clip (index k-1 = camera k); which of them open is decided by the + /// clip's layout regions (`extra_cameras_for_clip`). + additional_cameras: Vec, /// Identité dans le flux `Scene.clips` trié (les chemins ne suffisent pas pour un asset partagé). clip_index: usize, /// Playhead exprimé sur l'horloge source écran du nouveau clip. @@ -756,15 +1046,34 @@ fn scene_for_clip(scene: &Scene, clip_index: usize) -> Scene { /// Dernière frame readback vers CPU, prête pour le napi `read_frame`. /// -/// `(gen, w, h, vec)` où `vec.len() == w*h*4` octets RGBA8 tightly-packed (R, G, B, A -/// en mémoire — cf. `Compositor::readback_resized`). `gen` est une génération monotone +/// `(gen, w, h, vec, métrage, position)` où `vec.len() == w*h*4` octets RGBA8 tightly-packed +/// (R, G, B, A en mémoire — cf. `Compositor::readback_resized`). `gen` est une génération monotone /// (≥ 1, `0` réservé à « le consommateur n'a encore rien vu ») incrémentée à CHAQUE /// publication, càd uniquement quand une nouvelle frame a réellement été composée (le /// thread de rendu ne republie pas une frame identique — cf. `stepped || first`). Elle /// est l'IDENTITÉ de la frame : le consommateur (`read_frame`) ne repaie le clone + l'IPC /// que lorsqu'elle change. `None` = "aucune frame composée pour l'instant" (toutes les /// lectures avant la 1re frame composée retournent `None` côté napi, jamais un buffer vide). -type LatestFrame = (u64, u32, u32, Vec); +pub type LatestFrame = ( + u64, + u32, + u32, + Vec, + Option, + FramePosition, +); + +/// Où en est la vue quand elle compose une frame : le clip actif de la scène et le temps +/// source de la frame écran. Voyage avec chaque frame publiée, quel que soit le transport, +/// pour que l'app compare la position RÉELLE de la vue à sa propre tête de lecture au lieu +/// de la deviner à partir de l'horloge murale. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct FramePosition { + /// Index du clip actif dans `scene.clips`. + pub clip_index: u32, + /// pts de la frame écran composée, en secondes du fichier source. + pub source_time_sec: f64, +} /// État partagé thread appelant → thread de rendu (commandes sans blocage). struct Shared { @@ -800,6 +1109,15 @@ struct Shared { /// vide la ferait repartir à 1 — donc rejouer des générations déjà peintes. Monotone, /// jamais remise à zéro. frame_gen: AtomicU64, + /// La vue livre ses frames dans l'anneau de textures partagées (`take_shared_frame`) au + /// lieu de les relire en RAM (`latest_frame_since`). Posé par JS ; le thread de rendu le + /// rabat à `false` si l'anneau ne peut pas servir, et les frames repassent par la RAM. + shared_frames: AtomicBool, + /// Tenue des cases de l'anneau, entre le thread de rendu et le thread Node. + slot_book: Mutex, + /// Republier la frame courante au prochain tour, même en pause : posé quand le transport + /// change, pour que le consommateur ait une image sans attendre que quelque chose bouge. + republish: AtomicBool, /// Erreur fatale du thread de rendu (device D3D11 introuvable, décodeur qui refuse /// le fichier…). Le thread meurt sur la première erreur ; sans ce champ, elle /// finissait dans un `eprintln!` que personne ne lit et l'utilisateur n'avait @@ -853,6 +1171,9 @@ impl LiveView { stop: AtomicBool::new(false), latest_frame: Mutex::new(None), frame_gen: AtomicU64::new(0), + shared_frames: AtomicBool::new(false), + slot_book: Mutex::new(SlotBook::default()), + republish: AtomicBool::new(false), fatal: Mutex::new(None), }); let sh = shared.clone(); @@ -894,7 +1215,7 @@ impl LiveView { /// FFI vers le Buffer napi). Le `Vec` retourné a `len() == w*h*4`. /// Préférer `latest_frame_since` sur le chemin chaud : il évite ce clone quand /// le consommateur possède déjà la génération courante. - pub fn latest_frame(&self) -> Option<(u64, u32, u32, Vec)> { + pub fn latest_frame(&self) -> Option { self.shared .latest_frame .lock() @@ -910,7 +1231,7 @@ impl LiveView { /// frame figée — on n'exécute PAS le clone `O(w·h)` : c'est tout l'intérêt du /// compteur. Le consommateur passe la dernière génération qu'il a peinte (`0` au /// départ) ; `None` ⇒ il ne fait rien, `Some` ⇒ il peint et retient `gen`. - pub fn latest_frame_since(&self, since_gen: u64) -> Option<(u64, u32, u32, Vec)> { + pub fn latest_frame_since(&self, since_gen: u64) -> Option { let mut guard = self.shared.latest_frame.lock().ok()?; match guard.as_ref() { // Le buffer est EMPORTÉ, pas copié. Le thread de rendu le remplace à chaque @@ -929,6 +1250,35 @@ impl LiveView { } } + /// Livrer les frames par textures partagées plutôt que par readback. Seul le backend + /// matériel de Windows sait le faire : ailleurs la demande est refusée et la vue continue + /// de relire en RAM. Rend l'état obtenu. + pub fn set_shared_frames(&self, enabled: bool) -> bool { + let on = enabled && cfg!(windows) && Gpu::probe() == Some(Backend::Hardware); + if self.shared.shared_frames.swap(on, Ordering::Relaxed) != on { + if !on { + if let Ok(mut book) = self.shared.slot_book.lock() { + book.clear_ready(); + } + } + self.shared.republish.store(true, Ordering::Relaxed); + } + on + } + + /// La dernière frame posée dans l'anneau, si elle est plus récente que `since_gen`. Sa + /// case reste tenue jusqu'à `release_shared_frame`. `None` aussi quand la vue relit en RAM. + pub fn take_shared_frame(&self, since_gen: u64) -> Option { + self.shared.slot_book.lock().ok()?.take(since_gen, Instant::now()) + } + + /// Chromium a relâché la frame `gen` de la case `slot` : le thread de rendu peut y réécrire. + pub fn release_shared_frame(&self, slot: u32, gen: u64) { + if let Ok(mut book) = self.shared.slot_book.lock() { + book.release(slot, gen); + } + } + /// Switch inspector (booléen). pub fn set_param_bool(&self, key: &str, value: bool) { if let Ok(mut p) = self.shared.inspector.lock() { @@ -1026,6 +1376,7 @@ impl LiveView { screen_path: &str, webcam_path: &str, webcam_offset_sec: f64, + additional_cameras: Vec, clip_index: usize, source_time_sec: f64, ) { @@ -1034,6 +1385,7 @@ impl LiveView { screen_path: screen_path.to_string(), webcam_path: webcam_path.to_string(), webcam_offset_sec, + additional_cameras, clip_index, source_time_sec: source_time_sec.max(0.0), }); @@ -1132,7 +1484,7 @@ type PendingPrefetch = (usize, std::sync::mpsc::Receiver> /// tâche de fond. Assez large pour couvrir un `Decoder::open` typique (ouverture fichier + /// `avformat_find_stream_info` + init D3D11VA), assez court pour ne pas garder deux paires de /// décodeurs ouvertes plus longtemps que nécessaire. -const PREFETCH_LEAD_SEC: f64 = 0.75; +pub(crate) const PREFETCH_LEAD_SEC: f64 = 0.75; /// Durée pendant laquelle la boucle continue de recomposer après un changement en pause, le /// temps qu'un effet asynchrone (segmentation webcam) livre son résultat. Généreuse : à @@ -1186,6 +1538,7 @@ unsafe fn maybe_start_prefetch( 0 }; let next_clip = scene.clips[next_index].clone(); + let extra_keys = scene_extra_camera_keys(scene, next_index); // Copie légère (COM refcount, pas de nouveau device) — même motif que `Player::open`. let gpu_clone = Gpu { device: gpu.device.clone(), @@ -1200,6 +1553,7 @@ unsafe fn maybe_start_prefetch( &next_clip.screen_path, &next_clip.webcam_path, next_clip.webcam_offset_sec, + &extra_keys, next_clip.source_start_sec, &gpu_clone, ) @@ -1237,6 +1591,7 @@ unsafe fn advance_to_next_scene_clip( active_screen_path: &mut String, active_webcam_path: &mut String, active_webcam_offset_sec: &mut f64, + active_additional_cameras: &mut Vec, active_clip_index: &mut usize, raw_cursor: &mut Option, loaded_cursor_path: &mut String, @@ -1251,6 +1606,7 @@ unsafe fn advance_to_next_scene_clip( 0 }; let next_clip = &scene.clips[next_index]; + let extra_keys = scene_extra_camera_keys(scene, next_index); // N'importe quel préchargement en cours ne concerne plus que CETTE frontière (on vient // de la franchir, bien ou mal ciblée) — on le consomme s'il correspond, on l'abandonne @@ -1278,6 +1634,7 @@ unsafe fn advance_to_next_scene_clip( &next_clip.screen_path, &next_clip.webcam_path, next_clip.webcam_offset_sec, + &extra_keys, next_clip.source_start_sec, ) } @@ -1285,6 +1642,7 @@ unsafe fn advance_to_next_scene_clip( &next_clip.screen_path, &next_clip.webcam_path, next_clip.webcam_offset_sec, + &extra_keys, next_clip.source_start_sec, ), }; @@ -1296,12 +1654,18 @@ unsafe fn advance_to_next_scene_clip( // sur l'adresse de la texture, et garder des entrées d'un décodeur fermé fait // fuir de la VRAM puis, en cas de réutilisation d'adresse, rendre l'image du clip // précédent. + let windowed = scene_for_clip(scene, next_index); + // The prefetch opened exactly these keys, so this only installs the regions and + // positions the cameras; the cache is cleared right below in any case. + // The screen position jumped to the next clip: always position the cameras. + player.set_extra_cameras(windowed.camera_layout_regions.clone(), extra_keys, false); comp.clear_srv_cache(); *active_screen_path = next_clip.screen_path.clone(); *active_webcam_path = next_clip.webcam_path.clone(); *active_webcam_offset_sec = next_clip.webcam_offset_sec; + *active_additional_cameras = next_clip.additional_cameras.clone(); *active_clip_index = next_index; - comp.set_scene(Some(scene_for_clip(scene, *active_clip_index))); + comp.set_scene(Some(windowed)); player.set_programme_clock(Some(scene), *active_clip_index); // Réutilise le curseur préchargé s'il est disponible (voir plus haut) — sinon // (préchargement pas encore prêt / raté) on retombe sur la lecture synchrone @@ -1379,6 +1743,9 @@ unsafe fn render_thread( let mut active_screen_path = screen.to_string(); let mut active_webcam_path = webcam.to_string(); let mut active_webcam_offset_sec = 0.0f64; + // Cameras 2-4 of the active clip, as the app (`set_active_clip`) or the scene (auto-advance) + // gave them. The view starts without: `create_view` carries camera 0 only. + let mut active_additional_cameras: Vec = Vec::new(); let mut active_clip_index = 0usize; // Copie de la Scene complète (tous les clips), tenue à jour à chaque push de l'app — // permet à la boucle de lecture libre de connaître la fenêtre source @@ -1426,8 +1793,19 @@ unsafe fn render_thread( // appliquée, on refuse de jouer le layout fixture (POC) : un fallback fixture ne ferait que // MASQUER un scene-push cassé. On attend la scène avant de produire le 1er frame. let mut scene_applied = false; + // Textures partagées de la preview, créées au premier besoin (voir `publish_shared`). + let mut ring = Ring::default(); + // Une frame composée qui n'a trouvé aucune case libre dans l'anneau : le RT la garde, elle + // repart dès qu'une case se libère. Sans ça, une frame composée en pause (seek, réglage) + // restait invisible jusqu'au changement suivant — en lecture, la suivante la remplace. + let mut publish_pending = false; while !shared.stop.load(Ordering::SeqCst) { + // Le transport vient de changer : le consommateur doit recevoir la frame courante + // sans attendre qu'une autre soit composée (en pause, il n'y en aurait pas). + if shared.republish.swap(false, Ordering::Relaxed) { + first = true; + } // params inspector : booléens/taps → cfg ; valeurs continues → live_params let ip = *shared.inspector.lock().unwrap(); let mut clip_changed = false; @@ -1448,6 +1826,26 @@ unsafe fn render_thread( // JAMAIS fatal : on retombe sur l'ouverture complète, chemin connu comme sûr. // L'optimisation ne s'applique donc que là où elle fonctionne démontrablement. let repositioned = same_media && matches!(player.seek_active(request.source_time_sec), Ok(true)); + // The extra cameras this clip's layout regions show, from the files the request + // carries. Resolved against the scene before the switch so that the pool and the + // fresh open both see the full key. + let request_scene = shared.scene.lock().unwrap().clone(); + let request_clip_index = request_scene.as_ref().and_then(|scene| { + resolve_scene_clip_index( + scene, + request.clip_index, + &request.screen_path, + &request.webcam_path, + request.webcam_offset_sec, + ) + }); + let request_extras = match (&request_scene, request_clip_index) { + (Some(scene), Some(index)) => extra_camera_keys( + &extra_cameras_for_clip(scene, index), + &request.additional_cameras, + ), + _ => Vec::new(), + }; let switch_result = if repositioned { Ok(()) } else { @@ -1455,13 +1853,24 @@ unsafe fn render_thread( // ouverte si possible (reseek au lieu de rouvrir) et met en pool celle qu'on // quitte, au lieu du couple ouvrir-puis-fermer. Voir `swap_clip_pooled` et la // mesure de ~120 ms/franchissement qui l'a motivé. + let incoming = ClipKey { + screen_path: request.screen_path.clone(), + webcam_path: request.webcam_path.clone(), + webcam_offset_sec: request.webcam_offset_sec, + extras: request_extras.clone(), + }; + let active = ClipKey { + screen_path: active_screen_path.clone(), + webcam_path: active_webcam_path.clone(), + webcam_offset_sec: active_webcam_offset_sec, + extras: player.extra_keys().to_vec(), + }; swap_clip_pooled( &mut player, &mut decoder_pool, - &request, - &active_screen_path, - &active_webcam_path, - active_webcam_offset_sec, + &incoming, + request.source_time_sec, + &active, ) }; match switch_result { @@ -1483,8 +1892,18 @@ unsafe fn render_thread( active_screen_path = request.screen_path; active_webcam_path = request.webcam_path; active_webcam_offset_sec = request.webcam_offset_sec; - let scene = shared.scene.lock().unwrap().clone(); + active_additional_cameras = request.additional_cameras; + let scene = request_scene; full_scene = scene.clone(); + // No scene clip for the request → no layout regions (and no extras). + let regions = match (&scene, request_clip_index) { + (Some(s), Some(index)) => clip_layout_regions(s, index), + _ => Vec::new(), + }; + // A clip request moves the screen position: position the cameras. + if player.set_extra_cameras(regions, request_extras, false) { + comp.clear_srv_cache(); + } if let Some(base_scene) = scene { if let Some(index) = resolve_scene_clip_index( &base_scene, @@ -1590,6 +2009,7 @@ unsafe fn render_thread( prefetch = None; let scene = shared.scene.lock().unwrap().clone(); full_scene = scene.clone(); + let mut clip_resolved = false; let scene = scene.map(|base_scene| { scene_applied = true; if let Some(index) = resolve_scene_clip_index( @@ -1600,9 +2020,35 @@ unsafe fn render_thread( active_webcam_offset_sec, ) { active_clip_index = index; + clip_resolved = true; } scene_for_clip(&base_scene, active_clip_index) }); + // The resolved scene clip's own camera files win over the last clip request's: an + // offset or file edited mid-clip arrives with the scene, not with a new request. + if clip_resolved { + if let Some(clip) = full_scene.as_ref().and_then(|s| s.clips.get(active_clip_index)) { + active_additional_cameras = clip.additional_cameras.clone(); + } + } + // A new scene can add, move or drop the layout regions that show an extra camera: + // open what the active clip now shows, close what it no longer does. + let (regions, keys) = match (&full_scene, &scene) { + (Some(full), Some(windowed)) => ( + windowed.camera_layout_regions.clone(), + extra_camera_keys( + &extra_cameras_for_clip(full, active_clip_index), + &active_additional_cameras, + ), + ), + _ => (Vec::new(), Vec::new()), + }; + // A scene push leaves the screen position where it is: while playing, unchanged + // cameras are stepped by playback instead of sought again. + let steady = shared.playing.load(Ordering::Relaxed); + if player.set_extra_cameras(regions, keys, steady) { + comp.clear_srv_cache(); + } comp.set_scene(scene); } // Le temps programme dépend du clip actif ET de la scène entière (durées des clips @@ -1717,6 +2163,7 @@ unsafe fn render_thread( &mut active_screen_path, &mut active_webcam_path, &mut active_webcam_offset_sec, + &mut active_additional_cameras, &mut active_clip_index, &mut raw_cursor, &mut loaded_cursor_path, @@ -1745,6 +2192,7 @@ unsafe fn render_thread( &mut active_screen_path, &mut active_webcam_path, &mut active_webcam_offset_sec, + &mut active_additional_cameras, &mut active_clip_index, &mut raw_cursor, &mut loaded_cursor_path, @@ -1783,39 +2231,56 @@ unsafe fn render_thread( stepped = true; } - if stepped || first { + if stepped || first || publish_pending { if pw > 0 && ph > 0 { - // Step complet : `compose_frame` (déjà appelé par `step`/`present_frame`/ - // `recompose`) a rastérisé le RT à la géométrie de sortie ramenée au panneau. - // On lit ce RT DIRECTEMENT à sa résolution de rendu (`readback_direct` : copy - // rt → staging → Map/Unmap), sans le resize `blit_resized` qui, depuis la - // refonte ratio, n'était plus qu'une copie identité + une alloc NV12 inutile. - match comp.readback_direct() { - Ok((rw, rh, rgba)) => { - // Publie dans `latest_frame` : on remplace le buffer précédent - // (le canvas ne montre que la dernière frame, peu importe combien - // le renderer en a raté entre deux lectures napi). On incrémente - // la génération sous le MÊME lock que l'écriture du buffer, pour - // qu'un lecteur ne puisse jamais voir un `gen` neuf appairé à un - // buffer périmé (ou l'inverse). `+ 1` depuis la précédente, `1` au - // premier publish. Les dims publiées sont celles du RENDU (`rw`×`rh`) : - // le canvas JS s'y dimensionne (packet auto-descriptif) puis CSS met à - // l'échelle vers la boîte du panneau — plus de resize GPU intermédiaire. - // La génération vient d'un compteur atomique et non du slot : la - // livraison sans copie VIDE le slot en le lisant, et un - // `unwrap_or(1)` repartirait alors de 1 — le consommateur recevrait - // des générations déjà peintes et boucherait. Séquence identique à - // l'ancienne dérivation tant que le slot n'est pas vidé. - let next_gen = shared.frame_gen.fetch_add(1, Ordering::Relaxed) + 1; - if let Ok(mut slot) = shared.latest_frame.lock() { - *slot = Some((next_gen, rw, rh, rgba)); - } + let position = FramePosition { + clip_index: active_clip_index as u32, + source_time_sec: player.screen_time_sec(), + }; + match publish_shared(&shared, &gpu, &comp, &mut ring, position) { + SharedPublish::Published => { first = false; + publish_pending = false; } - Err(e) => { - eprintln!("[live] readback_direct: {e:#}"); - std::thread::sleep(Duration::from_millis(8)); + // Chromium tient toutes les cases : la frame attend dans le RT, et la boucle + // ne tourne pas à vide le temps qu'il en relâche une. + SharedPublish::NoFreeSlot => { + publish_pending = true; + std::thread::sleep(Duration::from_millis(2)); } + // Step complet : `compose_frame` (déjà appelé par `step`/`present_frame`/ + // `recompose`) a rastérisé le RT à la géométrie de sortie ramenée au panneau. + // On lit ce RT DIRECTEMENT à sa résolution de rendu (`readback_direct` : copy + // rt → staging → Map/Unmap), sans le resize `blit_resized` qui, depuis la + // refonte ratio, n'était plus qu'une copie identité + une alloc NV12 inutile. + SharedPublish::Off => match comp.readback_direct() { + Ok((rw, rh, rgba)) => { + // Publie dans `latest_frame` : on remplace le buffer précédent + // (le canvas ne montre que la dernière frame, peu importe combien + // le renderer en a raté entre deux lectures napi). On incrémente + // la génération sous le MÊME lock que l'écriture du buffer, pour + // qu'un lecteur ne puisse jamais voir un `gen` neuf appairé à un + // buffer périmé (ou l'inverse). `+ 1` depuis la précédente, `1` au + // premier publish. Les dims publiées sont celles du RENDU (`rw`×`rh`) : + // le canvas JS s'y dimensionne (packet auto-descriptif) puis CSS met à + // l'échelle vers la boîte du panneau — plus de resize GPU intermédiaire. + // La génération vient d'un compteur atomique et non du slot : la + // livraison sans copie VIDE le slot en le lisant, et un + // `unwrap_or(1)` repartirait alors de 1 — le consommateur recevrait + // des générations déjà peintes et boucherait. Séquence identique à + // l'ancienne dérivation tant que le slot n'est pas vidé. + let next_gen = shared.frame_gen.fetch_add(1, Ordering::Relaxed) + 1; + if let Ok(mut slot) = shared.latest_frame.lock() { + *slot = Some((next_gen, rw, rh, rgba, comp.footage_quad(), position)); + } + first = false; + publish_pending = false; + } + Err(e) => { + eprintln!("[live] readback_direct: {e:#}"); + std::thread::sleep(Duration::from_millis(8)); + } + }, } } } else { @@ -1825,6 +2290,101 @@ unsafe fn render_thread( Ok(()) } +/// Ce que la publication dans l'anneau partagé a fait de la frame composée. +enum SharedPublish { + /// Posée dans une case de l'anneau. + Published, + /// Chromium tient toutes les cases : la frame attend dans le RT qu'il en relâche une. + NoFreeSlot, + /// Transport partagé coupé ou indisponible : la frame repasse par le readback. + Off, +} + +#[cfg(windows)] +type Ring = Option; +#[cfg(not(windows))] +type Ring = (); + +/// Pose la frame composée dans l'anneau de textures partagées, si la vue livre ainsi. +/// +/// Toute panne coupe le transport (`shared_frames` à `false`) et rend `Off` : la frame part +/// par le readback, et le service, qui lit les deux, n'a rien à décider. +#[cfg(windows)] +unsafe fn publish_shared( + shared: &Shared, + gpu: &Gpu, + comp: &Compositor, + ring: &mut Ring, + position: FramePosition, +) -> SharedPublish { + if !shared.shared_frames.load(Ordering::Relaxed) { + return SharedPublish::Off; + } + let turn_off = |why: String| { + eprintln!("[live] textures partagées coupées ({why}) — retour au readback"); + shared.shared_frames.store(false, Ordering::Relaxed); + SharedPublish::Off + }; + // WARP ne partage rien avec le device de Chromium. + if gpu.backend != Backend::Hardware { + return turn_off("backend logiciel".into()); + } + if ring.is_none() { + match SharedRing::new(gpu) { + Ok(created) => *ring = Some(created), + Err(e) => return turn_off(format!("{e:#}")), + } + } + let Some(ring) = ring.as_mut() else { + return SharedPublish::Off; + }; + let (claimed, lost) = match shared.slot_book.lock() { + Ok(mut book) => (book.claim(crate::shared_frames::RING_SLOTS), book.lost(Instant::now())), + Err(_) => (None, false), + }; + let Some(slot) = claimed else { + // Chromium relâche d'ordinaire une case dans la milliseconde. Une case perdue ne + // reviendra plus : la réécrire déchirerait peut-être une image qu'il lit encore, et + // l'attendre figerait la preview. + if lost { + return turn_off("une case jamais relâchée par Chromium".into()); + } + return SharedPublish::NoFreeSlot; + }; + let (width, height) = comp.render_size(); + match ring.write(slot, comp.render_target(), width, height) { + Ok(handle) => { + if let Ok(mut book) = shared.slot_book.lock() { + // Le compteur du readback : une seule suite de générations quel que soit le + // transport, incrémentée sous le lock de la publication comme là-bas. + let gen = shared.frame_gen.fetch_add(1, Ordering::Relaxed) + 1; + book.publish(SharedFrame { + gen, + slot, + handle, + width, + height, + footage: comp.footage_quad(), + position, + }); + } + SharedPublish::Published + } + Err(e) => turn_off(format!("{e:#}")), + } +} + +#[cfg(not(windows))] +unsafe fn publish_shared( + _: &Shared, + _: &Gpu, + _: &Compositor, + _: &mut Ring, + _: FramePosition, +) -> SharedPublish { + SharedPublish::Off +} + // ---------- harnais standalone (poc-d3d.exe --live) ---------- // // Le harnais crée une fenêtre Win32 simple qui héberge la preview live ; c'est un @@ -1958,7 +2518,7 @@ pub fn run_standalone(screen: &str, webcam: &str, cursor_json: &str) -> Result<( // standalone n'affiche pas réellement les pixels ici (l'embed Electron est // le consumer réel). On imprime juste une frame de temps en temps pour // confirmer que la chaîne fonctionne. - if let Some((_gen, fw, fh, _pixels)) = view.latest_frame() { + if let Some((_gen, fw, fh, _pixels, _, _)) = view.latest_frame() { if (fw, fh) != (w, h) { // garde-fou : la staging de readback suit `set_rect` côté thread // de rendu, donc ce serait une désynchro transitoire — acceptable. @@ -2092,8 +2652,147 @@ mod tests { #[test] fn webcam_seek_uses_screen_source_time_and_offset() { - assert_eq!(webcam_seek_time(22.5, 1.25), 21.25); - assert_eq!(webcam_seek_time(0.5, 1.25), 0.0); + assert_eq!(camera_source_time(22.5, 1.25), 21.25); + assert_eq!(camera_source_time(0.5, 1.25), 0.0); + } + + // --- extra cameras (2-4) ---------------------------------------------------- + + /// Two clips: clip 0 has files for cameras 1 and 3 (camera 2's slot is empty), clip 1 for + /// camera 1 only. Layout regions: clip 0 shows cameras 1 and 2 on 2-5 s and camera 3 only + /// in an empty region; clip 1 shows camera 1 on 21-22 s. + fn layout_scene() -> Scene { + let layer = |camera: usize| { + format!(r#"{{"camera":{camera},"rect":{{"x":0,"y":0,"width":0.5,"height":0.5}},"radiusFrac":0,"shape":"rectangle","fillsFrame":false}}"#) + }; + let json = format!( + r##"{{ + "clips": [ + {{"screenPath":"/s0.mp4","webcamPath":"/w0.mp4","sourceStartSec":0,"sourceEndSec":10,"webcamOffsetSec":0,"hasAudio":true, + "additionalCameras":[{{"path":"/c1.mp4","offsetSec":0.5}},{{"path":"","offsetSec":0}},{{"path":"/c3.mp4","offsetSec":0}}]}}, + {{"screenPath":"/s1.mp4","webcamPath":"/w1.mp4","sourceStartSec":20,"sourceEndSec":30,"webcamOffsetSec":0,"hasAudio":true, + "additionalCameras":[{{"path":"/d1.mp4","offsetSec":0}}]}} + ], + "layout":{{"preset":"picture-in-picture","webcamSize":1,"webcamShape":"rectangle","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false}}, + "effects":{{"padding":0,"blur":false,"shadow":0,"roundnessFrac":0,"motionBlur":0}}, + "background":{{"kind":"color","color":"#000000"}}, + "zoomRegions":[], + "cameraLayoutRegions":[ + {{"clipIndex":0,"startSec":2,"endSec":5,"layers":[{l0},{l1},{l2}]}}, + {{"clipIndex":0,"startSec":7,"endSec":7,"layers":[{l3}]}}, + {{"clipIndex":1,"startSec":21,"endSec":22,"layers":[{l1}]}} + ], + "cursor":{{"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0,"clipToBounds":false,"theme":"default"}}, + "cropByClip":[null,null], + "output":{{"width":1920,"height":1080,"fps":30}} + }}"##, + l0 = layer(0), + l1 = layer(1), + l2 = layer(2), + l3 = layer(3), + ); + Scene::from_json(&json).expect("layout scene") + } + + fn camera(path: &str, offset_sec: f64) -> SceneClipCamera { + SceneClipCamera { path: path.to_string(), offset_sec } + } + + #[test] + fn extra_cameras_to_open_are_those_the_clips_regions_reference() { + let scene = layout_scene(); + // Camera 2 is shown but clip 0 has no file for it; camera 3 has a file but only an + // empty region; camera 0 is not an extra camera. + assert_eq!(extra_cameras_for_clip(&scene, 0), vec![1]); + // Clip 1 only sees its own region. + assert_eq!(extra_cameras_for_clip(&scene, 1), vec![1]); + assert!(extra_cameras_for_clip(&scene, 9).is_empty()); + // No layout region at all: nothing to open, whatever files the clip has. + assert!(extra_cameras_for_clip(&multiclip_scene(), 0).is_empty()); + + let keys = scene_extra_camera_keys(&scene, 0); + assert_eq!(keys.len(), 1, "trailing empty slots are dropped"); + let first = keys[0].as_ref().expect("camera 1"); + assert_eq!((first.path.as_str(), first.offset_sec), ("/c1.mp4", 0.5)); + // A shown camera after an unshown one keeps its index; nothing shown is an empty list. + let sources = [camera("/a.mp4", 0.0), camera("/b.mp4", 0.0), camera("/c.mp4", 0.0)]; + let keys = extra_camera_keys(&[3], &sources); + assert_eq!(keys.len(), 3); + assert!(keys[0].is_none() && keys[1].is_none()); + assert_eq!(keys[2].as_ref().map(|k| k.path.as_str()), Some("/c.mp4")); + assert!(extra_camera_keys(&[], &sources).is_empty()); + assert!(extra_camera_keys(&[2], &[camera("/a.mp4", 0.0)]).is_empty()); + } + + #[test] + fn an_extra_camera_is_decoded_only_near_its_regions() { + let regions = scene_for_clip(&layout_scene(), 0).camera_layout_regions; + // From PREFETCH_LEAD_SEC before the region to its end. + assert!(!extra_camera_active(®ions, 1, 2.0 - PREFETCH_LEAD_SEC - 0.01)); + assert!(extra_camera_active(®ions, 1, 2.0 - PREFETCH_LEAD_SEC)); + assert!(extra_camera_active(®ions, 1, 3.0)); + assert!(extra_camera_active(®ions, 1, 5.0)); + assert!(!extra_camera_active(®ions, 1, 5.01)); + // A camera the region does not show, and one only an empty region names. + assert!(!extra_camera_active(®ions, 3, 7.0)); + assert!(!extra_camera_active(&[], 1, 3.0)); + } + + /// Playing on with the same files and region timing, a scene push does not seek the extra + /// cameras again; any change that matters, a jump, or a pause does. + #[test] + fn extra_cameras_are_sought_again_only_when_it_matters() { + assert!(!extras_need_seek(false, false, true)); + assert!(extras_need_seek(true, false, true), "new files"); + assert!(extras_need_seek(false, true, true), "regions moved"); + assert!(extras_need_seek(false, false, false), "paused, or the position jumped"); + + let regions = scene_for_clip(&layout_scene(), 0).camera_layout_regions; + assert!(same_region_timing(®ions, ®ions.clone())); + // A moved rect is the same timing; a moved bound or another camera is not. + let mut moved_rect = regions.clone(); + moved_rect[0].layers[0].rect.x += 0.1; + assert!(same_region_timing(®ions, &moved_rect)); + let mut moved_end = regions.clone(); + moved_end[0].end_sec += 0.5; + assert!(!same_region_timing(®ions, &moved_end)); + let mut other_camera = regions.clone(); + other_camera[0].layers[0].camera = 3; + assert!(!same_region_timing(®ions, &other_camera)); + assert!(!same_region_timing(®ions, ®ions[..1])); + } + + #[test] + fn a_missing_extra_camera_skips_its_layer() { + let failed: Result = Err(anyhow::anyhow!("0-byte file")); + assert_eq!(opened_or_skipped("/c1.mp4", failed), None); + assert_eq!(opened_or_skipped("/c1.mp4", Ok(7u8)), Some(7)); + + // Slot 0 failed to open, slot 1 is open with a frame, slot 2 is open with nothing to + // show (idle or past its end): only slot 1 hands a frame to the compositor. + let frame = 0x10usize as *const AVFrame; + let slots = [None, Some(frame), Some(std::ptr::null())]; + let list = extra_frame_list(&slots, |f| *f); + assert_eq!(list, vec![std::ptr::null(), frame, std::ptr::null()]); + assert!(extra_frame_list::<*const AVFrame>(&[], |f| *f).is_empty()); + } + + #[test] + fn the_pool_key_includes_the_extra_cameras() { + let key = |extras: ExtraCameraKeys| ClipKey { + screen_path: "/s0.mp4".into(), + webcam_path: "/w0.mp4".into(), + webcam_offset_sec: 0.0, + extras, + }; + let with_camera_1 = key(vec![Some(camera("/c1.mp4", 0.5))]); + assert!(with_camera_1.matches(&key(vec![Some(camera("/c1.mp4", 0.5))]))); + // Same screen and camera 0, but camera 1 missing, another file or another offset. + assert!(!with_camera_1.matches(&key(Vec::new()))); + assert!(!with_camera_1.matches(&key(vec![Some(camera("/other.mp4", 0.5))]))); + assert!(!with_camera_1.matches(&key(vec![Some(camera("/c1.mp4", 0.75))]))); + assert!(!with_camera_1.matches(&key(vec![None, Some(camera("/c1.mp4", 0.5))]))); + assert!(key(Vec::new()).matches(&key(Vec::new()))); } #[test] diff --git a/crates/compositor/src/pipeline_linux.rs b/crates/compositor/src/pipeline_linux.rs index b118befdda..c31eb494fd 100644 --- a/crates/compositor/src/pipeline_linux.rs +++ b/crates/compositor/src/pipeline_linux.rs @@ -54,8 +54,13 @@ pub struct ClipSource { pub source_end_sec: f64, pub webcam_offset_sec: f64, pub has_audio: bool, + /// Cameras 2-4 of the clip (index k-1 = camera k, an empty `path` = none). Only those a + /// layout region of the clip shows are decoded. + pub additional_cameras: Vec, } +pub use crate::extra_cameras::ClipCamera; + /// Codec cible. Memes variantes que `pipeline_macos::ExportCodec`. #[derive(Clone, Copy, Debug)] pub enum ExportCodec { @@ -127,6 +132,12 @@ impl Decoder { self.decode_present(idx) } + /// Contrat de `pipeline_windows::Decoder::seek_to_or_last`, que `seek_to` remplit déjà ici : + /// au-delà de la fin, `pump_to_target` tient la dernière image décodée. + pub unsafe fn seek_to_or_last(&mut self, seconds: f64) -> Result<*mut AVFrame> { + self.seek_to(seconds) + } + /// Decode la frame SEQUENTIELLE suivante — pompage `next_frame`, PAS de seek. /// La frame rendue appartient au decodeur (valide jusqu'au prochain appel), /// donc elle ne se libere pas ici, contrairement au chemin `decode_at`. @@ -939,6 +950,7 @@ pub fn run_composited_multi( let mut screen_decs: HashMap = HashMap::new(); let mut webcam_decs: HashMap = HashMap::new(); + let mut extra_decs: HashMap = HashMap::new(); // ---- muxer MP4 (flux video + flux AAC) ---- let outc = CString::new(out)?; @@ -1032,6 +1044,7 @@ pub fn run_composited_multi( &scene, &mut screen_decs, &mut webcam_decs, + &mut extra_decs, &mut |n| { // Soumet la copie de la frame n SANS l'attendre et recolte la // precedente : c'est tout le pipelining GPU. L'encodage, lui, diff --git a/crates/compositor/src/pipeline_macos.rs b/crates/compositor/src/pipeline_macos.rs index 93827b00ce..6fc75f01ab 100644 --- a/crates/compositor/src/pipeline_macos.rs +++ b/crates/compositor/src/pipeline_macos.rs @@ -410,6 +410,25 @@ impl Decoder { } } + /// `seek_to`, mais une cible au-delà de la dernière image se pose sur cette dernière image au + /// lieu de ne rien rendre. L'audio d'un enregistrement dure souvent un peu plus que sa vidéo + /// (de 12 ms à près d'une demi-seconde mesurés) : son clip finit alors après la dernière + /// image, et y entrer par la fin, en scrubant de droite à gauche, demandait une image qui + /// n'existe pas. Le changement de clip échouait sans bruit, et la vue gardait la scène du + /// clip qu'on quittait. + pub unsafe fn seek_to_or_last(&mut self, seconds: f64) -> Result<*mut crate::ffi::AVFrame> { + let frame = self.seek_to(seconds)?; + if !frame.is_null() { + return Ok(frame); + } + // Le seek a décodé jusqu'à l'EOF : `cur_pts` est celui de la dernière image. L'EOF a vidé + // la frame courante, donc il faut le seek complet : `take` écarte le chemin rapide. + match self.cur_pts.take() { + Some(last) => self.seek_to(last as f64 * self.tb_sec()), + None => Ok(frame), + } + } + /// Déroule le décodeur en avant jusqu'à la première frame à `seconds` ou après, SANS /// jeter son état. Symétrique de `pipeline_windows::Decoder::decode_forward_to`. unsafe fn decode_forward_to(&mut self, seconds: f64, tb_sec: f64) -> Result<*mut crate::ffi::AVFrame> { @@ -644,8 +663,13 @@ pub struct ClipSource { pub source_end_sec: f64, pub webcam_offset_sec: f64, pub has_audio: bool, + /// Cameras 2-4 of the clip (index k-1 = camera k, an empty `path` = none). Only those a + /// layout region of the clip shows are decoded. + pub additional_cameras: Vec, } +pub use crate::extra_cameras::ClipCamera; + /// Codec cible pour l'export. Identique à `pipeline_windows::ExportCodec`. pub enum ExportCodec { H264, @@ -1141,6 +1165,8 @@ pub fn run_composited_multi( std::collections::HashMap::new(); let mut webcam_decs: std::collections::HashMap = std::collections::HashMap::new(); + let mut extra_decs: std::collections::HashMap = + std::collections::HashMap::new(); // ---- encodeur (candidat VT ou software) ---- let mut enc = @@ -1222,6 +1248,7 @@ pub fn run_composited_multi( &scene, &mut screen_decs, &mut webcam_decs, + &mut extra_decs, &mut |n| { enc.send_composited(comp, out_w, out_h, n as i64)?; { diff --git a/crates/compositor/src/pipeline_windows.rs b/crates/compositor/src/pipeline_windows.rs index fabe31207e..8094fd44b0 100644 --- a/crates/compositor/src/pipeline_windows.rs +++ b/crates/compositor/src/pipeline_windows.rs @@ -968,6 +968,25 @@ impl Decoder { } } + /// `seek_to`, mais une cible au-delà de la dernière image se pose sur cette dernière image au + /// lieu de ne rien rendre. L'audio d'un enregistrement dure souvent un peu plus que sa vidéo + /// (de 12 ms à près d'une demi-seconde mesurés) : son clip finit alors après la dernière + /// image, et y entrer par la fin, en scrubant de droite à gauche, demandait une image qui + /// n'existe pas. Le changement de clip échouait sans bruit, et la vue gardait la scène du + /// clip qu'on quittait. + pub(crate) unsafe fn seek_to_or_last(&mut self, seconds: f64) -> Result<*mut AVFrame> { + let frame = self.seek_to(seconds)?; + if !frame.is_null() { + return Ok(frame); + } + // Le seek a décodé jusqu'à l'EOF : `cur_pts` est celui de la dernière image. L'EOF a vidé + // la frame courante, donc il faut le seek complet : `take` écarte le chemin rapide. + match self.cur_pts.take() { + Some(last) => self.seek_to(last as f64 * self.tb_sec()), + None => Ok(frame), + } + } + /// Déroule le décodeur en avant jusqu'à la première frame à `seconds` ou après, SANS /// jeter son état. Critère d'arrêt identique à celui du seek complet — c'est ce qui /// garantit que les deux chemins rendent exactement la même frame. @@ -1265,8 +1284,13 @@ pub struct ClipSource { pub source_end_sec: f64, pub webcam_offset_sec: f64, pub has_audio: bool, + /// Cameras 2-4 of the clip (index k-1 = camera k, an empty `path` = none). Only those a + /// layout region of the clip shows are decoded. + pub additional_cameras: Vec, } +pub use crate::extra_cameras::ClipCamera; + /// Export **multiclip** : rend la timeline (clips ordonnés, avec trims) en un seul MP4. /// Perf (contrainte §multiclip) : décodeurs ouverts une fois par source (cache) et réutilisés /// entre clips du même asset ; **un seul seek keyframe par frontière de clip** ; décodage @@ -1631,6 +1655,7 @@ unsafe fn run_multi_inner( // pour deux &mut indépendants). let mut screen_decs: HashMap = HashMap::new(); let mut webcam_decs: HashMap = HashMap::new(); + let mut extra_decs: HashMap = HashMap::new(); // fps de sortie : choix explicite de l'app si fourni, sinon dérivé du 1er clip (recordings // uniformes) — comportement historique. @@ -1711,6 +1736,7 @@ unsafe fn run_multi_inner( &scene, &mut screen_decs, &mut webcam_decs, + &mut extra_decs, &mut |frame_index| { // Backend CPU (WARP) : la frame composée descend en mémoire système via // `send_composited` (le compositeur relit son NV12 interne vers un AVFrame @@ -2836,6 +2862,29 @@ mod tests { ); } + #[test] + fn a_seek_past_the_last_frame_can_land_on_it() { + let Some(gpu) = strict_hardware_gpu("a_seek_past_the_last_frame_can_land_on_it") else { + return; + }; + let path = encode_color(&["-c:v", "libopenh264", "-b:v", "200k"], "last-frame.mp4"); + let mut dec = unsafe { Decoder::open(path.to_str().expect("utf8 path"), &gpu) } + .unwrap_or_else(|e| panic!("H.264 Decoder::open: {e:#}")); + + let frame = unsafe { dec.seek_to_or_last(10.0) }.expect("seek past the end"); + assert!(!frame.is_null(), "a target past the end must land on the last frame"); + assert!(!dec.cur_frame().is_null(), "the last frame must be presentable"); + assert!( + unsafe { dec.next() }.expect("decode after the last frame").is_null(), + "the frame it landed on must be the last one" + ); + assert!( + unsafe { dec.seek_to(10.0) }.expect("plain seek past the end").is_null(), + "seek_to itself still reports no frame past the end" + ); + println!("CORE_ASSERTIONS_COMPLETED:a_seek_past_the_last_frame_can_land_on_it"); + } + /// Playhead crossing clips is `Decoder::open` of the next source on the /// same `Gpu` (#554). H.264 must stay on D3D11VA after an AV1 software /// decoder has been opened and dropped. diff --git a/crates/compositor/src/prism_mesh.rs b/crates/compositor/src/prism_mesh.rs new file mode 100644 index 0000000000..c045462b04 --- /dev/null +++ b/crates/compositor/src/prism_mesh.rs @@ -0,0 +1,876 @@ +//! Les deux curseurs de Prism Glow en MAILLAGE : les facettes tracées sur l'art 2D +//! (`design/cursors/prism-glow/model`), taillées en cristal. Fichier GÉNÉRÉ par +//! `design/cursors/prism-glow/model/export_compositor.py` : ne pas l'éditer à la main. +//! +//! Repère du modèle du mode 15 : unité = côté du sprite 128 px du thème, origine au hotspot, +//! x à droite, y vers le bas, z vers la caméra, dessus du serti en z = 0. Chaque triangle du +//! cristal tient en quatre float4 : (v0, drapeaux), (e1, r), (e2, g), (normale, b) ; drapeaux = +//! paroi (1) + 2 × arêtes réelles (bit k : l'arête opposée au sommet k). Le fond plat, en +//! z = -épaisseur, est laissé au shader. `POLYS` tient les silhouettes (bord extérieur du +//! serti) puis les contours du cristal (son bord intérieur). `BOXES` range les triangles +//! par voisinage : deux float4 par boîte, (coin bas, premier triangle), (coin haut, nombre). + +/// Un des deux modèles : ses triangles et sa silhouette dans les tables, sa boîte de dessin. +pub struct PrismModel { + pub tri_start: usize, + pub tri_count: usize, + pub sil_start: usize, + pub sil_count: usize, + pub out_start: usize, + pub out_count: usize, + pub box_start: usize, + pub box_count: usize, + /// Largeur de la boîte, hauteur 1 (le sprite), en unités du modèle. + pub size: [f32; 2], + /// Hotspot, fraction de la boîte. + pub hotspot: [f32; 2], + /// Haut de la silhouette, fraction de la hauteur. + pub top: f32, + /// Sommet du cristal au-dessus de z = 0, et épaisseur sous z = 0. + pub height: f32, + pub thick: f32, +} + +/// Arrondi des arêtes du serti, en unités du modèle (`PRISM_BEVEL` des shaders). +pub const BEVEL: f32 = 0.012883; + +/// Flèche puis main, dans l'ordre des identifiants du shader. +pub const MODELS: [PrismModel; 2] = [ + PrismModel { + tri_start: 0, + tri_count: 31, + sil_start: 0, + sil_count: 25, + out_start: 67, + out_count: 9, + box_start: 0, + box_count: 4, + size: [0.63089, 1.0], + hotspot: [0.022658, 0.064647], + top: 0.0625, + height: 0.115951, + thick: 0.042945, + }, + PrismModel { + tri_start: 31, + tri_count: 130, + sil_start: 25, + sil_count: 42, + out_start: 76, + out_count: 34, + box_start: 4, + box_count: 18, + size: [0.797658, 1.0], + hotspot: [0.343388, 0.063582], + top: 0.0625, + height: 0.0822, + thick: 0.043263, + }, +]; + +pub const TRIS: [[f32; 4]; 644] = [ + [0.252301, 0.266258, -0.042945, 11.0], + [-0.221166, -0.203988, 0.0, 0.01033], + [-0.221166, -0.203988, 0.042945, 0.045182], + [0.677984, -0.735077, 0.0, 0.577584], + [0.252301, 0.266258, -0.042945, 3.0], + [-0.221166, -0.203988, 0.042945, 0.01033], + [0.0, 0.0, 0.042945, 0.045182], + [0.677984, -0.735077, 0.0, 0.577584], + [0.031135, 0.06227, -0.042945, 7.0], + [0.004294, 0.419785, 0.042945, 0.01033], + [0.0, 0.0, 0.042945, 0.045182], + [-0.999948, 0.01023, 0.0, 0.577584], + [0.169632, 0.356442, 0.115951, 14.0], + [0.082669, -0.090184, -0.115951, 0.590616], + [-0.138497, -0.294172, -0.115951, 0.737911], + [0.466385, -0.505659, 0.725806, 0.973449], + [0.035429, 0.482055, 0.0, 14.0], + [0.134202, -0.125614, 0.115951, 0.000305], + [-0.004294, -0.419785, 0.0, 0.938689], + [-0.650191, 0.006652, 0.759742, 0.854994], + [0.031135, 0.06227, -0.042945, 9.0], + [0.004294, 0.419785, 0.0, 0.01033], + [0.004294, 0.419785, 0.042945, 0.045182], + [-0.999948, 0.01023, 0.0, 0.577584], + [0.169632, 0.356442, 0.115951, 14.0], + [0.159969, 0.080521, -0.045092, 0.381323], + [0.082669, -0.090184, -0.115951, 0.056123], + [0.461407, -0.510214, 0.725799, 0.973449], + [0.541104, 0.544325, -0.042945, 9.0], + [-0.288804, -0.278068, 0.0, 0.01033], + [-0.288804, -0.278068, 0.042945, 0.045182], + [0.69359, -0.72037, 0.0, 0.577584], + [0.329601, 0.436963, 0.070859, 14.0], + [0.211503, 0.107362, -0.070859, 0.545724], + [-0.077301, -0.170706, -0.070859, 0.064805], + [0.49567, -0.514808, 0.699488, 0.973449], + [0.268405, 0.533589, 0.081595, 14.0], + [0.061196, -0.096626, -0.010736, 0.000916], + [-0.098773, -0.177147, 0.034356, 0.046662], + [0.247834, 0.049458, 0.967539, 0.904662], + [0.541104, 0.544325, -0.042945, 7.0], + [-0.288804, -0.278068, 0.042945, 0.01033], + [0.0, 0.0, 0.042945, 0.045182], + [0.69359, -0.72037, 0.0, 0.577584], + [0.199693, 0.595859, 0.0, 14.0], + [0.068712, -0.06227, 0.081595, 0.0], + [-0.030061, -0.239417, 0.115951, 0.01445], + [-0.504461, 0.426837, 0.750553, 0.254154], + [0.268405, 0.533589, 0.081595, 14.0], + [0.272699, 0.010736, -0.081595, 0.0], + [0.061196, -0.096626, -0.010736, 0.025193], + [0.283244, 0.073137, 0.956255, 0.527108], + [0.035429, 0.482055, 0.0, 14.0], + [0.0, 0.258742, 0.0, 0.0], + [0.134202, -0.125614, 0.115951, 0.473533], + [-0.653777, 0.0, 0.756687, 0.964691], + [0.359663, 0.559356, 0.0, 14.0], + [0.181442, -0.015031, 0.0, 0.006043], + [-0.091258, -0.025767, 0.081595, 0.076188], + [0.076465, 0.923048, 0.37701, 0.964691], + [0.035429, 0.482055, -0.042945, 3.0], + [0.0, 0.258742, 0.042945, 0.01033], + [0.0, 0.0, 0.042945, 0.045182], + [-1.0, 0.0, 0.0, 0.577584], + [0.035429, 0.482055, -0.042945, 11.0], + [0.0, 0.258742, 0.0, 0.01033], + [0.0, 0.258742, 0.042945, 0.045182], + [-1.0, 0.0, 0.0, 0.577584], + [0.035429, 0.740798, -0.042945, 7.0], + [0.164264, -0.144939, 0.042945, 0.01033], + [0.0, 0.0, 0.042945, 0.045182], + [0.661622, 0.749838, 0.0, 0.577584], + [0.035429, 0.740798, 0.0, 14.0], + [0.164264, -0.144939, 0.0, 0.0], + [0.134202, -0.384356, 0.115951, 0.010956], + [0.332572, 0.376915, 0.864483, 0.215854], + [0.035429, 0.740798, -0.042945, 11.0], + [0.164264, -0.144939, 0.0, 0.01033], + [0.164264, -0.144939, 0.042945, 0.045182], + [0.661622, 0.749838, 0.0, 0.577584], + [0.199693, 0.595859, -0.042945, 7.0], + [0.105215, 0.22546, 0.042945, 0.01033], + [0.0, 0.0, 0.042945, 0.045182], + [-0.906183, 0.422886, 0.0, 0.577584], + [0.199693, 0.595859, 0.0, 14.0], + [0.105215, 0.22546, 0.0, 0.0], + [0.068712, -0.06227, 0.081595, 0.015991], + [-0.613882, 0.286478, 0.735581, 0.473533], + [0.199693, 0.595859, -0.042945, 11.0], + [0.105215, 0.22546, 0.0, 0.01033], + [0.105215, 0.22546, 0.042945, 0.045182], + [-0.906183, 0.422886, 0.0, 0.577584], + [0.304908, 0.821319, 0.0, 14.0], + [0.128834, -0.067638, 0.0, 0.0], + [-0.036503, -0.28773, 0.081595, 0.254154], + [0.133686, 0.254641, 0.957751, 0.973449], + [0.304908, 0.821319, -0.042945, 7.0], + [0.128834, -0.067638, 0.042945, 0.01033], + [0.0, 0.0, 0.042945, 0.045182], + [0.464834, 0.885398, 0.0, 0.577584], + [0.433742, 0.753681, 0.0, 14.0], + [-0.07408, -0.194325, 0.0, 0.0], + [-0.165337, -0.220092, 0.081595, 0.964691], + [0.683362, -0.260508, 0.682021, 0.964691], + [0.433742, 0.753681, -0.042945, 11.0], + [-0.07408, -0.194325, 0.0, 0.01033], + [-0.07408, -0.194325, 0.042945, 0.045182], + [0.934406, -0.35621, 0.0, 0.577584], + [0.304908, 0.821319, -0.042945, 11.0], + [0.128834, -0.067638, 0.0, 0.01033], + [0.128834, -0.067638, 0.042945, 0.045182], + [0.464834, 0.885398, 0.0, 0.577584], + [0.433742, 0.753681, -0.042945, 7.0], + [-0.07408, -0.194325, 0.042945, 0.01033], + [0.0, 0.0, 0.042945, 0.045182], + [0.934406, -0.35621, 0.0, 0.577584], + [0.359663, 0.559356, -0.042945, 7.0], + [0.181442, -0.015031, 0.042945, 0.01033], + [0.0, 0.0, 0.042945, 0.045182], + [0.082557, 0.996586, 0.0, 0.577584], + [0.359663, 0.559356, -0.042945, 11.0], + [0.181442, -0.015031, 0.0, 0.01033], + [0.181442, -0.015031, 0.042945, 0.045182], + [0.082557, 0.996586, 0.0, 0.577584], + [0.044345, 0.043263, -0.043263, 11.0], + [-0.043263, -0.005408, 0.0, 0.01033], + [-0.043263, -0.005408, 0.043263, 0.045182], + [0.124035, -0.992278, 0.0, 0.577584], + [0.044345, 0.043263, -0.043263, 7.0], + [-0.043263, -0.005408, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.124035, -0.992278, 0.0, 0.577584], + [0.001082, 0.037855, -0.043263, 7.0], + [-0.044345, 0.040019, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [-0.669964, -0.742393, 0.0, 0.577584], + [0.044345, 0.043263, 0.0, 10.0], + [-0.043263, -0.005408, 0.0, 0.000305], + [-0.040019, 0.030284, 0.049753, 0.964691], + [0.101434, -0.811469, 0.575525, 0.973449], + [0.004326, 0.073548, 0.049753, 14.0], + [-0.003245, -0.035692, -0.049753, 0.0], + [-0.04759, 0.004326, -0.049753, 0.955978], + [-0.580475, -0.643229, 0.499304, 0.896269], + [0.001082, 0.037855, -0.043263, 11.0], + [-0.044345, 0.040019, 0.0, 0.01033], + [-0.044345, 0.040019, 0.043263, 0.045182], + [-0.669964, -0.742393, 0.0, 0.577584], + [-0.031366, 0.229295, 0.058405, 14.0], + [0.035692, -0.155748, -0.008653, 0.000305], + [-0.011897, -0.151422, -0.058405, 0.955978], + [-0.716533, -0.20131, 0.667874, 0.904662], + [-0.043263, 0.077874, -0.043263, 7.0], + [-0.001082, 0.154666, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [-0.999976, -0.006993, 0.0, 0.577584], + [-0.031366, 0.229295, 0.058405, 14.0], + [0.089771, -0.137361, -0.02055, 0.445197], + [0.035692, -0.155748, -0.008653, 0.973449], + [0.216369, -0.004654, 0.976301, 0.973449], + [-0.031366, 0.229295, 0.058405, 14.0], + [-0.011897, -0.151422, -0.058405, 0.000305], + [-0.012979, 0.003245, -0.058405, 0.955978], + [-0.976245, -0.006827, 0.216564, 0.904662], + [-0.043263, 0.077874, -0.043263, 9.0], + [-0.001082, 0.154666, 0.0, 0.01033], + [-0.001082, 0.154666, 0.043263, 0.045182], + [-0.999976, -0.006993, 0.0, 0.577584], + [-0.045426, 0.564586, 0.052998, 14.0], + [0.014061, -0.335291, 0.005408, 0.520989], + [0.001082, -0.332046, -0.052998, 0.088655], + [-0.975941, -0.037462, 0.214794, 0.973449], + [-0.044345, 0.23254, 0.0, 3.0], + [0.0, 0.0, -0.043263, 0.01033], + [-0.001082, 0.332046, -0.043263, 0.045182], + [-0.999995, -0.003257, 0.0, 0.577584], + [-0.163319, 0.380717, -0.043263, 7.0], + [-0.060569, 0.02704, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [-0.407651, -0.913138, 0.0, 0.577584], + [-0.094098, 0.431551, -0.043263, 11.0], + [-0.069221, -0.050834, 0.0, 0.01033], + [-0.069221, -0.050834, 0.043263, 0.045182], + [0.591909, -0.806004, 0.0, 0.577584], + [-0.163319, 0.380717, -0.043263, 11.0], + [-0.060569, 0.02704, 0.0, 0.01033], + [-0.060569, 0.02704, 0.043263, 0.045182], + [-0.407651, -0.913138, 0.0, 0.577584], + [0.075711, 0.08869, -0.043263, 11.0], + [-0.031366, -0.045426, 0.0, 0.01033], + [-0.031366, -0.045426, 0.043263, 0.045182], + [0.822897, -0.568191, 0.0, 0.577584], + [0.044345, 0.043263, 0.0, 6.0], + [-0.040019, 0.030284, 0.049753, 0.000305], + [0.014061, 0.048671, 0.037855, 0.964691], + [0.365607, -0.634949, 0.680566, 0.973449], + [0.075711, 0.08869, -0.043263, 7.0], + [-0.031366, -0.045426, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.822897, -0.568191, 0.0, 0.577584], + [0.058405, 0.091934, 0.037855, 14.0], + [0.017305, -0.003245, -0.037855, 0.0], + [-0.014061, -0.048671, -0.037855, 0.473533], + [0.757369, -0.522945, 0.39105, 0.904662], + [-0.031366, 0.229295, 0.058405, 14.0], + [0.107077, -0.140606, -0.058405, 0.0], + [0.089771, -0.137361, -0.02055, 0.376257], + [0.812052, 0.481346, 0.329966, 0.830777], + [0.077874, 0.407756, -0.043263, 11.0], + [-0.002163, -0.319067, 0.0, 0.01033], + [-0.002163, -0.319067, 0.043263, 0.045182], + [0.999977, -0.006779, 0.0, 0.577584], + [-0.031366, 0.229295, 0.058405, 14.0], + [0.10924, 0.178461, -0.058405, 0.0], + [0.107077, -0.140606, -0.058405, 0.376257], + [0.475583, -0.003224, 0.879665, 0.830777], + [0.147095, 0.272559, -0.043263, 7.0], + [-0.035692, 0.034611, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [-0.696146, -0.7179, 0.0, 0.577584], + [0.147095, 0.272559, -0.043263, 11.0], + [-0.035692, 0.034611, 0.0, 0.01033], + [-0.035692, 0.034611, 0.043263, 0.045182], + [-0.696146, -0.7179, 0.0, 0.577584], + [0.144932, 0.307169, 0.049753, 14.0], + [0.002163, -0.034611, -0.049753, 0.0], + [-0.033529, 0.0, -0.049753, 0.9131], + [-0.630237, -0.649931, 0.424725, 0.904662], + [0.183869, 0.317985, 0.037855, 6.0], + [-0.036774, -0.045426, -0.037855, 0.0], + [-0.038937, -0.010816, 0.011897, 0.806958], + [0.374433, -0.753477, 0.540438, 0.854994], + [0.077874, 0.407756, -0.043263, 7.0], + [-0.002163, -0.319067, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.999977, -0.006779, 0.0, 0.577584], + [0.111403, 0.307169, -0.043263, 7.0], + [-0.002163, 0.131953, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [-0.999866, -0.016391, 0.0, 0.577584], + [0.10924, 0.439122, 0.0, 6.0], + [0.035692, -0.131953, 0.049753, 0.18117], + [0.002163, -0.131953, 0.0, 0.955978], + [-0.82919, -0.013593, 0.558802, 0.964691], + [0.128708, 0.463999, 0.063813, 14.0], + [0.055161, -0.146014, -0.025958, 0.246204], + [0.016224, -0.156829, -0.014061, 0.964691], + [0.305397, -0.053644, 0.950713, 0.973449], + [0.111403, 0.307169, -0.043263, 11.0], + [-0.002163, 0.131953, 0.0, 0.01033], + [-0.002163, 0.131953, 0.043263, 0.045182], + [-0.999866, -0.016391, 0.0, 0.577584], + [0.189277, 0.28013, -0.043263, 11.0], + [-0.042182, -0.007571, 0.0, 0.01033], + [-0.042182, -0.007571, 0.043263, 0.045182], + [0.176664, -0.984271, 0.0, 0.577584], + [0.183869, 0.317985, 0.037855, 10.0], + [0.005408, -0.037855, -0.037855, 0.0], + [-0.036774, -0.045426, -0.037855, 0.806958], + [0.124328, -0.692684, 0.710445, 0.854994], + [0.189277, 0.28013, -0.043263, 7.0], + [-0.042182, -0.007571, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.176664, -0.984271, 0.0, 0.577584], + [0.183869, 0.317985, 0.037855, 14.0], + [0.02704, -0.004326, -0.037855, 0.0], + [0.005408, -0.037855, -0.037855, 0.514916], + [0.70062, -0.452013, 0.552101, 0.846876], + [0.210908, 0.313659, -0.043263, 11.0], + [-0.021632, -0.033529, 0.0, 0.01033], + [-0.021632, -0.033529, 0.043263, 0.045182], + [0.840297, -0.542127, 0.0, 0.577584], + [0.210908, 0.445612, 0.0, 10.0], + [0.0, -0.131953, 0.0, 0.0], + [-0.02704, -0.127627, 0.037855, 0.439658], + [0.813734, 0.0, 0.581238, 0.871366], + [0.210908, 0.313659, -0.043263, 7.0], + [-0.021632, -0.033529, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.840297, -0.542127, 0.0, 0.577584], + [0.210908, 0.445612, -0.043263, 11.0], + [0.0, -0.131953, 0.0, 0.01033], + [0.0, -0.131953, 0.043263, 0.045182], + [1.0, 0.0, 0.0, 0.577584], + [0.241193, 0.343943, -0.043263, 7.0], + [-0.002163, 0.107077, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [-0.999796, -0.020198, 0.0, 0.577584], + [0.276885, 0.311496, -0.043263, 11.0], + [-0.035692, 0.032447, 0.0, 0.01033], + [-0.035692, 0.032447, 0.043263, 0.045182], + [-0.672673, -0.73994, 0.0, 0.577584], + [0.23903, 0.45102, 0.0, 14.0], + [0.042182, -0.108158, 0.046508, 0.000305], + [0.002163, -0.107077, 0.0, 0.846876], + [-0.758098, -0.015315, 0.651961, 0.896269], + [0.276885, 0.311496, -0.043263, 7.0], + [-0.035692, 0.032447, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [-0.672673, -0.73994, 0.0, 0.577584], + [0.281211, 0.342862, 0.046508, 14.0], + [-0.004326, -0.031366, -0.046508, 0.0], + [-0.040019, 0.001082, -0.046508, 0.752941], + [-0.58651, -0.645161, 0.489667, 0.846876], + [0.246601, 0.459672, 0.06165, 14.0], + [0.070303, -0.101669, -0.024876, 0.262257], + [0.034611, -0.116811, -0.015142, 0.973449], + [0.27928, -0.041605, 0.959308, 0.973449], + [0.319067, 0.319067, -0.043263, 11.0], + [-0.042182, -0.007571, 0.0, 0.01033], + [-0.042182, -0.007571, 0.043263, 0.045182], + [0.176664, -0.984271, 0.0, 0.577584], + [0.319067, 0.319067, 0.0, 10.0], + [-0.042182, -0.007571, 0.0, 0.0], + [-0.037855, 0.023795, 0.046508, 0.896269], + [0.1483, -0.826245, 0.54344, 0.896269], + [0.319067, 0.319067, -0.043263, 7.0], + [-0.042182, -0.007571, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.176664, -0.984271, 0.0, 0.577584], + [0.319067, 0.319067, 0.0, 6.0], + [-0.037855, 0.023795, 0.046508, 0.0], + [-0.002163, 0.038937, 0.036774, 0.896269], + [0.437906, -0.604311, 0.665618, 0.896269], + [0.316904, 0.358004, 0.036774, 14.0], + [0.024876, -0.004326, -0.036774, 0.0], + [0.002163, -0.038937, -0.036774, 0.617212], + [0.707338, -0.464191, 0.533104, 0.921584], + [0.34178, 0.353677, -0.043263, 11.0], + [-0.022713, -0.034611, 0.0, 0.01033], + [-0.022713, -0.034611, 0.043263, 0.045182], + [0.836048, -0.548657, 0.0, 0.577584], + [0.340698, 0.474815, 0.0, 14.0], + [0.001082, -0.121137, 0.0, 0.0], + [-0.023795, -0.116811, 0.036774, 0.473533], + [0.828664, 0.007399, 0.559697, 0.930114], + [0.34178, 0.353677, -0.043263, 7.0], + [-0.022713, -0.034611, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.836048, -0.548657, 0.0, 0.577584], + [0.340698, 0.474815, -0.043263, 11.0], + [0.001082, -0.121137, 0.0, 0.01033], + [0.001082, -0.121137, 0.043263, 0.045182], + [0.99996, 0.008928, 0.0, 0.577584], + [0.411001, 0.351514, -0.043263, 11.0], + [-0.034611, 0.030284, 0.0, 0.01033], + [-0.034611, 0.030284, 0.043263, 0.045182], + [-0.658505, -0.752577, 0.0, 0.577584], + [0.411001, 0.351514, -0.043263, 7.0], + [-0.034611, 0.030284, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [-0.658505, -0.752577, 0.0, 0.577584], + [0.412083, 0.38288, 0.043263, 14.0], + [-0.001082, -0.031366, -0.043263, 0.0], + [-0.035692, -0.001082, -0.043263, 0.745403], + [-0.574039, -0.656045, 0.489984, 0.854994], + [0.45102, 0.359085, -0.043263, 11.0], + [-0.040019, -0.007571, 0.0, 0.01033], + [-0.040019, -0.007571, 0.043263, 0.045182], + [0.185892, -0.98257, 0.0, 0.577584], + [0.45102, 0.359085, 0.0, 10.0], + [-0.040019, -0.007571, 0.0, 0.000305], + [-0.038937, 0.023795, 0.043263, 0.879622], + [0.151736, -0.802035, 0.577682, 0.9131], + [0.45102, 0.359085, -0.043263, 7.0], + [-0.040019, -0.007571, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.185892, -0.98257, 0.0, 0.577584], + [0.45102, 0.359085, 0.0, 6.0], + [-0.038937, 0.023795, 0.043263, 0.000305], + [0.003245, 0.0411, 0.032447, 0.879622], + [0.417852, -0.583049, 0.696744, 0.9131], + [0.473733, 0.391533, -0.043263, 11.0], + [-0.022713, -0.032447, 0.0, 0.01033], + [-0.022713, -0.032447, 0.043263, 0.045182], + [0.819232, -0.573462, 0.0, 0.577584], + [0.454264, 0.400185, 0.032447, 14.0], + [0.019468, -0.008653, -0.032447, 0.0], + [-0.003245, -0.0411, -0.032447, 0.527108], + [0.688617, -0.482032, 0.541712, 0.887922], + [0.473733, 0.391533, -0.043263, 7.0], + [-0.022713, -0.032447, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.819232, -0.573462, 0.0, 0.577584], + [-0.223888, 0.407756, -0.043263, 7.0], + [0.002163, 0.06814, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [-0.999496, 0.03173, 0.0, 0.577584], + [-0.223888, 0.407756, -0.043263, 11.0], + [0.002163, 0.06814, 0.0, 0.01033], + [0.002163, 0.06814, 0.043263, 0.045182], + [-0.999496, 0.03173, 0.0, 0.577584], + [-0.167645, 0.416409, 0.046508, 14.0], + [-0.056242, -0.008653, -0.046508, 0.000305], + [-0.054079, 0.059487, -0.046508, 0.955978], + [-0.638985, 0.020285, 0.768952, 0.973449], + [-0.167645, 0.416409, 0.046508, 14.0], + [0.004326, -0.035692, -0.046508, 0.000305], + [-0.056242, -0.008653, -0.046508, 0.955978], + [-0.339782, -0.761112, 0.552501, 0.973449], + [-0.221724, 0.475896, 0.0, 14.0], + [0.139524, 0.055161, 0.043263, 0.520989], + [0.054079, -0.059487, 0.046508, 0.973449], + [-0.39307, 0.317373, 0.863001, 0.973449], + [-0.167645, 0.416409, 0.046508, 6.0], + [0.073548, 0.015142, -0.046508, 0.0], + [0.004326, -0.035692, -0.046508, 0.768154], + [0.490917, -0.668482, 0.558688, 0.871366], + [-0.094098, 0.431551, -0.043263, 7.0], + [-0.069221, -0.050834, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.591909, -0.806004, 0.0, 0.577584], + [-0.167645, 0.416409, 0.046508, 10.0], + [0.085445, 0.114648, -0.003245, 0.0], + [0.073548, 0.015142, -0.046508, 0.768154], + [0.548356, -0.38771, 0.740936, 0.871366], + [-0.0822, 0.531057, -0.043263, 11.0], + [-0.011897, -0.099506, 0.0, 0.01033], + [-0.011897, -0.099506, 0.043263, 0.045182], + [0.992928, -0.11872, 0.0, 0.577584], + [-0.044345, 0.23254, 0.0, 7.0], + [-0.001082, 0.332046, -0.043263, 0.01033], + [-0.001082, 0.332046, 0.052998, 0.045182], + [-0.999995, -0.003257, 0.0, 0.577584], + [-0.045426, 0.564586, 0.052998, 14.0], + [0.1233, -0.156829, -0.052998, 0.520989], + [0.014061, -0.335291, 0.005408, 0.088655], + [0.429355, 0.032562, 0.902549, 0.973449], + [0.10924, 0.439122, 0.0, 14.0], + [-0.031366, -0.031366, 0.0, 0.132876], + [-0.154666, 0.125464, 0.052998, 0.028428], + [0.182761, -0.182761, 0.966021, 0.955978], + [0.10924, 0.439122, -0.043263, 11.0], + [-0.031366, -0.031366, 0.0, 0.01033], + [-0.031366, -0.031366, 0.043263, 0.045182], + [0.707107, -0.707107, 0.0, 0.577584], + [0.10924, 0.439122, -0.043263, 7.0], + [-0.031366, -0.031366, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.707107, -0.707107, 0.0, 0.577584], + [0.10924, 0.439122, 0.0, 10.0], + [0.019468, 0.024876, 0.063813, 0.18117], + [0.035692, -0.131953, 0.049753, 0.955978], + [-0.933936, -0.126584, 0.334276, 0.964691], + [0.128708, 0.463999, 0.063813, 14.0], + [0.050834, 0.002163, -0.005408, 0.0], + [0.0822, -0.018387, -0.063813, 0.439658], + [0.07859, -0.92643, 0.368172, 0.871366], + [-0.221724, 0.475896, -0.043263, 7.0], + [0.122219, 0.207664, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [-0.861819, 0.507216, 0.0, 0.577584], + [-0.221724, 0.475896, -0.043263, 9.0], + [0.122219, 0.207664, 0.0, 0.01033], + [0.122219, 0.207664, 0.043263, 0.045182], + [-0.861819, 0.507216, 0.0, 0.577584], + [-0.099506, 0.68356, 0.0, 14.0], + [0.017305, -0.152503, 0.043263, 0.0], + [-0.122219, -0.207664, 0.0, 0.723049], + [-0.365879, 0.215335, 0.905408, 0.854994], + [-0.0822, 0.531057, -0.043263, 7.0], + [-0.011897, -0.099506, 0.043263, 0.01033], + [0.0, 0.0, 0.086527, 0.045182], + [0.992928, -0.11872, 0.0, 0.577584], + [-0.099506, 0.68356, 0.0, 14.0], + [0.054079, -0.118974, 0.052998, 0.0], + [0.017305, -0.152503, 0.043263, 0.514916], + [-0.41958, 0.203352, 0.884647, 0.863157], + [-0.099506, 0.68356, -0.043263, 3.0], + [0.086527, 0.149258, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [-0.86514, 0.501531, 0.0, 0.577584], + [-0.045426, 0.564586, -0.043263, 11.0], + [-0.036774, -0.033529, 0.0, 0.01033], + [-0.036774, -0.033529, 0.086527, 0.045182], + [0.673754, -0.738956, 0.0, 0.577584], + [-0.045426, 0.564586, -0.043263, 7.0], + [-0.036774, -0.033529, 0.086527, 0.01033], + [0.0, 0.0, 0.096261, 0.045182], + [0.673754, -0.738956, 0.0, 0.577584], + [-0.099506, 0.68356, 0.0, 14.0], + [0.086527, 0.149258, 0.0, 0.0], + [0.054079, -0.118974, 0.052998, 0.262257], + [-0.385563, 0.223515, 0.8952, 0.9131], + [-0.099506, 0.68356, -0.043263, 11.0], + [0.086527, 0.149258, 0.0, 0.01033], + [0.086527, 0.149258, 0.043263, 0.045182], + [-0.86514, 0.501531, 0.0, 0.577584], + [-0.012979, 0.832818, 0.0, 14.0], + [0.214153, -0.194685, 0.0822, 0.000305], + [-0.032447, -0.268232, 0.052998, 0.930114], + [-0.176863, 0.211325, 0.961281, 0.973449], + [0.128708, 0.463999, 0.063813, 14.0], + [-0.019468, -0.024876, -0.063813, 0.168276], + [-0.174135, 0.100587, -0.010816, 0.038209], + [-0.469238, -0.764881, 0.441331, 0.955978], + [0.201174, 0.638133, 0.0822, 14.0], + [-0.072466, -0.174135, -0.018387, 0.18117], + [-0.246601, -0.073548, -0.029203, 0.082277], + [-0.098561, -0.063844, 0.993081, 0.973449], + [-0.012979, 0.832818, -0.043263, 7.0], + [0.339617, -0.002163, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.006369, 0.99998, 0.0, 0.577584], + [0.179543, 0.466162, 0.058405, 10.0], + [-0.050834, -0.002163, 0.005408, 0.000305], + [0.021632, 0.171972, 0.023795, 0.022171], + [0.110896, -0.149888, 0.982464, 0.381323], + [-0.012979, 0.832818, 0.0, 14.0], + [0.339617, -0.002163, 0.0, 0.0], + [0.214153, -0.194685, 0.0822, 0.027314], + [0.002492, 0.391296, 0.920261, 0.49102], + [0.183869, 0.317985, 0.037855, 10.0], + [-0.055161, 0.146014, 0.025958, 0.0], + [0.02704, 0.127627, -0.037855, 0.439658], + [0.623841, 0.097823, 0.775405, 0.871366], + [0.210908, 0.445612, -0.043263, 7.0], + [0.0, -0.131953, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [1.0, 0.0, 0.0, 0.577584], + [0.210908, 0.445612, 0.0, 10.0], + [-0.031366, 0.02055, 0.058405, 0.000305], + [0.035692, 0.014061, 0.06165, 0.022171], + [-0.105864, -0.954447, 0.278971, 0.381323], + [0.23903, 0.45102, -0.043263, 11.0], + [-0.028121, -0.005408, 0.0, 0.01033], + [-0.028121, -0.005408, 0.043263, 0.045182], + [0.188847, -0.982006, 0.0, 0.577584], + [0.23903, 0.45102, -0.043263, 7.0], + [-0.028121, -0.005408, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.188847, -0.982006, 0.0, 0.577584], + [0.246601, 0.459672, 0.06165, 11.0], + [-0.007571, -0.008653, -0.06165, 0.000305], + [-0.035692, -0.014061, -0.06165, 0.022171], + [0.187618, -0.975617, 0.113888, 0.381323], + [0.241193, 0.343943, -0.043263, 11.0], + [-0.002163, 0.107077, 0.0, 0.01033], + [-0.002163, 0.107077, 0.043263, 0.045182], + [-0.999796, -0.020198, 0.0, 0.577584], + [0.23903, 0.45102, 0.0, 15.0], + [0.007571, 0.008653, 0.06165, 0.000305], + [0.042182, -0.108158, 0.046508, 0.846876], + [-0.94107, -0.299262, 0.157572, 0.896269], + [0.316904, 0.358004, 0.036774, 14.0], + [-0.070303, 0.101669, 0.024876, 0.0], + [0.023795, 0.116811, -0.036774, 0.473533], + [0.523424, 0.157027, 0.837478, 0.930114], + [0.340698, 0.474815, -0.043263, 7.0], + [0.001082, -0.121137, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.99996, 0.008928, 0.0, 0.577584], + [0.376391, 0.381799, -0.043263, 11.0], + [-0.001082, 0.107077, 0.0, 0.01033], + [-0.001082, 0.107077, 0.043263, 0.045182], + [-0.999949, -0.0101, 0.0, 0.577584], + [0.376391, 0.381799, -0.043263, 7.0], + [-0.001082, 0.107077, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [-0.999949, -0.0101, 0.0, 0.577584], + [0.375309, 0.488875, 0.0, 14.0], + [0.036774, -0.105995, 0.043263, 0.000305], + [0.001082, -0.107077, 0.0, 0.879622], + [-0.771254, -0.00779, 0.63648, 0.896269], + [0.375309, 0.488875, 0.0, 14.0], + [0.078955, -0.08869, 0.032447, 0.223224], + [0.036774, -0.105995, 0.043263, 0.964691], + [0.071225, 0.39802, 0.914608, 0.973449], + [0.473733, 0.603523, 0.0, 14.0], + [0.0, -0.21199, 0.0, 0.158953], + [-0.019468, -0.203337, 0.032447, 0.030716], + [0.857493, 0.0, 0.514496, 0.964691], + [0.473733, 0.603523, -0.043263, 11.0], + [0.0, -0.21199, 0.0, 0.01033], + [0.0, -0.21199, 0.043263, 0.045182], + [1.0, 0.0, 0.0, 0.577584], + [0.276885, 0.484549, 0.058405, 10.0], + [-0.030284, -0.024876, 0.003245, 0.000305], + [-0.097342, -0.018387, 0.0, 0.022171], + [-0.03153, 0.166923, 0.985466, 0.381323], + [0.246601, 0.459672, 0.06165, 14.0], + [0.030284, 0.024876, -0.003245, 0.0], + [0.094098, 0.015142, -0.06165, 0.473533], + [0.518869, -0.545855, 0.657888, 0.930114], + [0.375309, 0.488875, -0.043263, 11.0], + [-0.034611, -0.014061, 0.0, 0.01033], + [-0.034611, -0.014061, 0.043263, 0.045182], + [0.376377, -0.926467, 0.0, 0.577584], + [0.340698, 0.474815, 0.0, 14.0], + [-0.063813, 0.009734, 0.058405, 0.000305], + [0.034611, 0.014061, 0.0, 0.022171], + [0.3276, -0.806402, 0.492334, 0.381323], + [0.375309, 0.488875, -0.043263, 7.0], + [-0.034611, -0.014061, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.376377, -0.926467, 0.0, 0.577584], + [0.454264, 0.400185, 0.032447, 12.0], + [-0.078955, 0.08869, -0.032447, 0.158953], + [0.019468, 0.203337, -0.032447, 0.030716], + [-0.201684, 0.173144, 0.964025, 0.964691], + [0.201174, 0.638133, 0.0822, 8.0], + [0.075711, -0.153585, -0.023795, 0.000305], + [-0.021632, -0.171972, -0.023795, 0.022171], + [0.026497, -0.140281, 0.989757, 0.381323], + [0.473733, 0.603523, -0.043263, 7.0], + [0.0, -0.21199, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [1.0, 0.0, 0.0, 0.577584], + [0.276885, 0.484549, 0.058405, 14.0], + [-0.075711, 0.153585, 0.023795, 0.000305], + [0.098424, 0.004326, -0.058405, 0.022171], + [0.50316, 0.115345, 0.856461, 0.381323], + [0.375309, 0.488875, 0.0, 10.0], + [0.090853, 0.12979, 0.0, 0.158953], + [0.098424, 0.114648, 0.0, 0.030716], + [0.0, 0.0, 1.0, 0.964691], + [0.466162, 0.618665, -0.043263, 11.0], + [0.007571, -0.015142, 0.0, 0.01033], + [0.007571, -0.015142, 0.043263, 0.045182], + [0.894427, 0.447214, 0.0, 0.577584], + [0.466162, 0.618665, -0.043263, 3.0], + [0.007571, -0.015142, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.894427, 0.447214, 0.0, 0.577584], + [0.326638, 0.830655, 0.0, 14.0], + [0.139524, -0.21199, 0.0, 0.0], + [0.048671, -0.34178, 0.0, 0.015213], + [0.0, 0.0, 1.0, 0.428687], + [0.326638, 0.830655, 0.0, 14.0], + [0.048671, -0.34178, 0.0, 0.007492], + [-0.125464, -0.192522, 0.0822, 0.054475], + [0.472493, 0.067285, 0.878762, 0.964691], + [0.326638, 0.830655, -0.043263, 9.0], + [0.139524, -0.21199, 0.0, 0.01033], + [0.139524, -0.21199, 0.043263, 0.045182], + [0.835314, 0.549773, 0.0, 0.577584], + [0.326638, 0.830655, -0.043263, 7.0], + [0.139524, -0.21199, 0.043263, 0.01033], + [0.0, 0.0, 0.043263, 0.045182], + [0.835314, 0.549773, 0.0, 0.577584], + [-0.012979, 0.832818, -0.043263, 11.0], + [0.339617, -0.002163, 0.0, 0.01033], + [0.339617, -0.002163, 0.043263, 0.045182], + [0.006369, 0.99998, 0.0, 0.577584], +]; + +pub const POLYS: [[f32; 2]; 110] = [ + [0.006442, -0.002147], + [-0.001074, 0.001074], + [-0.004294, 0.007515], + [-0.004294, 0.806288], + [-0.001074, 0.809509], + [0.004294, 0.809509], + [0.022546, 0.796626], + [0.185736, 0.655982], + [0.281288, 0.86319], + [0.286656, 0.870706], + [0.293098, 0.871779], + [0.465951, 0.781595], + [0.478834, 0.773006], + [0.478834, 0.763344], + [0.407975, 0.592638], + [0.45092, 0.586196], + [0.601227, 0.573313], + [0.606595, 0.569018], + [0.606595, 0.564724], + [0.601227, 0.556135], + [0.50138, 0.453067], + [0.425153, 0.37684], + [0.289877, 0.246933], + [0.081595, 0.057975], + [0.013957, 0.0], + [-0.011897, 0.0], + [-0.055161, 0.037855], + [-0.081119, 0.063813], + [-0.083282, 0.06814], + [-0.084363, 0.38288], + [-0.089771, 0.38288], + [-0.158993, 0.33529], + [-0.166564, 0.336372], + [-0.250927, 0.379635], + [-0.261743, 0.388288], + [-0.263906, 0.461836], + [-0.262824, 0.486712], + [-0.173053, 0.638133], + [-0.044345, 0.860939], + [-0.036774, 0.867429], + [-0.024876, 0.871755], + [0.33529, 0.871755], + [0.349351, 0.867429], + [0.363412, 0.853368], + [0.513752, 0.619747], + [0.513752, 0.379635], + [0.466162, 0.322311], + [0.461836, 0.320148], + [0.399104, 0.311496], + [0.393696, 0.313659], + [0.374227, 0.332046], + [0.343943, 0.288782], + [0.336372, 0.282293], + [0.269314, 0.271477], + [0.263906, 0.27364], + [0.243356, 0.292027], + [0.21199, 0.244438], + [0.207664, 0.241193], + [0.149258, 0.231459], + [0.138443, 0.231459], + [0.118974, 0.248764], + [0.116811, 0.222806], + [0.117892, 0.162237], + [0.115729, 0.076792], + [0.06814, 0.011897], + [0.062732, 0.007571], + [0.005408, 0.0], + [0.031135, 0.06227], + [0.252301, 0.266258], + [0.541104, 0.544325], + [0.359663, 0.559356], + [0.433742, 0.753681], + [0.304908, 0.821319], + [0.199693, 0.595859], + [0.035429, 0.740798], + [0.035429, 0.482055], + [-0.043263, 0.077874], + [0.001082, 0.037855], + [0.044345, 0.043263], + [0.075711, 0.08869], + [0.077874, 0.407756], + [0.10924, 0.439122], + [0.111403, 0.307169], + [0.147095, 0.272559], + [0.189277, 0.28013], + [0.210909, 0.313659], + [0.210909, 0.445612], + [0.23903, 0.45102], + [0.241193, 0.343943], + [0.276885, 0.311496], + [0.319067, 0.319067], + [0.34178, 0.353677], + [0.340698, 0.474815], + [0.375309, 0.488875], + [0.376391, 0.381799], + [0.411001, 0.351514], + [0.45102, 0.359085], + [0.473733, 0.391533], + [0.473733, 0.603523], + [0.466162, 0.618665], + [0.326638, 0.830655], + [-0.012979, 0.832818], + [-0.099506, 0.68356], + [-0.221724, 0.475896], + [-0.223888, 0.407756], + [-0.163319, 0.380717], + [-0.094098, 0.431551], + [-0.0822, 0.531057], + [-0.045426, 0.564586], + [-0.044345, 0.23254], +]; + +pub const BOXES: [[f32; 4]; 44] = [ + [0.030135, 0.06127, -0.043945, 0.0], + [0.330601, 0.483055, 0.116951, 7.0], + [0.034429, 0.265258, -0.043945, 7.0], + [0.542104, 0.741798, 0.116951, 8.0], + [0.034429, 0.355442, -0.043945, 15.0], + [0.305908, 0.822319, 0.116951, 8.0], + [0.267405, 0.532589, -0.043945, 23.0], + [0.542104, 0.822319, 0.082595, 8.0], + [-0.045345, 0.036855, -0.044263, 31.0], + [0.045345, 0.23354, 0.059405, 8.0], + [-0.224888, 0.072548, -0.044263, 39.0], + [0.059405, 0.565586, 0.059405, 8.0], + [-0.032366, 0.042263, -0.044263, 47.0], + [0.148095, 0.408756, 0.059405, 8.0], + [0.074711, 0.08769, -0.044263, 55.0], + [0.184869, 0.464999, 0.064813, 8.0], + [0.146095, 0.271559, -0.044263, 63.0], + [0.211908, 0.446612, 0.038855, 8.0], + [0.23803, 0.310496, -0.044263, 71.0], + [0.320067, 0.460672, 0.06265, 8.0], + [0.275885, 0.310496, -0.044263, 79.0], + [0.412001, 0.475815, 0.047508, 8.0], + [0.375391, 0.350514, -0.044263, 87.0], + [0.45202, 0.38388, 0.044263, 4.0], + [0.410001, 0.350514, -0.044263, 91.0], + [0.474733, 0.401185, 0.044263, 5.0], + [-0.224888, 0.379717, -0.044263, 96.0], + [-0.0812, 0.532057, 0.047508, 8.0], + [-0.095098, 0.228295, -0.044263, 104.0], + [0.211908, 0.565586, 0.064813, 8.0], + [-0.222724, 0.430551, -0.044263, 112.0], + [-0.011979, 0.833818, 0.053998, 8.0], + [-0.100506, 0.438122, -0.044263, 120.0], + [0.327638, 0.833818, 0.0832, 8.0], + [0.127708, 0.312659, -0.044263, 128.0], + [0.282211, 0.467162, 0.064813, 8.0], + [0.245601, 0.352677, -0.044263, 136.0], + [0.474733, 0.604523, 0.06265, 8.0], + [0.178543, 0.390533, -0.044263, 144.0], + [0.474733, 0.639133, 0.0832, 8.0], + [0.200174, 0.483549, -0.044263, 152.0], + [0.474733, 0.639133, 0.0832, 4.0], + [-0.013979, 0.487875, -0.044263, 156.0], + [0.467162, 0.833818, 0.0832, 5.0], +]; diff --git a/crates/compositor/src/regions.rs b/crates/compositor/src/regions.rs index 097863e299..971525b531 100644 --- a/crates/compositor/src/regions.rs +++ b/crates/compositor/src/regions.rs @@ -8,7 +8,9 @@ use crate::camera::Follow; use crate::cursor::CursorTrack; -use crate::scene::{SceneCameraFullscreenRegion, SceneSpeedRegion, SceneZoomRegion}; +use crate::scene::{ + SceneCameraFullscreenRegion, SceneCameraLayoutRegion, SceneSpeedRegion, SceneZoomRegion, +}; /// Quantification commune vidéo/audio : le web retranche exactement 1 ms avant `ceil`. pub const SPEED_FRAME_EPSILON_SEC: f64 = 0.001; @@ -49,18 +51,22 @@ pub fn speed_segments_for_window( for region in overlapping { let start = region.start_sec.max(source_start_sec).max(cursor); let end = region.end_sec.min(source_end_sec); + // Projection can leave a zero-width region at a trim boundary. It may still pass + // the overlap filter when rounding puts its start just inside the clip window. Skip + // it before emitting the 1x gap, or the unchanged cursor makes that gap render twice. + if end <= start { + continue; + } if start > cursor { push_speed_segment(&mut spans, cursor, start, 1.0, fps); } - if end > start { - let speed = if region.speed.is_finite() && region.speed > 0.0 { - region.speed - } else { - 1.0 - }; - push_speed_segment(&mut spans, start, end, speed, fps); - cursor = end; - } + let speed = if region.speed.is_finite() && region.speed > 0.0 { + region.speed + } else { + 1.0 + }; + push_speed_segment(&mut spans, start, end, speed, fps); + cursor = end; } if cursor < source_end_sec { push_speed_segment(&mut spans, cursor, source_end_sec, 1.0, fps); @@ -267,8 +273,11 @@ fn push_speed_segment( } // mêmes fenêtres de transition que le web (TRANSITION_WINDOW_MS etc., converties en secondes). -const TRANSITION_WINDOW_S: f32 = 1.01505; -const FULLSCREEN_LEAD_OUT_WINDOW_S: f32 = TRANSITION_WINDOW_S * 1.5; +pub(crate) const TRANSITION_WINDOW_S: f32 = 1.01505; +pub(crate) const FULLSCREEN_LEAD_OUT_WINDOW_S: f32 = TRANSITION_WINDOW_S * 1.5; +/// Two camera regions (layout or Full Camera) closer than this (source seconds) meet at a +/// seam: they hand over directly, without the default PiP in between. +pub(crate) const REGION_SEAM_S: f64 = 0.001; // Durée d'un zoom à l'écran selon l'échelle visée, cf. `zoom_transition_s` ; miroir de // `ZOOM_TRANSITION_BASE_MS` / `ZOOM_TRANSITION_PER_LN_MS` (TS). const ZOOM_TRANSITION_BASE_S: f32 = 0.6; @@ -324,7 +333,7 @@ fn cubic_bezier(x1: f32, y1: f32, x2: f32, y2: f32, t: f32) -> f32 { } /// Port de `easeOutScreenStudio` (TS) : cubic-bezier(0.16, 1, 0.3, 1). -fn ease_out_screen_studio(t: f32) -> f32 { +pub(crate) fn ease_out_screen_studio(t: f32) -> f32 { cubic_bezier(0.16, 1.0, 0.3, 1.0, t) } @@ -462,6 +471,10 @@ pub fn zoom_cursor_alpha(regions: &[SceneZoomRegion], t: f32, clock: &ScreenCloc /// pixel shader par `compositor.rs`, ce module ne fait que le calcul temporel). pub struct ZoomState { pub scale: f32, + /// Force de la région (0..1) : plein régime sur son span et sur les transitions/paliers + /// chaînés, 0 hors zoom. C'est l'enveloppe du zoom — le rétrécissement réactif de la webcam + /// (`reactive_scale`) s'y accroche pour animer sa taille, au lieu de suivre la profondeur. + pub strength: f32, pub focus: [f32; 2], pub rotation: [f32; 3], /// À quel point un préset 3D est installé (0..1) : la force de la région quand elle en porte @@ -479,6 +492,7 @@ pub struct ZoomState { const IDENTITY_ZOOM: ZoomState = ZoomState { scale: 1.0, + strength: 0.0, focus: [0.5, 0.5], rotation: [0.0, 0.0, 0.0], tilt: 0.0, @@ -865,6 +879,7 @@ pub fn zoom_state_in( |p: [f32; 2], q: [f32; 2]| [lerp(p[0], q[0], progress), lerp(p[1], q[1], progress)]; return ZoomState { scale: scale_lerp(cur.scale, next.scale, progress), + strength: 1.0, focus: [ lerp(cur_focus[0], next_focus[0], progress), lerp(cur_focus[1], next_focus[1], progress), @@ -886,6 +901,7 @@ pub fn zoom_state_in( let seen = orbit_at(next, t, frame); return ZoomState { scale: next.scale, + strength: 1.0, focus: resolve_focus(next, t, cursor, clock), rotation: fixed_rotation(next), tilt: fixed_flag(next), @@ -939,6 +955,7 @@ pub fn zoom_state_in( let seen = orbit_at(r, t, frame); ZoomState { scale, + strength, focus: [ease(focus[0]), ease(focus[1])], rotation: lerp_rotation3d([0.0, 0.0, 0.0], fixed_rotation(r), strength), tilt: fixed_flag(r) * strength, @@ -961,16 +978,20 @@ fn camera_fullscreen_region_phase( region: &SceneCameraFullscreenRegion, t: f32, clock: &ScreenClock, + layouts: &[SceneCameraLayoutRegion], ) -> f32 { let start = region.start_sec as f32; let end = region.end_sec as f32; if t <= start || t >= end { return 0.0; } + // At a seam with a layout region the camera is already frame-filling on that side: the + // layout region's own glide (`camera_layers_at`) moves it, so the phase holds 1 there. + let seams = full_camera_seams(region, layouts); let (start, end, t) = (clock.at(start), clock.at(end), clock.at(t)); let half = (end - start) * 0.5; - let lead_in = TRANSITION_WINDOW_S.min(half); - let lead_out = FULLSCREEN_LEAD_OUT_WINDOW_S.min(half); + let lead_in = if seams.start { 0.0 } else { TRANSITION_WINDOW_S.min(half) }; + let lead_out = if seams.end { 0.0 } else { FULLSCREEN_LEAD_OUT_WINDOW_S.min(half) }; let lead_in_end = start + lead_in; let lead_out_start = end - lead_out; if t < lead_in_end { @@ -990,16 +1011,38 @@ fn camera_fullscreen_region_phase( } } +/// Which ends of a Full Camera region meet a (non-empty) camera layout region at a seam +/// (`REGION_SEAM_S`): `start` = a layout region ends where it starts, `end` = one starts where +/// it ends. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub(crate) struct FullCameraSeams { + pub start: bool, + pub end: bool, +} + +pub(crate) fn full_camera_seams( + region: &SceneCameraFullscreenRegion, + layouts: &[SceneCameraLayoutRegion], +) -> FullCameraSeams { + let mut seams = FullCameraSeams::default(); + for l in layouts.iter().filter(|l| l.end_sec > l.start_sec) { + seams.start |= (l.end_sec - region.start_sec).abs() <= REGION_SEAM_S; + seams.end |= (l.start_sec - region.end_sec).abs() <= REGION_SEAM_S; + } + seams +} + /// Phase Full Camera (0..1) au temps `t`. Régions superposées (ne devrait pas arriver, gardé /// défensif comme le web) → la plus forte gagne. fn camera_fullscreen_phase_at( regions: &[SceneCameraFullscreenRegion], t: f32, clock: &ScreenClock, + layouts: &[SceneCameraLayoutRegion], ) -> f32 { let mut strongest = 0.0f32; for r in regions { - let s = camera_fullscreen_region_phase(r, t, clock); + let s = camera_fullscreen_region_phase(r, t, clock, layouts); if s > strongest { strongest = s; } @@ -1012,8 +1055,9 @@ pub fn camera_fullscreen_progress_at( regions: &[SceneCameraFullscreenRegion], t: f32, clock: &ScreenClock, + layouts: &[SceneCameraLayoutRegion], ) -> f32 { - ease_out_screen_studio(camera_fullscreen_phase_at(regions, t, clock)) + ease_out_screen_studio(camera_fullscreen_phase_at(regions, t, clock, layouts)) } /// Ce qui reste de la bulle PiP (coins arrondis, ombre) au temps `t` : 1 = bulle entière, @@ -1026,8 +1070,72 @@ pub fn camera_fullscreen_shape_at( regions: &[SceneCameraFullscreenRegion], t: f32, clock: &ScreenClock, + layouts: &[SceneCameraLayoutRegion], +) -> f32 { + 1.0 - smoothstep(0.5, 1.0, camera_fullscreen_phase_at(regions, t, clock, layouts)) +} + +/// The Full Camera region in effect at `t`: the one with the strongest phase, so a region's +/// orientation and its grow/shrink always refer to the same region. `None` wherever the +/// envelope is 0 — outside every region and on their bounds. +pub fn camera_fullscreen_region_at<'a>( + regions: &'a [SceneCameraFullscreenRegion], + t: f32, + clock: &ScreenClock, + layouts: &[SceneCameraLayoutRegion], +) -> Option<&'a SceneCameraFullscreenRegion> { + let mut best: Option<(&'a SceneCameraFullscreenRegion, f32)> = None; + for r in regions { + let phase = camera_fullscreen_region_phase(r, t, clock, layouts); + if phase > 0.0 && best.map_or(true, |(_, b)| phase > b) { + best = Some((r, phase)); + } + } + best.map(|(r, _)| r) +} + +/// How long a turned section's camera takes to sharpen after the hold, and to blur again +/// before the shrink. See `camera_fullscreen_cover_at`. +pub const DESK_COVER_FADE_S: f32 = 0.5; + +/// Cover strength of the webcam at `t` (0 = sharp, 1 = fully blurred and dimmed). Only a +/// turned section (a camera tilted onto the desk) is covered: the camera is moving at both +/// ends of such a section, so the picture is hidden for exactly the grow and the shrink, plus a +/// short fade into and out of the steady part. Measured on screen time like the grow, and 0 +/// on and outside the bounds like the grow. At a seam with a layout region (`layouts`) that +/// side has no grow or shrink to hold over: the cover is only its own fade there. +pub fn camera_fullscreen_cover_at( + regions: &[SceneCameraFullscreenRegion], + t: f32, + clock: &ScreenClock, + layouts: &[SceneCameraLayoutRegion], ) -> f32 { - 1.0 - smoothstep(0.5, 1.0, camera_fullscreen_phase_at(regions, t, clock)) + let Some(r) = camera_fullscreen_region_at(regions, t, clock, layouts) else { + return 0.0; + }; + if r.rotation != 180 { + return 0.0; + } + let seams = full_camera_seams(r, layouts); + let (start, end, t) = (clock.at(r.start_sec as f32), clock.at(r.end_sec as f32), clock.at(t)); + // Each hold gets at most half the section; the two fades share what is left, so the start + // fade ends at or before the point where the end fade begins and the cover never jumps. + let len = end - start; + let half = len * 0.5; + let hold_in = if seams.start { 0.0 } else { TRANSITION_WINDOW_S.min(half) }; + let hold_out = if seams.end { 0.0 } else { FULLSCREEN_LEAD_OUT_WINDOW_S.min(half) }; + let steady = (len - hold_in - hold_out).max(0.0); + let fade = DESK_COVER_FADE_S.min(steady * 0.5); + let side = |since: f32, hold: f32, fade: f32| -> f32 { + if since <= hold { + 1.0 + } else if fade > 0.0 && since < hold + fade { + 1.0 - smoothstep(0.0, fade, since - hold) + } else { + 0.0 + } + }; + side(t - start, hold_in, fade).max(side(end - t, hold_out, fade)) } // ============ Rotation 3D (tilt perspective, présets iso/left/right) ================ @@ -1965,6 +2073,9 @@ mod zoom_focus_tests { clip_index: None, start_sec: 18.0 * k, end_sec: 22.0 * k, + rotation: 0, + mirror: None, + full_frame: false, }] }; let (sped, plain) = (zooms(4.0), zooms(1.0)); @@ -1984,21 +2095,223 @@ mod zoom_focus_tests { b.scale ); assert!((a.focus[0] - b.focus[0]).abs() < 1e-4, "t={t}"); - let a = camera_fullscreen_progress_at(&full_camera(4.0), t, &clock); + let a = camera_fullscreen_progress_at(&full_camera(4.0), t, &clock, &[]); let b = - camera_fullscreen_progress_at(&full_camera(1.0), t / 4.0, &ScreenClock::default()); + camera_fullscreen_progress_at(&full_camera(1.0), t / 4.0, &ScreenClock::default(), &[]); assert!((a - b).abs() < 1e-4, "Full Camera t={t} : {a} vs {b}"); } } + fn cam(start: f64, end: f64, rotation: u16) -> SceneCameraFullscreenRegion { + SceneCameraFullscreenRegion { + clip_index: None, + start_sec: start, + end_sec: end, + rotation, + mirror: None, + full_frame: rotation != 0, + } + } + + /// Orientation and transition must name the same region: inside it the region, at its + /// bounds and outside nothing, exactly where the progress envelope is 0. + #[test] + fn the_full_camera_region_at_t_is_the_one_the_envelope_uses() { + let r = [cam(10.0, 20.0, 180), cam(30.0, 40.0, 0)]; + let clock = ScreenClock::default(); + assert_eq!(camera_fullscreen_region_at(&r, 15.0, &clock, &[]).map(|c| c.rotation), Some(180)); + assert_eq!(camera_fullscreen_region_at(&r, 35.0, &clock, &[]).map(|c| c.rotation), Some(0)); + for t in [5.0, 10.0, 20.0, 25.0] { + assert!(camera_fullscreen_region_at(&r, t, &clock, &[]).is_none(), "t={t}"); + assert_eq!(camera_fullscreen_progress_at(&r, t, &clock, &[]), 0.0, "t={t}"); + } + } + + + /// The hold covers exactly the grow; then the picture sharpens over the fade. + #[test] + fn a_turned_section_is_covered_while_the_camera_moves() { + let r = [cam(10.0, 30.0, 180)]; + let clock = ScreenClock::default(); + let c = |t: f32| camera_fullscreen_cover_at(&r, t, &clock, &[]); + assert_eq!(c(10.0 + 0.01), 1.0, "start of the hold"); + assert_eq!(c(10.0 + TRANSITION_WINDOW_S - 0.01), 1.0, "end of the hold"); + let mid_fade = c(10.0 + TRANSITION_WINDOW_S + DESK_COVER_FADE_S / 2.0); + assert!((mid_fade - 0.5).abs() < 1e-3, "half way through the fade: {mid_fade}"); + assert_eq!(c(20.0), 0.0, "steady part is sharp"); + // Mirrored at the end: fade up, then hold for the whole shrink. + let mid_rise = c(30.0 - FULLSCREEN_LEAD_OUT_WINDOW_S - DESK_COVER_FADE_S / 2.0); + assert!((mid_rise - 0.5).abs() < 1e-3, "half way up: {mid_rise}"); + assert_eq!(c(30.0 - FULLSCREEN_LEAD_OUT_WINDOW_S + 0.01), 1.0); + assert_eq!(c(30.0 - 0.01), 1.0); + } + + fn layout_region(start: f64, end: f64) -> SceneCameraLayoutRegion { + SceneCameraLayoutRegion { clip_index: None, start_sec: start, end_sec: end, layers: Vec::new() } + } + + /// A layout region meeting a Full Camera region at a seam: the phase holds 1 on that side + /// (no lead-out before it, no lead-in after it); the other side is unchanged. + #[test] + fn the_full_camera_phase_holds_at_a_seam_with_a_layout_region() { + let clock = ScreenClock::default(); + let r = [cam(10.0, 20.0, 0)]; + let p = |t: f32, layouts: &[SceneCameraLayoutRegion]| { + camera_fullscreen_progress_at(&r, t, &clock, layouts) + }; + let after = [layout_region(20.0, 25.0)]; + let before = [layout_region(5.0, 10.0)]; + for t in [19.0, 19.5, 19.99] { + assert_eq!(p(t, &after), 1.0, "t={t}"); + assert!(p(t, &[]) < 1.0, "a lone region leads out (t={t})"); + } + assert!(p(10.3, &after) < 1.0, "the far side keeps its lead-in"); + for t in [10.01, 10.3, 10.9] { + assert_eq!(p(t, &before), 1.0, "t={t}"); + } + assert!(p(19.5, &before) < 1.0, "the far side keeps its lead-out"); + // On the bounds the phase is 0 as always: the layout region draws the seam. + assert_eq!(p(20.0, &after), 0.0); + assert_eq!(p(10.0, &before), 0.0); + // Off the seam (10 ms), or an empty layout region, changes nothing. + for layouts in [[layout_region(20.01, 25.0)], [layout_region(20.0, 20.0)]] { + assert_eq!(p(19.5, &layouts), p(19.5, &[])); + } + } + + /// A turned section at a seam: no hold on that side, the cover is its own fade there. + #[test] + fn at_a_seam_the_cover_is_only_its_fade() { + let clock = ScreenClock::default(); + let r = [cam(10.0, 30.0, 180)]; + let layouts = [layout_region(5.0, 10.0), layout_region(30.0, 35.0)]; + let c = |t: f32| camera_fullscreen_cover_at(&r, t, &clock, &layouts); + assert!(c(10.001) > 0.99, "covered at the seam: {}", c(10.001)); + let mid_fade = c(10.0 + DESK_COVER_FADE_S / 2.0); + assert!((mid_fade - 0.5).abs() < 1e-3, "half way through the fade: {mid_fade}"); + assert_eq!(c(10.0 + DESK_COVER_FADE_S + 0.01), 0.0, "sharp after the fade"); + let mid_rise = c(30.0 - DESK_COVER_FADE_S / 2.0); + assert!((mid_rise - 0.5).abs() < 1e-3, "half way up: {mid_rise}"); + assert_eq!(c(30.0 - DESK_COVER_FADE_S - 0.01), 0.0); + assert!(c(29.999) > 0.99); + // Without the seams the holds are back. + let lone = |t: f32| camera_fullscreen_cover_at(&r, t, &clock, &[]); + assert_eq!(lone(10.0 + DESK_COVER_FADE_S + 0.01), 1.0); + } + + #[test] + fn the_cover_is_zero_on_and_outside_the_bounds() { + let r = [cam(10.0, 30.0, 180)]; + let clock = ScreenClock::default(); + for t in [0.0, 9.99, 10.0, 30.0, 30.01, 40.0] { + assert_eq!(camera_fullscreen_cover_at(&r, t, &clock, &[]), 0.0, "t={t}"); + } + } + + #[test] + fn a_plain_section_is_never_covered() { + let r = [cam(10.0, 30.0, 0)]; + let clock = ScreenClock::default(); + for step in 0..=400 { + let t = step as f32 * 0.1; + assert_eq!(camera_fullscreen_cover_at(&r, t, &clock, &[]), 0.0, "t={t}"); + } + } + + /// Each hold gets at most half the section and the fades share what is left, so a section + /// shorter than both holds is covered throughout and a longer one keeps its full fades. + #[test] + fn short_sections_shrink_the_fades_first() { + let clock = ScreenClock::default(); + let short = [cam(10.0, 11.5, 180)]; // half = 0.75 < hold-in: no fade at all + for step in 1..150 { + let t = 10.0 + step as f32 * 0.01; + let c = camera_fullscreen_cover_at(&short, t, &clock, &[]); + assert!((0.0..=1.0).contains(&c), "t={t}: {c}"); + assert_eq!(c, 1.0, "a section shorter than both holds is covered throughout (t={t})"); + } + // 3.6 s: steady = 3.6 - 1.015 - 1.523 = 1.06, so both ends keep their full 0.5 s fade. + let medium = [cam(10.0, 13.6, 180)]; + let c = |t: f32| camera_fullscreen_cover_at(&medium, t, &clock, &[]); + let mid_fade = c(10.0 + TRANSITION_WINDOW_S + DESK_COVER_FADE_S / 2.0); + assert!((mid_fade - 0.5).abs() < 1e-3, "start fade half way: {mid_fade}"); + let mid_rise = c(13.6 - FULLSCREEN_LEAD_OUT_WINDOW_S - DESK_COVER_FADE_S / 2.0); + assert!((mid_rise - 0.5).abs() < 1e-3, "end fade half way: {mid_rise}"); + } + + /// 2.6 s: hold-in 1.015, hold-out 1.3 (half), steady 0.285 — the two fades share it, + /// 0.1425 s each, so the start fade ends exactly where the end fade begins. + #[test] + fn medium_sections_share_the_steady_part_between_the_fades() { + let clock = ScreenClock::default(); + let r = [cam(10.0, 12.6, 180)]; + let c = |t: f32| camera_fullscreen_cover_at(&r, t, &clock, &[]); + let fade = (2.6 - TRANSITION_WINDOW_S - 1.3) / 2.0; + let meet = 10.0 + TRANSITION_WINDOW_S + fade; + let mid_fade = c(10.0 + TRANSITION_WINDOW_S + fade / 2.0); + assert!((mid_fade - 0.5).abs() < 1e-2, "start fade half way: {mid_fade}"); + assert!(c(meet) < 1e-2, "both fades are near zero where they meet: {}", c(meet)); + let mid_rise = c(12.6 - 1.3 - fade / 2.0); + assert!((mid_rise - 0.5).abs() < 1e-2, "end fade half way: {mid_rise}"); + assert_eq!(c(12.6 - 1.3 + 0.01), 1.0, "the end hold is half the section"); + } + + /// No section length makes the cover jump: sampled every 1 ms, no step exceeds 0.1. (At + /// 10 ms the shortest legitimate fade — 42 ms in a 2.2 s section — already steps ~0.3 per + /// sample, so the finer grid is what tells a ramp from a jump.) + #[test] + fn the_cover_is_continuous_for_every_section_length() { + let clock = ScreenClock::default(); + for len in [2.0_f32, 2.2, 2.6, 3.0, 3.5, 4.0, 4.5] { + let r = [cam(10.0, 10.0 + len as f64, 180)]; + let steps = (len * 1000.0).round() as i32; + let mut prev = camera_fullscreen_cover_at(&r, 10.0 + 0.0005, &clock, &[]); + for step in 1..steps { + let t = 10.0 + step as f32 * 0.001; + let c = camera_fullscreen_cover_at(&r, t, &clock, &[]); + assert!((0.0..=1.0).contains(&c), "len={len} t={t}: {c}"); + assert!((c - prev).abs() <= 0.1, "len={len} t={t}: {prev} -> {c}"); + prev = c; + } + } + } + + /// Under a speed change the windows are measured on screen time, like the grow. + #[test] + fn the_cover_runs_on_screen_time() { + let clock = ScreenClock::new(&[speed(None, 0.0, 1000.0, 2.0)], 0); + let r = [cam(10.0, 40.0, 180)]; + // At 2x, the 1.015 s hold spans 2.03 s of source time. + assert_eq!(camera_fullscreen_cover_at(&r, 10.0 + 1.9, &clock, &[]), 1.0); + assert!(camera_fullscreen_cover_at(&r, 10.0 + 3.6, &clock, &[]) < 1.0); + } + #[test] + fn the_region_fields_parse_and_default() { + let plain: SceneCameraFullscreenRegion = + serde_json::from_str(r#"{"startSec":1,"endSec":2}"#).unwrap(); + assert_eq!((plain.rotation, plain.mirror, plain.full_frame), (0, None, false)); + let desk: SceneCameraFullscreenRegion = serde_json::from_str( + r#"{"startSec":1,"endSec":2,"rotation":180,"mirror":false,"fullFrame":true}"#, + ) + .unwrap(); + assert_eq!((desk.rotation, desk.mirror, desk.full_frame), (180, Some(false), true)); + } + /// Les coins suivent la phase, pas le rect. Avec `1 - progrès`, ils étaient carrés presque /// tout du long de la montée et ne revenaient qu'à la toute fin du retour. #[test] fn full_camera_corners_dissolve_late_and_come_back_early() { - let r = [SceneCameraFullscreenRegion { clip_index: None, start_sec: 10.0, end_sec: 20.0 }]; + let r = [SceneCameraFullscreenRegion { + clip_index: None, + start_sec: 10.0, + end_sec: 20.0, + rotation: 0, + mirror: None, + full_frame: false, + }]; let clock = ScreenClock::default(); let at = |t: f32| { - (camera_fullscreen_progress_at(&r, t, &clock), camera_fullscreen_shape_at(&r, t, &clock)) + (camera_fullscreen_progress_at(&r, t, &clock, &[]), camera_fullscreen_shape_at(&r, t, &clock, &[])) }; // Premier tiers de la montée : la caméra a fait l'essentiel du chemin, coins intacts. let (progress, shape) = at(10.0 + TRANSITION_WINDOW_S / 3.0); @@ -2178,6 +2491,32 @@ mod arrow_tests { let (_, half) = arrow_local_geometry("right", -5.0, [100.0, 100.0]); assert_eq!(half, 0.0); } + + /// L'éditeur encadre chaque flèche par l'enveloppe de ses traits (`ARROW_EXTENTS`, + /// `src/lib/ai-edition/annotations/arrowBounds.ts`) : elle doit être celle que dessine ce + /// module, sinon le gimbal ne tombe plus sur la flèche. + #[test] + fn the_editor_frames_each_arrow_by_its_strokes() { + let ts = include_str!("../../../src/lib/ai-edition/annotations/arrowBounds.ts"); + let table = &ts[ts.find("ARROW_EXTENTS").expect("la table")..]; + let table = &table[..table.find("};").expect("sa fin")]; + for dir in ["up", "down", "left", "right", "up-right", "up-left", "down-right", "down-left"] { + let key = if dir.contains('-') { format!("\"{dir}\":") } else { format!("\t{dir}:") }; + let row = &table[table.find(&key).unwrap_or_else(|| panic!("{dir} absente"))..]; + let row = &row[row.find('[').expect("[") + 1..row.find(']').expect("]")]; + let got: Vec = row.split(',').map(|v| v.trim().parse().expect("nombre")).collect(); + let segs = arrow_segments_viewbox(dir); + let xs = segs.iter().flat_map(|s| [s[0], s[2]]); + let ys = segs.iter().flat_map(|s| [s[1], s[3]]); + let want = [ + xs.clone().fold(f32::MAX, f32::min), + ys.clone().fold(f32::MAX, f32::min), + xs.fold(f32::MIN, f32::max), + ys.fold(f32::MIN, f32::max), + ]; + assert_eq!(got, want, "{dir}"); + } + } } #[cfg(test)] @@ -3125,6 +3464,16 @@ mod exporter_frame_totals { 60, "région débordant la fenêtre gardée" ); + let issue_876_fps = 60.0; + let first_clip = frames( + 0.0, + 44.81830642526596, + &[region(44.818, 44.818, 1.5)], + issue_876_fps, + ); + let second_clip = frames(0.0, 47.833333, &[region(0.0, 1.87, 1.5)], issue_876_fps); + assert_eq!(first_clip, 2690, "une région vide ne double pas le premier clip"); + assert_eq!(first_clip + second_clip, 5523, "total corrigé du repro #876"); assert_eq!( frames( 0.0, diff --git a/crates/compositor/src/scene.rs b/crates/compositor/src/scene.rs index de24278fdb..98a5957c46 100644 --- a/crates/compositor/src/scene.rs +++ b/crates/compositor/src/scene.rs @@ -19,6 +19,67 @@ pub struct SceneClip { /// Une source sans piste audio décodable garde sa durée via du silence natif. #[serde(default)] pub has_audio: bool, + /// Cameras 2-4: index k-1 = camera k. An empty `path` means "no camera in this slot". + #[serde(default)] + pub additional_cameras: Vec, +} + +/// An additional camera (2-4) of a clip. = `CompositorClipCamera` (TS). +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SceneClipCamera { + pub path: String, + /// Camera source time = screen source time - this. + pub offset_sec: f64, +} + +/// Settings of one camera (index 0 = camera 1). With a `homography` the camera's rotation, +/// mirror and crop are not sent (the corner order already defines the picture). +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SceneCamera { + pub index: usize, + /// 0 or 180. + #[serde(default)] + pub rotation: u16, + #[serde(default)] + pub mirror: Option, + #[serde(default)] + pub crop: Option, + /// Row-major 3x3, target uv -> camera uv. + #[serde(default)] + pub homography: Option<[f32; 9]>, + /// Width/height of the corrected picture. + #[serde(default)] + pub aspect: Option, +} + +/// One camera placed by a layout region, in draw order. +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SceneCameraLayer { + pub camera: usize, + pub rect: SceneRect, + #[serde(default)] + pub radius_frac: f32, + /// Same vocabulary as `SceneLayout::webcam_shape` (`webcam_shape_code`). + #[serde(default)] + pub shape: String, + /// Covers the screen: drawn without a shadow. + #[serde(default)] + pub fills_frame: bool, +} + +/// A camera layout (anything but a plain Full Camera) on a span of a clip's source time. +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SceneCameraLayoutRegion { + #[serde(default)] + pub clip_index: Option, + pub start_sec: f64, + pub end_sec: f64, + #[serde(default)] + pub layers: Vec, } #[derive(Debug, Clone, Copy, Deserialize)] @@ -291,7 +352,7 @@ impl SceneFrame { } /// Fond derrière l'écran (parsé depuis `settings.wallpaper`). -#[derive(Debug, Clone, Deserialize)] +#[derive(Debug, Clone, PartialEq, Deserialize)] #[serde(tag = "kind", rename_all = "lowercase")] pub enum SceneBackground { Color { color: String }, @@ -528,7 +589,8 @@ pub struct SceneSpeedRegion { /// Une zone "Full Camera" de la timeline (temps en secondes) : la caméra PREND tout le cadre /// pendant cette fenêtre (plein écran net — ni marge, ni arrondi, ni masque, ni fond derrière). -/// Pas de champs au-delà des bornes temporelles (miroir de `CameraFullscreenRegion`, TS). +/// The orientation fields are resolved by the app (`sceneDescription.ts`): a desk shot +/// arrives as `rotation: 180`, `mirror: false`, `fullFrame: true`. #[derive(Debug, Clone, Copy, Deserialize)] #[serde(rename_all = "camelCase")] pub struct SceneCameraFullscreenRegion { @@ -537,6 +599,15 @@ pub struct SceneCameraFullscreenRegion { pub clip_index: Option, pub start_sec: f64, pub end_sec: f64, + /// 0 or 180. Anything else is treated as 0 by `frame_geometry::webcam_orientation`. + #[serde(default)] + pub rotation: u16, + /// Mirror inside this region; `None` = the layout's `webcam_mirror`. + #[serde(default)] + pub mirror: Option, + /// Ignore `layout.webcam_crop` inside this region. + #[serde(default)] + pub full_frame: bool, } /// Rendu du curseur. @@ -730,6 +801,12 @@ pub struct Scene { /// `#[serde(default)]` : champ ajouté après coup, absent des JSON de test existants. #[serde(default)] pub camera_fullscreen_regions: Vec, + /// Settings of the cameras that have any (see `Scene::camera`). + #[serde(default)] + pub cameras: Vec, + /// Layouts that put several cameras on screen. Absent = none. + #[serde(default)] + pub camera_layout_regions: Vec, pub cursor: SceneCursor, /// Global audio finishing. Default keeps old scene payloads bit-for-bit compatible. #[serde(default)] @@ -751,6 +828,11 @@ pub struct Scene { } impl Scene { + /// The settings of camera `index` (0 = camera 1), if the scene carries any. + pub fn camera(&self, index: usize) -> Option<&SceneCamera> { + self.cameras.iter().find(|c| c.index == index) + } + /// Parse le JSON produit par `buildSceneDescription` (TS). pub fn from_json(json: &str) -> anyhow::Result { Ok(serde_json::from_str(json)?) @@ -788,6 +870,9 @@ impl Scene { scene.camera_fullscreen_regions.retain(|region| { belongs(region.clip_index, region.start_sec, region.end_sec) }); + scene.camera_layout_regions.retain(|region| { + belongs(region.clip_index, region.start_sec, region.end_sec) + }); scene.annotations.retain(|annotation| { belongs(annotation.clip_index, annotation.start_sec, annotation.end_sec) }); @@ -943,6 +1028,83 @@ mod tests { } } + /// A scene JSON with one clip; the arguments are spliced in as extra clip / top-level fields. + fn scene_json_with(clip_extra: &str, top_extra: &str) -> String { + format!( + r##"{{"clips":[{{"screenPath":"/s.mp4","webcamPath":"/w.mp4","sourceStartSec":0,"sourceEndSec":10,"webcamOffsetSec":0{clip_extra}}}],"layout":{{"preset":"picture-in-picture","webcamSize":1,"webcamShape":"rectangle","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false}},"effects":{{"padding":0,"blur":false,"shadow":0,"roundnessFrac":0,"motionBlur":0}},"background":{{"kind":"color","color":"#000000"}},"zoomRegions":[],"cursor":{{"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0,"clipToBounds":false,"theme":"default"}},"cropByClip":[],"output":{{"width":1920,"height":1080,"fps":null}}{top_extra}}}"## + ) + } + + #[test] + fn parses_extra_cameras_settings_and_layout_regions() { + let json = scene_json_with( + r#","additionalCameras":[{"path":"/w-2.mp4","offsetSec":0.12},{"path":"","offsetSec":0}]"#, + r#","cameras":[{"index":1,"rotation":180,"mirror":true,"crop":{"x":0,"y":0.1,"width":0.5,"height":0.8}},{"index":2,"homography":[1,0,0,0,1,0,0,0,1],"aspect":1.5}], + "cameraLayoutRegions":[{"clipIndex":0,"startSec":1,"endSec":4,"underTrim":true,"layers":[ + {"camera":1,"rect":{"x":0,"y":0,"width":1,"height":1},"radiusFrac":0,"shape":"rectangle","fillsFrame":true}, + {"camera":0,"rect":{"x":0.7,"y":0.7,"width":0.22,"height":0.2},"radiusFrac":0.12,"shape":"rounded","fillsFrame":false}]}]"#, + ); + let s = Scene::from_json(&json).expect("parse"); + assert_eq!(s.clips[0].additional_cameras.len(), 2); + assert_eq!(s.clips[0].additional_cameras[0].path, "/w-2.mp4"); + assert!((s.clips[0].additional_cameras[0].offset_sec - 0.12).abs() < 1e-9); + assert_eq!(s.clips[0].additional_cameras[1].path, ""); + assert!(s.camera(0).is_none()); + let c1 = s.camera(1).expect("camera 1"); + assert_eq!(c1.rotation, 180); + assert_eq!(c1.mirror, Some(true)); + assert_eq!(c1.crop.expect("crop").height, 0.8); + assert!(c1.homography.is_none() && c1.aspect.is_none()); + let c2 = s.camera(2).expect("camera 2"); + assert_eq!(c2.rotation, 0); + assert_eq!(c2.homography, Some([1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0])); + assert_eq!(c2.aspect, Some(1.5)); + assert_eq!(s.camera_layout_regions.len(), 1); + let r = &s.camera_layout_regions[0]; + assert_eq!((r.clip_index, r.start_sec, r.end_sec), (Some(0), 1.0, 4.0)); + assert_eq!(r.layers.len(), 2); + assert_eq!(r.layers[0].camera, 1); + assert!(r.layers[0].fills_frame); + assert_eq!(r.layers[1].shape, "rounded"); + assert!((r.layers[1].radius_frac - 0.12).abs() < 1e-6); + assert!((r.layers[1].rect.width - 0.22).abs() < 1e-6); + } + + #[test] + fn old_scene_json_parses_without_the_new_fields() { + let s = Scene::from_json(&scene_json_with("", "")).expect("parse"); + assert!(s.clips[0].additional_cameras.is_empty()); + assert!(s.cameras.is_empty()); + assert!(s.camera_layout_regions.is_empty()); + assert!(s.camera(0).is_none()); + } + + #[test] + fn for_clip_window_keeps_only_this_clips_layout_regions() { + let layers = r#""layers":[{"camera":0,"rect":{"x":0,"y":0,"width":1,"height":1},"radiusFrac":0,"shape":"rectangle","fillsFrame":true}]"#; + let json = scene_json_with( + "", + &format!( + r#","cameraLayoutRegions":[ + {{"clipIndex":0,"startSec":1,"endSec":2,{layers}}}, + {{"clipIndex":1,"startSec":1,"endSec":2,{layers}}}, + {{"startSec":8,"endSec":9,{layers}}}, + {{"startSec":20,"endSec":30,{layers}}}]"# + ), + ); + let s = Scene::from_json(&json).expect("parse"); + let w = s.for_clip_window(0, 0.0, 10.0); + // Clip 0's own region and the unaddressed one that overlaps the window; not clip 1's, + // not the unaddressed one outside the window. + let kept: Vec<(Option, f64)> = w + .camera_layout_regions + .iter() + .map(|r| (r.clip_index, r.start_sec)) + .collect(); + assert_eq!(kept, vec![(Some(0), 1.0), (None, 8.0)]); + assert_eq!(s.for_clip_window(1, 0.0, 10.0).camera_layout_regions.len(), 2); + } + #[test] fn parses_webcam_rect_payload() { // webcamRect est une fraction 0..1 du cadre de sortie ; sa présence doit désactiver diff --git a/crates/compositor/src/sculpt.rs b/crates/compositor/src/sculpt.rs index de9c4e65a0..89fa9a277a 100644 --- a/crates/compositor/src/sculpt.rs +++ b/crates/compositor/src/sculpt.rs @@ -1,17 +1,18 @@ -//! Curseurs sculptés : la flèche et la main des cinq thèmes d'origine, modelées en volumes dans les -//! shaders (mode 15, `sculpt_proto`) au lieu d'extruder leur PNG — capsules et unions lissées, -//! extrusions arrondies, voxels, polyèdres taillés. Ce module en tient ce que la géométrie doit -//! savoir côté CPU : l'identifiant que lit le shader, et la boîte du modèle (`SpriteShape`) qui pose -//! le hotspot, règle la garde au sol et borne la boîte de dessin. +//! Curseurs sculptés : la flèche et la main des thèmes d'origine, modelées en volumes dans les +//! shaders (mode 15, `sculpt_proto`) au lieu d'extruder leur PNG — voxels (Pixel Candy), pièces +//! cerclées du trait de leur dessin (Studio Ink, Pop Coral, Star Sprout). Prism Glow est à part : +//! un MAILLAGE tracé sur son dessin à facettes, un cristal lancé de rayons boîte par boîte +//! dans son serti marine (`prism_mesh`). Ce module en tient ce que la géométrie doit savoir côté +//! CPU : l'identifiant que lit le shader, et la boîte du modèle (`SpriteShape`) qui pose le +//! hotspot, règle la garde au sol et borne la boîte de dessin. //! //! Les formes sont écrites dans le repère du PROTOTYPE où elles ont été dessinées : hauteur du //! curseur 1, x à droite, y VERS LE HAUT, z vers la caméra, l'écran en z = 0. Les constantes //! ci-dessous sont le miroir des `SCULPT_*` des trois shaders. //! -//! Les voxels de la flèche de Pixel Candy (`SCULPT_ARROWPIX`, `SCULPT_ARROWBACK`) sont son polygone -//! échantillonné : une cellule de 1/16 porte un voxel quand son centre est à plus de 0,01 dedans, un -//! bloc de la couche violette quand il est à moins de 0,9 cellule. Changer le polygone, c'est -//! régénérer ces deux tables. +//! Les grilles de voxels de Pixel Candy sont des tables que +//! `scripts/generate-pixel-candy-voxels.mjs` écrit dans les trois shaders : on change la grille +//! dans le script, on le relance, et on ajuste ici la boîte si la forme a bougé. use crate::frame_geometry::SpriteShape; @@ -24,29 +25,48 @@ const ZREF_ARROW: f32 = 0.2; const ZREF_HAND: f32 = 0.185; /// Le dessous de tous les modèles, dans le prototype. const Z_LOW: f32 = 0.05; -/// Le plus haut : la table de la flèche taillée, l'étoile de Star Sprout sur la manchette. -const Z_HIGH_ARROW: f32 = 0.29; +/// Le côté d'un cube de Pixel Candy : seize lignes font un curseur haut de 1. +const VOX: f32 = 0.0625; +/// Le plus haut : les yeux de l'étoile de Star Sprout, sur la flèche comme sur la manchette. +const Z_HIGH_ARROW: f32 = 0.31; const Z_HIGH_HAND: f32 = 0.4; -/// Boîtes des deux formes dans le prototype (x0, x1, haut), qui contiennent leurs cinq variantes : -/// couche violette des voxels, étoile, cristal compris. Leur hauteur est celle du sprite, 1 / SCULPT_SCALE : -/// le mode 15 tient le plus grand côté de la boîte pour 1 (`sprite_size`). -const BOX_ARROW: [f32; 3] = [-0.1, 0.75, 0.09]; -const BOX_HAND: [f32; 3] = [-0.27, 0.73, 0.07]; +/// Boîtes des modèles dans le prototype (x0, x1, haut). Leur hauteur est celle du sprite, +/// 1 / SCULPT_SCALE : le mode 15 tient le plus grand côté de la boîte pour 1 (`sprite_size`). +/// Celles des cubes de Pixel Candy ; puis des thèmes cerclés, que leur trait élargit et que +/// débordent l'étoile et les feuilles (Star Sprout), les tirets du clic et les calques (Pop +/// Coral), le pouce. +const BOX_PIXEL_ARROW: [f32; 3] = [-0.03, 0.72, 0.03]; +const BOX_PIXEL_HAND: [f32; 3] = [-0.345, 0.72, 0.03]; +const BOX_INK_ARROW: [f32; 3] = [-0.08, 0.6, 0.03]; +const BOX_INK_HAND: [f32; 3] = [-0.36, 0.6, 0.03]; +const BOX_CORAL_ARROW: [f32; 3] = [-0.35, 0.6, 0.06]; +const BOX_CORAL_HAND: [f32; 3] = [-0.38, 0.62, 0.04]; +const BOX_SPROUT_ARROW: [f32; 3] = [-0.08, 0.87, 0.03]; +const BOX_SPROUT_HAND: [f32; 3] = [-0.33, 0.56, 0.03]; /// Dans l'ordre des identifiants du shader. const THEMES: [&str; 5] = ["studio-ink", "prism-glow", "pop-coral", "pixel-candy", "star-sprout"]; -/// Le haut de la silhouette (y du prototype) : la couche violette des voxels dépasse la pointe -/// d'une cellule, la pointe arrondie de la flèche et le sommet de sa table taillée dépassent le -/// hotspot de 0,03, le bout de l'index y est. -fn silhouette_top(theme: usize, arrow: bool) -> f32 { +/// La boîte du modèle (x0, x1, haut) dans le prototype. +fn model_box(theme: usize, arrow: bool) -> [f32; 3] { match (THEMES[theme], arrow) { - ("pixel-candy", _) => 0.0625, - (_, true) => 0.03, - (_, false) => 0.0, + ("studio-ink", true) => BOX_INK_ARROW, + ("studio-ink", false) => BOX_INK_HAND, + ("pop-coral", true) => BOX_CORAL_ARROW, + ("pop-coral", false) => BOX_CORAL_HAND, + ("pixel-candy", true) => BOX_PIXEL_ARROW, + ("pixel-candy", false) => BOX_PIXEL_HAND, + ("star-sprout", true) => BOX_SPROUT_ARROW, + ("star-sprout", false) => BOX_SPROUT_HAND, + (other, _) => unreachable!("thème sculpté inconnu : {other}"), } } +/// Le curseur sculpté que nomme la scène est de verre (Prism Glow) : il réfracte ce qui est dessous. +pub fn refracts(name: &str) -> bool { + sculpted_shape(name).is_some_and(|s| THEMES[((s.sculpt - 1) / 2) as usize] == "prism-glow") +} + /// Le curseur sculpté que nomme la scène (`"/<état>"`, cf. `resolveCursorSprites`), sous /// la forme que le mode 15 attend. `None` pour un nom inconnu : l'appelant extrude le sprite. pub fn sculpted_shape(name: &str) -> Option { @@ -57,7 +77,20 @@ pub fn sculpted_shape(name: &str) -> Option { "pointer" => false, _ => return None, }; - let [x0, x1, y1] = if arrow { BOX_ARROW } else { BOX_HAND }; + let sculpt = 1 + 2 * theme as u32 + u32::from(!arrow); + if THEMES[theme] == "prism-glow" { + // Le maillage est déjà dans le repère du modèle : sa boîte, son dessus et son dessous. + let m = &crate::prism_mesh::MODELS[usize::from(!arrow)]; + return Some(SpriteShape { + size: m.size, + hotspot: m.hotspot, + top: m.top, + max_height: m.height, + thick: m.thick, + sculpt, + }); + } + let [x0, x1, y1] = model_box(theme, arrow); let (zref, z_high) = if arrow { (ZREF_ARROW, Z_HIGH_ARROW) } else { (ZREF_HAND, Z_HIGH_HAND) }; let s = SCULPT_SCALE; // Repère du modèle : y vers le bas, le coin haut-gauche de la boîte est donc (x0, y1). @@ -65,10 +98,10 @@ pub fn sculpted_shape(name: &str) -> Option { Some(SpriteShape { size, hotspot: [-x0 * s / size[0], y1 * s], - top: (y1 - silhouette_top(theme, arrow)) * s, + top: y1 * s, max_height: (z_high - zref) * s, thick: (zref - Z_LOW) * s, - sculpt: 1 + 2 * theme as u32 + u32::from(!arrow), + sculpt, }) } @@ -99,20 +132,21 @@ mod tests { #[test] fn other_states_and_themes_keep_the_extruded_sprite() { - for name in ["default/arrow", "pop-coral/text", "pop-coral", "", "pop-coral/arrow/x"] { + for name in ["default/arrow", "pop-coral/text", "pop-coral", "", "pop-coral/arrow/x", "prism-glow/text"] { assert!(sculpted_shape(name).is_none(), "{name}"); } } /// Le hotspot est dans la boîte, sous le haut de la silhouette ; le modèle a une épaisseur et /// une hauteur ; la boîte a la hauteur du sprite, son plus grand côté, comme le veut le mode 15. + /// Le serti de Prism Glow est le plus mince (0,043) : c'est son épaisseur dans le dessin. #[test] fn the_box_holds_the_hotspot_and_the_silhouette_top() { for name in NAMES { let s = sculpted_shape(name).unwrap(); assert!((0.0..1.0).contains(&s.hotspot[0]) && (0.0..1.0).contains(&s.hotspot[1]), "{name}"); assert!(s.top <= s.hotspot[1] + 1e-6, "{name} : haut de silhouette sous le hotspot"); - assert!(s.thick > 0.05 && s.max_height > 0.0, "{name}"); + assert!(s.thick > 0.04 && s.max_height > 0.0, "{name}"); assert!(s.size[0] < 1.0 && s.size[1] == 1.0, "{name} : le plus grand côté n'est pas 1"); } } @@ -131,6 +165,8 @@ mod tests { ("SCULPT_ZREF_ARROW", ZREF_ARROW), ("SCULPT_ZREF_HAND", ZREF_HAND), ("SCULPT_HOVER", Z_LOW), + ("SCULPT_VOX", VOX), + ("PRISM_BEVEL", crate::prism_mesh::BEVEL), ] { let line = src .lines() @@ -145,4 +181,158 @@ mod tests { } } } + + /// Les nombres de la table `name` d'un shader, du `=` à la fin de la liste, constructeurs de + /// vecteurs retirés : la même table s'écrit alors pareil dans les trois langages. + fn table(src: &str, name: &str) -> Vec { + // La déclaration : `NAME =` en WGSL, `NAME[taille]` en HLSL et en Metal (pas un `NAME[i]` du code). + let decl = format!("{name}["); + let sized = |&i: &usize| src[i + decl.len()..].starts_with(|c: char| c.is_ascii_digit()); + let at = src.find(&format!("{name} =")).or_else(|| src.match_indices(&decl).map(|(i, _)| i).find(sized)); + let rest = &src[at.expect(name)..]; + let rest = &rest[rest.find('=').unwrap()..]; + let open = rest.find(['(', '{']).unwrap() + 1; + let close = [rest.find(");"), rest.find("};")].into_iter().flatten().min().unwrap(); + let mut list = rest[open..close].to_string(); + for ty in ["vec2", "vec3", "vec4", "vec4", "float2", "float3", "float4", "int4"] { + list = list.replace(ty, ""); + } + list.split(|c: char| !(c.is_ascii_digit() || c == '.' || c == '-')) + .filter(|t| !t.is_empty()) + .map(|t| t.parse().unwrap_or_else(|_| panic!("{name} : nombre illisible {t}"))) + .collect() + } + + /// Les grilles de Pixel Candy, que `scripts/generate-pixel-candy-voxels.mjs` écrit dans les + /// trois shaders, y sont les mêmes ; chaque pixel coloré est plein, et d'une seule couleur. + #[test] + fn the_voxel_tables_match_in_the_three_shaders() { + let hlsl = include_str!("shaders.hlsl"); + let metal = include_str!("shaders.metal"); + let wgsl = include_str!("vk_shaders/layer.wgsl"); + let names = ["PIX_BODY", "PIX_LINE", "PIX_HI", "PIX_SHADE", "PIX_RECT_N", "PIX_GRID", "PIX_ORIGIN", "PIX_BOX", "PIX_RECT"]; + for name in names { + let reference = table(wgsl, name); + assert!(!reference.is_empty(), "{name} vide"); + assert_eq!(table(hlsl, name), reference, "{name} : HLSL et WGSL diffèrent"); + assert_eq!(table(metal, name), reference, "{name} : Metal et WGSL diffèrent"); + } + let body = table(wgsl, "PIX_BODY"); + let classes = ["PIX_LINE", "PIX_HI", "PIX_SHADE"].map(|name| table(wgsl, name)); + for (i, &face) in body.iter().enumerate() { + let mut seen = 0u32; + for class in &classes { + let m = class[i] as u32; + assert_eq!(m & !(face as u32), 0, "couleur hors de la forme, ligne {i}"); + assert_eq!(m & seen, 0, "deux couleurs sur un pixel, ligne {i}"); + seen |= m; + } + } + } + + /// Seul Prism Glow est de verre, flèche et main. + #[test] + fn only_prism_glow_refracts() { + for name in NAMES { + assert_eq!(refracts(name), name.starts_with("prism-glow/"), "{name}"); + } + assert!(!refracts("prism-glow/text")); + } + + /// Le maillage de Prism Glow tient dans sa boîte (le shader n'y lance de rayons que là), son + /// dessous est à l'épaisseur annoncée (la garde au sol en dépend) et son sommet à la hauteur. + #[test] + fn the_prism_boxes_hold_their_mesh() { + use crate::prism_mesh::{MODELS, POLYS, TRIS}; + for (k, name) in ["prism-glow/arrow", "prism-glow/pointer"].iter().enumerate() { + let s = sculpted_shape(name).unwrap(); + let m = &MODELS[k]; + let [x0, y0] = [-s.hotspot[0] * s.size[0], -s.hotspot[1] * s.size[1]]; + let inside = |x: f32, y: f32| x >= x0 && x <= x0 + s.size[0] && y >= y0 && y <= y0 + s.size[1]; + let (mut zlo, mut zhi) = (f32::MAX, f32::MIN); + for t in TRIS[4 * m.tri_start..4 * (m.tri_start + m.tri_count)].chunks(4) { + for e in [[0.0; 3], [t[1][0], t[1][1], t[1][2]], [t[2][0], t[2][1], t[2][2]]] { + let [x, y, z] = [t[0][0] + e[0], t[0][1] + e[1], t[0][2] + e[2]]; + assert!(inside(x, y), "{name} : sommet ({x}, {y}) hors de la boîte"); + zlo = zlo.min(z); + zhi = zhi.max(z); + } + } + let sil = &POLYS[m.sil_start..m.sil_start + m.sil_count]; + let out = &POLYS[m.out_start..m.out_start + m.out_count]; + for p in sil.iter().chain(out) { + assert!(inside(p[0], p[1]), "{name} : contour ({}, {}) hors de la boîte", p[0], p[1]); + } + assert!((zlo + s.thick).abs() < 1e-5, "{name} : dessous {zlo}, épaisseur {}", s.thick); + assert!((zhi - s.max_height).abs() < 1e-5, "{name} : sommet {zhi}, hauteur {}", s.max_height); + } + } + + /// Les trois shaders portent le maillage de `prism_mesh.rs` tel quel : triangles, boîtes, + /// polygones, et les débuts et longueurs de chaque modèle dans ces tables. Le WGSL lit + /// triangles, polygones et boîtes dans un uniform rempli de `prism_mesh.rs` + /// (`compositor_linux.rs`) : il n'en déclare que la taille. + #[test] + fn the_shaders_carry_the_prism_mesh() { + use crate::prism_mesh::{PrismModel, BOXES, MODELS, POLYS, TRIS}; + let per_model = |f: fn(&PrismModel) -> usize| MODELS.iter().map(|m| f(m) as f32).collect::>(); + let want: [(&str, Vec); 11] = [ + ("PRISM_TRIS", TRIS.iter().flatten().copied().collect()), + ("PRISM_POLY", POLYS.iter().flatten().copied().collect()), + ("PRISM_BOXES", BOXES.iter().flatten().copied().collect()), + ("PRISM_BOX_START", per_model(|m| m.box_start)), + ("PRISM_BOX_COUNT", per_model(|m| m.box_count)), + ("PRISM_TRI_START", per_model(|m| m.tri_start)), + ("PRISM_TRI_COUNT", per_model(|m| m.tri_count)), + ("PRISM_SIL_START", per_model(|m| m.sil_start)), + ("PRISM_SIL_COUNT", per_model(|m| m.sil_count)), + ("PRISM_OUT_START", per_model(|m| m.out_start)), + ("PRISM_OUT_COUNT", per_model(|m| m.out_count)), + ]; + let wgsl = include_str!("vk_shaders/layer.wgsl"); + for field in [ + format!("tris: array, {}>", TRIS.len()), + format!("poly: array, {}>", POLYS.len()), + format!("boxes: array, {}>", BOXES.len()), + ] { + assert!(wgsl.contains(&field), "vk_shaders/layer.wgsl : `{field}` attendu dans PrismMesh"); + } + for (file, src, tables) in [ + ("shaders.hlsl", include_str!("shaders.hlsl"), &want[..]), + ("shaders.metal", include_str!("shaders.metal"), &want[..]), + ("vk_shaders/layer.wgsl", wgsl, &want[3..]), + ] { + for (name, values) in tables { + let got = table(src, name); + assert_eq!(got.len(), values.len(), "{file} : {name}, nombre de valeurs"); + for (i, (g, w)) in got.iter().zip(values).enumerate() { + assert!((g - w).abs() <= 1e-6 + 1e-6 * w.abs(), "{file} : {name}[{i}] = {g} au lieu de {w}"); + } + } + } + } + + /// Chaque triangle du cristal est dans une boîte et une seule, qui le contient : le shader ne + /// teste les triangles d'une boîte que si le rayon la touche, un triangle hors de sa boîte + /// disparaîtrait par endroits. + #[test] + fn each_crystal_triangle_sits_in_its_box() { + use crate::prism_mesh::{BOXES, MODELS, TRIS}; + for (k, m) in MODELS.iter().enumerate() { + let mut next = m.tri_start; + for b in m.box_start..m.box_start + m.box_count { + let (lo, hi) = (BOXES[2 * b], BOXES[2 * b + 1]); + let (first, count) = (lo[3] as usize, hi[3] as usize); + assert_eq!(first, next, "modèle {k}, boîte {b} : triangles non contigus"); + next += count; + for t in TRIS[4 * first..4 * (first + count)].chunks(4) { + for e in [[0.0; 3], [t[1][0], t[1][1], t[1][2]], [t[2][0], t[2][1], t[2][2]]] { + let v = [t[0][0] + e[0], t[0][1] + e[1], t[0][2] + e[2]]; + assert!((0..3).all(|a| v[a] >= lo[a] && v[a] <= hi[a]), "modèle {k}, boîte {b} : sommet {v:?} dehors"); + } + } + } + assert_eq!(next, m.tri_start + m.tri_count, "modèle {k} : triangles hors des boîtes"); + } + } } diff --git a/crates/compositor/src/shaders.hlsl b/crates/compositor/src/shaders.hlsl index 30fbf1bd54..dfc8360d47 100644 --- a/crates/compositor/src/shaders.hlsl +++ b/crates/compositor/src/shaders.hlsl @@ -1,5 +1,12 @@ // Compositeur — un draw par calque (quad). NV12->RGB maison (E1), coins arrondis SDF (E2). // Tout écrit depuis les maths (§7), rien repris de l'ancien paradigme. +// +// `ps_main` est compilé DEUX fois (`build.rs`) : sans `LAYER_MODELS` pour tous les calques, et +// avec, en `ps_main_models`, pour les seuls modèles 3D (modes 15 à 17, `LayerCB::needs_models`). +// Un pixel shader alloue à chaque draw les registres de sa branche la plus lourde, et ce sont +// ces modèles ray-tracés : le même shader les portait pour le fond, l'ombre et la vidéo, qui +// couvrent toute la sortie. Mesuré sur la version WGSL (RADV) : 168 VGPR, 6 vagues par SIMD au +// lieu de 32, et un export Linux deux fois plus lent ; sans eux, 48. cbuffer Layer : register(b0) { @@ -16,6 +23,9 @@ cbuffer Layer : register(b0) float4 trail_a; // mode 8 : coins TL, TR du plan à la frame précédente (px locaux, comme fx) ; mode 18 incliné : en fractions de sortie float4 trail_b; // mode 8 : coins BR, BL du plan à la frame précédente (comme src_prev) ; mode 18 incliné : en fractions de sortie float4 trail_mb; // mode 8 : x = taps, y = force du flou de mouvement (ceux du mode 0) ; mode 18 incliné : le `mb` du mode 8 (profondeur de champ), et `color.xy` sa lampe ; 0 ailleurs + float4 cover; // x = desk-view cover 0..1, y = blur radius (quad px), z = dim, w unused + float4 persp[3]; // mode 0: rows of the homography quad point (0..1 in dst) -> camera uv (0..1 of the valid frame), xyz; read when layer_fx.y = 1 + float4 layer_fx; // x = transparency 0..1 (0 = opaque), every mode; y = 1 when persp applies; z, w unused }; // Mode 15 (curseur modélisé) : le détail des emplacements est dans `frame_geometry.rs`, en tête // de la section « Curseur modélisé » (`cursor_model_cb`). Mode 17 (appareil modelé) : en tête de @@ -81,6 +91,7 @@ float3 sample_yuv(float2 uv) return yuv709_limited(y, cbcr); } +// La même, au niveau 0 : lisible dans une boucle ou une branche (le cristal du mode 15). // SDF segment à bouts ronds — la primitive des flèches d'annotation, dont les tracés SVG sont // trois segments `stroke-linecap="round"` (cf. ArrowSvgs.tsx). float sd_segment(float2 p, float2 a, float2 b) @@ -105,9 +116,13 @@ float sd_segment(float2 p, float2 a, float2 b) float sd_round_rect(float2 p, float2 halfsz, float r) { float hmin = min(halfsz.x, halfsz.y); - if (r <= 0.0 || hmin <= 0.0) + float2 q0 = abs(p) - halfsz; + // Hors des coins, la distance est celle du rectangle vif quel que soit l'exposant : un seul + // axe déborde de l'étendue E, qui s'y ajoute puis s'en retranche. E ne dépasse pas 1,42 r + // (n = 3) : sous 1,5 r du bord sur un axe, on rend la boîte sans calculer `n` ni E, deux + // transcendantes de moins sur presque tous les pixels d'un calque arrondi. + if (r <= 0.0 || hmin <= 0.0 || min(q0.x, q0.y) <= -min(1.5 * r, hmin)) { - float2 q0 = abs(p) - halfsz; return length(max(q0, 0.0)) + min(max(q0.x, q0.y), 0.0); } float n = 3.0 - smoothstep(0.5, 1.0, r / hmin); @@ -463,6 +478,19 @@ float band_cov(float x, float half_w) return saturate(half_w + 0.5 - abs(x)); } +// Camera homography (`persp`, read when `layer_fx.y` = 1): the camera point (0..1 of its valid +// frame) seen at `local` (0..1 in the layer's dst quad), and in z whether there is one -- 0 when +// the point is behind the projection (q.z <= 0) or outside the camera frame, where the layer is +// black. +float3 persp_camera(float2 local) +{ + float3 p = float3(local, 1.0); + float3 q = float3(dot(persp[0].xyz, p), dot(persp[1].xyz, p), dot(persp[2].xyz, p)); + float2 cam = q.xy / max(q.z, 1e-6); + bool inside = q.z > 0.0 && all(cam >= 0.0) && all(cam <= 1.0); + return float3(cam, inside ? 1.0 : 0.0); +} + // Fond flouté pour le mode "blur" de la webcam. // Disque de Vogel (spirale à angle d'or) à 21 échantillons avec pondération gaussienne et // rotation par pixel via Interleaved Gradient Noise (IGN) pour un bokeh photographique doux, isotrope et rapide. @@ -490,10 +518,15 @@ static const float3 VOGEL_TAPS[21] = { float3(-0.633036, -0.758588, 0.087119) }; -float3 blur_webcam_bg(float2 uv, float intensity, float2 qpx, float2 local_px) +// Vogel-disc blur of the camera texture at a radius in quad pixels. `valid` is the part of the +// texture the picture fills (fx.xy): decoders allocate aligned textures (a 1080-line camera in a +// 1088-line texture), so each tap is clamped half a chroma texel inside it, never into padding. +float3 blur_webcam_radius(float2 uv, float max_r_px, float2 qpx, float2 local_px, float2 valid) { - float max_r_px = max(intensity, 0.0) * 22.0 + 1.5; float2 step = max_r_px / max(qpx, 1.0); + float cw, ch; + texUV.GetDimensions(cw, ch); + float2 hi = max(valid - 0.5 / max(float2(cw, ch), 1.0), 0.0); // Interleaved Gradient Noise pour rotation aléatoire par pixel float noise = frac(52.9829189 * frac(0.06711056 * local_px.x + 0.00583715 * local_px.y)); float angle = noise * 6.2831853; @@ -506,12 +539,17 @@ float3 blur_webcam_bg(float2 uv, float intensity, float2 qpx, float2 local_px) float2 p = VOGEL_TAPS[k].xy; float w = VOGEL_TAPS[k].z; float2 rot_p = float2(p.x * c - p.y * s, p.x * s + p.y * c); - sum += sample_yuv(saturate(uv + rot_p * step)) * w; + sum += sample_yuv(clamp(uv + rot_p * step, 0.0, hi)) * w; total += w; } return sum / max(total, 1e-4); } +float3 blur_webcam_bg(float2 uv, float intensity, float2 qpx, float2 local_px, float2 valid) +{ + return blur_webcam_radius(uv, max(intensity, 0.0) * 22.0 + 1.5, qpx, local_px, valid); +} + // ============ Curseur MODÉLISÉ (mode 15) ============ // L'état courant en objet 3D, lancé de rayons par pixel. Deux sortes d'objets : // - un curseur SCULPTÉ (`trail_a.x` > 0) : la flèche ou la main d'un des cinq thèmes d'origine, @@ -587,13 +625,12 @@ static const float MODEL_CONTACT_ALPHA = 0.5; // curseur 1, x à droite, y VERS LE HAUT, z vers la caméra, l'écran en z = 0 et le modèle posé à // SCULPT_HOVER au-dessus. `sculpt_point` y amène un point du repère du modèle. Identifiant : // 1 + 2 × thème + forme ; thèmes 0 Studio Ink, 1 Prism Glow, 2 Pop Coral, 3 Pixel Candy, -// 4 Star Sprout ; formes 0 flèche, 1 main. Id des matières : 1 corps, 2 liseré ou manchette, -// 3 étoile, 4 feuilles, 5 yeux, 6 face des voxels, 7 couche violette, 8 cristal. +// 4 Star Sprout ; formes 0 flèche, 1 main. Id des matières : 1 corps, 2 bande, calque ou +// manchette, 3 tirets, calque corail ou étoile, 4 feuilles, 5 trait des thèmes cerclés (et +// yeux), 6 cubes de Pixel Candy, 8 serti de Prism Glow, 9 son cristal (plus bas, `prism_*`). static const float SCULPT_SCALE = 0.85; static const float SCULPT_HOVER = 0.05; static const float SCULPT_VOX = 0.0625; -static const float SCULPT_AR_ROUND = 0.03; -static const float SCULPT_HAND_ZC = 0.185; // Hauteur, dans le prototype, du z = 0 du modèle : le dessus de la pointe de la flèche, l'axe du // bout de l'index. static const float SCULPT_ZREF_ARROW = 0.2; @@ -619,13 +656,6 @@ float2 s_opu(float2 a, float2 b) return a.x < b.x ? a : b; } -float s_capsule(float3 p, float3 a, float3 b, float r) -{ - float3 pa = p - a, ba = b - a; - float h = saturate(dot(pa, ba) / dot(ba, ba)); - return length(pa - ba * h) - r; -} - float s_round_box(float3 p, float3 b, float r) { float3 q = abs(p) - b + r; @@ -652,25 +682,91 @@ float2 s_rot(float2 v, float a) return float2(c * v.x - s * v.y, s * v.x + c * v.y); } -// La flèche : polygone à sept sommets, pointe à l'origine. -static const float2 SCULPT_ARROW[7] = { - float2(0.0, 0.0), float2(0.0, -0.86), float2(0.215, -0.665), float2(0.37, -1.0), - float2(0.53, -0.93), float2(0.38, -0.60), float2(0.64, -0.60) +float s_star5(float2 p, float r, float rf) +{ + const float2 k1 = float2(0.809016994375, -0.587785252292); + const float2 k2 = float2(-0.809016994375, -0.587785252292); + p.x = abs(p.x); + p -= 2.0 * max(dot(k1, p), 0.0) * k1; + p -= 2.0 * max(dot(k2, p), 0.0) * k2; + p.x = abs(p.x); + p.y -= r; + float2 ba = rf * float2(-k1.y, k1.x) - float2(0.0, 1.0); + float h = clamp(dot(p, ba) / dot(ba, ba), 0.0, r); + return length(p - ba * h) * sign(p.y * ba.x - p.x * ba.y); +} + +// Lentille : deux disques de rayon r aux centres à ±d sur x, une feuille pointue à ses deux bouts +// (0, ±sqrt(r² - d²)). +float s_vesica(float2 p, float r, float d) +{ + p = abs(p); + float b = sqrt(r * r - d * d); + return (p.y - b) * d > p.x * b ? length(p - float2(0.0, b)) : length(p + float2(d, 0.0)) - r; +} + +// Les thèmes CERCLÉS (Studio Ink, Pop Coral, Star Sprout) suivent la planche 3D de référence +// (`design/cursors/3d-concept.png`) : le trait de leur dessin 2D n'est pas jeté, il devient la +// matière qui porte tout. Chaque pièce (la flèche, le gant, la manchette, les feuilles, l'étoile) +// est un plateau sous sa silhouette, un jonc sur le trait et un coussin de couleur bombé dedans. +// RIM_POLY : sept sommets par polygone (un sommet coupe un côté en deux quand il en faut six). +// D'abord la couleur de la flèche, commune aux trois thèmes (le trait l'élargit de 0,06, sa pointe +// arrondie touche le hotspot), puis les paumes de Star Sprout, de Studio Ink et de Pop Coral, +// arrondies de 0,05. +static const float2 RIM_POLY[28] = { + float2(0.0, -0.06), float2(0.0, -0.7712), float2(0.165, -0.6325), float2(0.313, -0.9318), + float2(0.4234, -0.8557), float2(0.2672, -0.5686), float2(0.456, -0.5393), + float2(-0.173, -0.5237), float2(-0.016, -0.75), float2(0.329, -0.75), float2(0.411, -0.592), + float2(0.411, -0.43), float2(0.19, -0.435), float2(-0.03, -0.44), + float2(-0.1825, -0.6215), float2(0.0037, -0.9195), float2(0.3462, -0.9195), float2(0.467, -0.6906), + float2(0.467, -0.49), float2(0.2, -0.49), float2(-0.03, -0.49), + float2(-0.1807, -0.5534), float2(0.0194, -0.872), float2(0.3485, -0.872), float2(0.47, -0.6795), + float2(0.47, -0.49), float2(0.2, -0.49), float2(-0.03, -0.49) +}; + +// Les trois gants, mesurés sur leur dessin (g : 0 Star Sprout, 1 Studio Ink, 2 Pop Coral), neuf +// lignes chacun : les segments de l'index, des trois doigts repliés et du pouce, ceux des trois +// fentes entre les doigts, puis le V entre le pouce et l'index (un point du bord du pouce et la +// normale qui entre dans le V). RIM_GLOVE_R, trois lignes par gant : les rayons des quatre doigts ; +// ceux du pouce et des trois fentes ; le bord de l'index que longe le V, le fond du V, le trait. +static const float4 RIM_GLOVE[27] = { + float4(0.0, -0.1077, 0.0, -0.6), float4(0.1493, -0.3279, 0.1493, -0.6), + float4(0.2863, -0.3654, 0.2863, -0.6), float4(0.4193, -0.4043, 0.4193, -0.6), + float4(-0.2, -0.478, -0.075, -0.625), float4(0.0788, -0.25, 0.0788, -0.3992), + float4(0.2188, -0.3, 0.2188, -0.4234), float4(0.3552, -0.33, 0.3552, -0.4534), + float4(-0.0845, -0.5386, 0.762, 0.648), + float4(0.0, -0.1186, 0.0, -0.65), float4(0.1614, -0.3838, 0.1614, -0.65), + float4(0.3177, -0.427, 0.3177, -0.65), float4(0.4645, -0.4848, 0.4645, -0.65), + float4(-0.215, -0.535, -0.1208, -0.6864), float4(0.0832, -0.3, 0.0832, -0.48), + float4(0.2414, -0.35, 0.2414, -0.482), float4(0.3959, -0.41, 0.3959, -0.533), + float4(-0.122, -0.5718, 0.867, 0.498), + float4(0.0, -0.122, 0.0, -0.65), float4(0.172, -0.387, 0.172, -0.65), + float4(0.3267, -0.427, 0.3267, -0.65), float4(0.4727, -0.4867, 0.4727, -0.65), + float4(-0.199, -0.483, -0.102, -0.645), float4(0.0927, -0.3, 0.0927, -0.476), + float4(0.2493, -0.35, 0.2493, -0.478), float4(0.402, -0.41, 0.402, -0.52), + float4(-0.0993, -0.5267, 0.858, 0.514) +}; +static const float4 RIM_GLOVE_R[9] = { + float4(0.0552, 0.047, 0.045, 0.0375), float4(0.052, 0.0235, 0.0225, 0.0225), float4(0.0552, -0.548, 0.0525, 0.0), + float4(0.064, 0.0585, 0.0585, 0.0515), float4(0.06, 0.0197, 0.0178, 0.0172), float4(0.0648, -0.6, 0.058, 0.0), + float4(0.066, 0.0553, 0.0553, 0.0507), float4(0.063, 0.024, 0.022, 0.02), float4(0.0647, -0.555, 0.056, 0.0) }; -float s_arrow2(float2 p) +// Distance signée au polygone de RIM_POLY qui commence en `base` : distance aux arêtes, signe par +// la parité des traversées d'une demi-droite. +float s_rim_poly(float2 p, int base) { - float d = dot(p - SCULPT_ARROW[0], p - SCULPT_ARROW[0]); + float d = dot(p - RIM_POLY[base], p - RIM_POLY[base]); float s = 1.0; - int j = 6; - [loop] for (int i = 0; i < 7; i++) + int j = base + 6; + [loop] for (int i = base; i < base + 7; i++) { - float2 e = SCULPT_ARROW[j] - SCULPT_ARROW[i]; - float2 w = p - SCULPT_ARROW[i]; + float2 e = RIM_POLY[j] - RIM_POLY[i]; + float2 w = p - RIM_POLY[i]; float2 b = w - e * saturate(dot(w, e) / dot(e, e)); d = min(d, dot(b, b)); - bool c0 = p.y >= SCULPT_ARROW[i].y; - bool c1 = p.y < SCULPT_ARROW[j].y; + bool c0 = p.y >= RIM_POLY[i].y; + bool c1 = p.y < RIM_POLY[j].y; bool c2 = e.x * w.y > e.y * w.x; if ((c0 && c1 && c2) || (!c0 && !c1 && !c2)) { @@ -681,311 +777,1262 @@ float s_arrow2(float2 p) return s * sqrt(d); } -float s_star5(float2 p, float r, float rf) +// Le gant `g` dans le plan, autour de sa paume `palm` : (silhouette, couleur). La silhouette unit +// la paume, l'index levé, trois doigts repliés et le pouce ; la couleur en est creusée des +// rainures du dessin, trois fentes entre les doigts et le V entre le pouce et l'index. +float2 s_rim_glove(float2 p, int g, float palm) { - const float2 k1 = float2(0.809016994375, -0.587785252292); - const float2 k2 = float2(-0.809016994375, -0.587785252292); - p.x = abs(p.x); - p -= 2.0 * max(dot(k1, p), 0.0) * k1; - p -= 2.0 * max(dot(k2, p), 0.0) * k2; - p.x = abs(p.x); - p.y -= r; - float2 ba = rf * float2(-k1.y, k1.x) - float2(0.0, 1.0); - float h = clamp(dot(p, ba) / dot(ba, ba), 0.0, r); - return length(p - ba * h) * sign(p.y * ba.x - p.x * ba.y); + float4 r0 = RIM_GLOVE_R[3 * g]; + float4 r1 = RIM_GLOVE_R[3 * g + 1]; + float4 r2 = RIM_GLOVE_R[3 * g + 2]; + int i = 9 * g; + float f = min(sd_segment(p, RIM_GLOVE[i].xy, RIM_GLOVE[i].zw) - r0.x, + sd_segment(p, RIM_GLOVE[i + 1].xy, RIM_GLOVE[i + 1].zw) - r0.y); + f = min(f, sd_segment(p, RIM_GLOVE[i + 2].xy, RIM_GLOVE[i + 2].zw) - r0.z); + f = min(f, sd_segment(p, RIM_GLOVE[i + 3].xy, RIM_GLOVE[i + 3].zw) - r0.w); + f = min(f, sd_segment(p, RIM_GLOVE[i + 4].xy, RIM_GLOVE[i + 4].zw) - r1.x); + float grooves = min(sd_segment(p, RIM_GLOVE[i + 5].xy, RIM_GLOVE[i + 5].zw) - r1.y, + sd_segment(p, RIM_GLOVE[i + 6].xy, RIM_GLOVE[i + 6].zw) - r1.z); + grooves = min(grooves, sd_segment(p, RIM_GLOVE[i + 7].xy, RIM_GLOVE[i + 7].zw) - r1.w); + float sil = s_smin(palm, f, 0.03); + float4 v = RIM_GLOVE[i + 8]; + float wedge = max(max(-dot(p - v.xy, v.zw), p.x + r2.x), r2.y - p.y); + return float2(sil, max(sil, -min(grooves, wedge))); } -// La flèche en volume : extrusion arrondie, bombée d'un dôme qui plafonne avant l'axe médian de -// la silhouette (un dôme qui monterait encore y plierait le dessus). -float s_arrow_solid(float3 p, float h, float re, float dome) +// Une pièce cerclée : (distance, matière). `d` : le bord intérieur du trait dans le plan, `sil` : +// le même avant les rainures, `dc` : le coussin. Le plateau (matière 5) monte de SCULPT_HOVER à +// `zt` sous la silhouette élargie du trait `w`. Le jonc (matière 5), un tore de rayon w/2 à la +// hauteur `zt`, suit le milieu du trait : dans une rainure plus étroite que le trait il n'en reste +// que la crête, un muret entre les doigts. Le coussin de la matière `mat` sort du plateau, haut de +// `h` au bord, bombé sur 0,045 puis de `bump` ; sa hauteur variable penche le champ, d'où la +// distance minorée (× 0,8). +float2 s_piece(float d, float dc, float sil, float z, float w, float zt, float h, float bump, float mat) { - float d2 = s_arrow2(p.xy) - SCULPT_AR_ROUND; - float hh = h + dome * smoothstep(0.0, 0.07, -d2); - return s_extrude(d2, p.z - (SCULPT_HOVER + h + dome), hh, re); + float tray = s_extrude(sil - w, z - 0.5 * (SCULPT_HOVER + zt), 0.5 * (zt - SCULPT_HOVER), 0.012); + float bead = length(float2(d - 0.5 * w, z - zt)) - 0.5 * w; + float u = saturate(-dc / 0.045); + float cushion = 0.8 * s_extrude(dc, z - zt, h + 0.022 * u * (2.0 - u) + bump, 0.75 * h); + return s_opu(float2(min(tray, bead), 5.0), float2(cushion, mat)); } -// Le liseré de Studio Ink : un jonc posé sur le dessus, en retrait du bord. -float s_piping(float3 p, float ztop) +// Pop Coral est du papier découpé : des feuilles à plat, au dessus plat et au bord à peine +// cassé. La feuille marine sous la silhouette élargie du trait `w`, de SCULPT_HOVER à 0,15 ; la +// feuille de couleur (matière 1) posée dessus, 0,02 d'épaisseur, découpée des rainures entre les +// doigts qui laissent voir la marine. +float2 s_paper(float d, float sil, float z, float w) { - float d2 = s_arrow2(p.xy) - SCULPT_AR_ROUND; - return length(float2(d2 + 0.075, p.z - ztop)) - 0.017; + float navy = s_extrude(sil - w, z - 0.5 * (SCULPT_HOVER + 0.15), 0.5 * (0.15 - SCULPT_HOVER), 0.006); + float sheet = s_extrude(d, z - 0.16, 0.01, 0.004); + return s_opu(float2(navy, 5.0), float2(sheet, 1.0)); } -// Le gant : l'index levé, trois doigts repliés, le pouce, fondus dans la paume. -float s_glove(float3 p, float zc) +// Bosse de rayon r en c, 1 au centre et 0 au bord : un coussin bombé sans pli. Une hauteur tirée +// de la distance au bord plierait le coussin sur l'axe médian de sa silhouette. +float s_bump(float2 p, float2 c, float r) { - float index = s_capsule(p, float3(0.0, -0.10, zc), float3(0.0, -0.52, zc), 0.098); - float palm = s_round_box(p - float3(0.185, -0.70, zc), float3(0.255, 0.19, 0.105), 0.1); - float f1 = s_capsule(p, float3(0.17, -0.57, zc + 0.012), float3(0.17, -0.41, zc + 0.045), 0.086); - float f2 = s_capsule(p, float3(0.31, -0.59, zc + 0.01), float3(0.31, -0.45, zc + 0.04), 0.08); - float f3 = s_capsule(p, float3(0.435, -0.625, zc + 0.005), float3(0.435, -0.52, zc + 0.03), 0.07); - float thumb = s_capsule(p, float3(0.03, -0.77, zc + 0.03), float3(-0.165, -0.60, zc + 0.065), 0.082); - float d = s_smin(palm, min(f1, min(f2, f3)), 0.035); - d = s_smin(d, index, 0.05); - return s_smin(d, thumb, 0.05); + float k = saturate(1.0 - dot(p - c, p - c) / (r * r)); + return k * k; } -// La manchette, ronde autour du poignet (elliptique en xz) : le gant y entre. -float s_cuff(float3 p, float zc) +// Un tiret du clic de Pop Coral : capsule inégale de a (rayon ra) à b (rayon rb), exacte. +float s_dash(float2 p, float2 a, float2 b, float ra, float rb) { - float3 q = p - float3(0.185, -0.925, zc); - float2 ab = float2(0.272, 0.12); - float e = (length(q.xz / ab) - 1.0) * min(ab.x, ab.y); - return s_extrude(e, q.y, 0.07, 0.06); + p -= a; + b -= a; + float hb = dot(b, b); + float2 q = float2(abs(dot(p, float2(b.y, -b.x))), dot(p, b)) / hb; + float2 c = float2(sqrt(hb - (ra - rb) * (ra - rb)), ra - rb); + float k = c.x * q.y - c.y * q.x; + if (k < 0.0) + { + return sqrt(hb * dot(q, q)) - ra; + } + if (k > c.x) + { + return sqrt(hb * (dot(q, q) + 1.0 - 2.0 * q.y)) - rb; + } + return dot(c, q) - ra; } -// L'étoile de Star Sprout, face à la caméra, centrée en c : son visage et ses deux feuilles. -float2 s_sprout(float3 p, float3 c) +// Les thèmes cerclés. La flèche : Star Sprout menthe, Pop Coral corail, et Studio Ink noir, dont le +// trait se partage en un jonc noir au bord et une bande ivoire en relief, le champ restant le +// dessus du plateau. Le gant ivoire, jaune ou ivoire. Pop Coral est en papier découpé +// (`s_paper`), ses calques et ses tirets aussi : des feuilles au dessus plat. Puis ce que chacun ajoute devant : le calque +// jaune de la flèche, le calque corail du gant et les tirets du clic de Pop Coral ; la manchette, +// l'étoile, ses feuilles et ses yeux de Star Sprout, lus dans le repère de l'étoile (centre `c`, +// tournée, en unités de son rayon `rs`). Matières : 1 corps, 2 bande ivoire, calque jaune ou +// manchette, 3 tirets, calque corail ou étoile, 4 feuilles, 5 le trait (plateaux, joncs, yeux). +float2 s_rimmed(float3 p, int theme, int shape) { - float3 q = p - c; - float st2 = s_star5(q.xy, 0.125, 0.52) - 0.022; - float2 r = float2(s_extrude(st2, q.z, 0.038, 0.034), 3.0); - float3 e = float3(abs(q.x) - 0.032, q.y + 0.005, q.z - 0.036); - r = s_opu(r, float2(length(e) - 0.0135, 5.0)); - float3 l1 = q - float3(-0.04, 0.15, 0.0); - l1.xy = s_rot(l1.xy, -0.6); - float3 l2 = q - float3(0.045, 0.155, 0.0); - l2.xy = s_rot(l2.xy, 0.7); - float leaves = min(s_ellipsoid(l1, float3(0.03, 0.058, 0.02)), s_ellipsoid(l2, float3(0.03, 0.058, 0.02))); - return s_opu(r, float2(leaves, 4.0)); + bool arrow = shape == 0; + int g = theme == 4 ? 0 : (theme == 0 ? 1 : 2); + float poly = s_rim_poly(p.xy, arrow ? 0 : 7 * g + 7); + float2 body = float2(poly, poly); + float w = 0.06, zt = 0.17, h = 0.03; + float bump = 0.022 * s_bump(p.xy, float2(0.15, -0.45), 0.3); + float back = 1e9; + if (!arrow) + { + w = RIM_GLOVE_R[3 * g + 2].z; + // Le gant, puis pour Pop Coral son calque corail : le même gant décalé en bas à gauche. Une + // boucle et non deux appels (FXC recopierait le gant) ; le calque est sauté quand `p` est + // au-dessus du plateau du gant, qui en est alors plus près que lui. + [loop] for (int i = 0; i < 2; i++) + { + float2 q = p.xy + (i == 1 ? float2(0.03, 0.04) : float2(0.0, 0.0)); + float2 v = s_rim_glove(q, g, (i == 1 ? s_rim_poly(q, 7 * g + 7) : poly) - 0.05); + if (i == 1) + { + back = v.x - w; + } + else + { + body = v; + if (theme != 2 || v.x - w <= 0.0) + { + break; + } + } + } + zt = 0.185; + h = 0.028; + bump = 0.03 * s_bump(p.xy, float2(0.19, -0.62), 0.28); + } + float dc = body.y; + if (arrow && theme == 0) + { + // Studio Ink : pas de coussin, le jonc sur le bord extérieur du trait. + body -= 0.015; + w = 0.045; + dc = 1.0; + } + float2 r = theme == 2 ? s_paper(body.y, body.x, p.z, w) + : s_piece(body.y, dc, body.x, p.z, w, zt, h, bump, 1.0); + if (theme == 0) + { + if (arrow) + { + float band = s_extrude(abs(poly + 0.0075) - 0.0225, p.z - zt, 0.028, 0.012); + r = s_opu(r, float2(band, 2.0)); + } + return r; + } + if (theme == 2) + { + float dash; + if (arrow) + { + // Le calque jaune, décalé en bas à gauche, derrière le plateau. + float back = s_rim_poly(p.xy + float2(0.025, 0.05), 0) - 0.06; + r = s_opu(r, float2(s_extrude(back, p.z - 0.08, 0.03, 0.006), 2.0)); + dash = min(s_dash(p.xy, float2(-0.1179, -0.1374), float2(-0.1799, -0.0443), 0.028, 0.045), + s_dash(p.xy, float2(-0.1347, -0.2493), float2(-0.248, -0.2056), 0.026, 0.042)); + } + else + { + r = s_opu(r, float2(s_extrude(back, p.z - 0.08, 0.03, 0.006), 3.0)); + dash = min(s_dash(p.xy, float2(0.216, -0.1695), float2(0.2593, -0.0685), 0.028, 0.042), + s_dash(p.xy, float2(0.3054, -0.2339), float2(0.3949, -0.1652), 0.0275, 0.041)); + } + return s_opu(r, float2(s_extrude(dash, p.z - 0.11, 0.06, 0.006), 3.0)); + } + if (!arrow) + { + // La manchette : un rectangle arrondi, cintré en sourire comme au dessin. + float2 cq = p.xy - float2(0.1541, -0.9047); + float2 bq = abs(float2(cq.x, cq.y - 0.2066 * cq.x * cq.x)) - float2(0.214, 0.0361); + float cuff = length(max(bq, 0.0)) + min(max(bq.x, bq.y), 0.0) - 0.03; + r = s_opu(r, s_piece(cuff, cuff, cuff, p.z, 0.0477, 0.225, 0.025, 0.0, 2.0)); + } + float2 c = arrow ? float2(0.6118, -0.8079) : float2(0.15, -0.885); + float rs = arrow ? 0.1678 : 0.128; + float zs = arrow ? 0.23 : 0.285; + float2 q = s_rot(p.xy - c, arrow ? 0.2443 : 0.0) / rs; + float star = (s_star5(q, 0.82, 0.55) - 0.18) * rs; + float leaves = min(s_vesica(s_rot(q - float2(-0.33, 1.38), -0.925), 0.4296, 0.2626), + s_vesica(s_rot(q - float2(0.53, 1.37), 0.873), 0.4296, 0.2626)) * rs; + r = s_opu(r, s_piece(leaves, leaves, leaves, p.z, 0.038, zs - 0.02, 0.018, 0.0, 4.0)); + r = s_opu(r, s_piece(star, star, star, p.z, 0.038, zs, 0.022, 0.018 * s_bump(q, float2(0.0, 0.0), 1.0), 3.0)); + // Les yeux : deux ovales marine qui affleurent du coussin. + float3 e = float3(abs(q.x) - 0.25, q.y - 0.08, (p.z - zs - 0.056) / rs); + return s_opu(r, float2(s_ellipsoid(e, float3(0.075, 0.13, 0.1)) * rs, 5.0)); } -// Pixel Candy : une ligne par entier, bit c = colonne c, ligne 0 juste sous la pointe. La flèche -// est le polygone échantillonné à 16 cellules par unité (`sculpt.rs` dit comment la régénérer) ; -// la main est dessinée à la main, un gant échantillonné fondant ses doigts. Les tables *BACK -// portent la couche violette, la face dilatée d'une cellule, décalée d'une ligne et d'une colonne. -static const int SCULPT_ARROWPIX[16] = { 0, 1, 3, 7, 31, 63, 127, 255, 511, 63, 55, 115, 113, 224, 224, 64 }; -static const int SCULPT_ARROWBACK[18] = { - 3, 7, 15, 31, 63, 127, 255, 511, 1023, 2047, 4095, 255, 511, 511, 999, 995, 960, 192 +// Pixel Candy est un pixel art, dessiné une fois : la même grille donne ses PNG 2D et, ici, un cube +// par pixel. Une ligne par entier, bit c = colonne c, ligne 0 en haut. `PIX_BODY` : les pixels +// pleins ; `PIX_LINE`, `PIX_HI`, `PIX_SHADE` : ceux du contour prune, du reflet rose pâle et de +// l'ombre rose foncé, le reste est rose. `PIX_GRID` : colonnes, lignes et début de chaque forme +// dans les tables ; `PIX_ORIGIN` : le coin haut-gauche du pixel (0, 0) ; `PIX_BOX` (centre, +// demi-côtés) : la boîte de la forme ; `PIX_RECT`, bornés par `PIX_RECT_N` : la forme en +// rectangles, pour l'ombre. Tables générées par scripts/generate-pixel-candy-voxels.mjs, qui +// réécrit les trois shaders et les PNG : c'est là qu'on redessine une grille. +// +static const int PIX_BODY[32] = { 1, 3, 7, 15, 31, 63, 127, 255, 511, 1023, 2047, 127, 247, 243, 480, 192, 48, 120, 120, 120, 504, 4088, 32760, 65534, 65535, 65535, 65534, 32766, 32764, 16380, 16376, 16376 }; +static const int PIX_LINE[32] = { 1, 3, 5, 9, 17, 33, 65, 129, 257, 513, 1985, 73, 149, 147, 288, 192, 48, 72, 72, 72, 456, 3656, 29256, 37454, 32777, 32769, 32770, 16386, 16388, 8196, 8200, 16376 }; +static const int PIX_HI[32] = { 0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 18, 34, 32, 64, 0, 0, 16, 16, 16, 16, 16, 16, 16, 22, 18, 4, 4, 8, 8, 16, 0 }; +static const int PIX_SHADE[32] = { 0, 0, 0, 4, 8, 16, 32, 64, 128, 448, 40, 36, 64, 64, 128, 0, 0, 32, 32, 32, 32, 288, 2336, 18720, 16384, 16384, 16384, 8192, 8192, 4096, 8160, 0 }; +static const int PIX_RECT_N[3] = { 0, 17, 29 }; +static const int4 PIX_GRID[2] = { + int4(11, 16, 0, 0), + int4(16, 16, 16, 0) }; -static const int SCULPT_HANDPIX[15] = { 4, 14, 14, 14, 110, 878, 7022, 7022, 8190, 8191, 8191, 8191, 8190, 4092, 4092 }; -static const int SCULPT_HANDBACK[17] = { - 28, 62, 62, 62, 510, 4094, 32766, 32766, 32766, 32767, 32767, 32767, 32767, 32767, 32766, 16380, 16380 +static const float2 PIX_ORIGIN[2] = { + float2(0.0, 0.0), + float2(-0.3125, 0.0) }; -// La couche violette de la main, une plage de colonnes par ligne (toutes pleines) : la distance -// exacte à son contour, pour l'ombre portée. -static const float2 SCULPT_HANDSPAN[17] = { - float2(2.0, 4.0), float2(1.0, 5.0), float2(1.0, 5.0), float2(1.0, 5.0), float2(1.0, 8.0), - float2(1.0, 11.0), float2(1.0, 14.0), float2(1.0, 14.0), float2(1.0, 14.0), float2(0.0, 14.0), - float2(0.0, 14.0), float2(0.0, 14.0), float2(0.0, 14.0), float2(0.0, 14.0), float2(1.0, 14.0), - float2(2.0, 13.0), float2(2.0, 13.0) +static const float4 PIX_BOX[2] = { + float4(0.3438, -0.5, 0.3438, 0.5), + float4(0.1875, -0.5, 0.5, 0.5) }; +static const float4 PIX_RECT[29] = { + float4(0.0313, -0.0313, 0.0313, 0.0313), + float4(0.0625, -0.0938, 0.0625, 0.0313), + float4(0.0938, -0.1563, 0.0938, 0.0313), + float4(0.125, -0.2188, 0.125, 0.0313), + float4(0.1563, -0.2813, 0.1563, 0.0313), + float4(0.1875, -0.3438, 0.1875, 0.0313), + float4(0.2188, -0.4063, 0.2188, 0.0313), + float4(0.25, -0.4688, 0.25, 0.0313), + float4(0.2813, -0.5313, 0.2813, 0.0313), + float4(0.3125, -0.5938, 0.3125, 0.0313), + float4(0.3438, -0.6563, 0.3438, 0.0313), + float4(0.2188, -0.7188, 0.2188, 0.0313), + float4(0.0938, -0.7813, 0.0938, 0.0313), + float4(0.375, -0.8125, 0.125, 0.0625), + float4(0.0625, -0.8438, 0.0625, 0.0313), + float4(0.4375, -0.9063, 0.125, 0.0313), + float4(0.4375, -0.9688, 0.0625, 0.0313), + float4(0.0, -0.0313, 0.0625, 0.0313), + float4(0.0, -0.1563, 0.125, 0.0938), + float4(0.0625, -0.2813, 0.1875, 0.0313), + float4(0.1563, -0.3438, 0.2813, 0.0313), + float4(0.25, -0.4063, 0.375, 0.0313), + float4(0.2188, -0.4688, 0.4688, 0.0313), + float4(0.1875, -0.5625, 0.5, 0.0625), + float4(0.2188, -0.6563, 0.4688, 0.0313), + float4(0.1875, -0.7188, 0.4375, 0.0313), + float4(0.2188, -0.7813, 0.4063, 0.0313), + float4(0.1875, -0.8438, 0.375, 0.0313), + float4(0.2188, -0.9375, 0.3438, 0.0625) +}; +// -// Origine de la grille : la flèche part de sa pointe, la main de 2,5 cellules à sa gauche. -float2 s_grid_origin(int shape) -{ - return shape == 0 ? float2(0.0, 0.0) : float2(-2.5 * SCULPT_VOX, 0.0); -} - -// Le bit `c` de la ligne `r` d'une table de `rows` lignes et `cols` colonnes. FXC évalue les deux -// côtés d'un `&&` : l'index est ramené dans la table avant la lecture, le test fait le reste. -bool s_bit(int row, int r, int c, int rows, int cols) +// Le bit `c` d'une ligne. FXC évalue les deux côtés d'un `&&` : chaque index est ramené dans sa +// table avant la lecture, les tests de bornes font le reste. +bool s_pix_bit(int m, int c) { - return r >= 0 && r < rows && c >= 0 && c < cols && ((row >> clamp(c, 0, 31)) & 1) == 1; + return ((m >> clamp(c, 0, 31)) & 1) == 1; } -// La cellule `id` (colonne, ligne, y vers le haut) porte-t-elle un voxel de la face ? -bool s_occ(float2 id, int shape) +// Le pixel (c, r) de la forme `shape`, r compté vers le bas depuis la ligne 0, est-il plein ? +bool s_pix_body(int shape, int c, int r) { - int c = (int)id.x; - int r = -(int)id.y - 1; - if (shape == 0) - { - return s_bit(SCULPT_ARROWPIX[clamp(r, 0, 15)], r, c, 16, 16); - } - return s_bit(SCULPT_HANDPIX[clamp(r, 0, 14)], r, c, 15, 13); + int4 g = PIX_GRID[shape]; + return c >= 0 && c < g.x && r >= 0 && r < g.y && s_pix_bit(PIX_BODY[g.z + clamp(r, 0, g.y - 1)], c); } -// … et un bloc de la couche violette. -bool s_occ_back(float2 id, int shape) +// La cellule qui contient `p` (prototype, y vers le haut). +int2 s_pix_cell(float2 p, int shape) { - int c = (int)id.x + 1; - int r = -(int)id.y; - if (shape == 0) - { - return s_bit(SCULPT_ARROWBACK[clamp(r, 0, 17)], r, c, 18, 17); - } - return s_bit(SCULPT_HANDBACK[clamp(r, 0, 16)], r, c, 17, 15); + float2 o = PIX_ORIGIN[shape]; + return int2((int)floor((p.x - o.x) / SCULPT_VOX), (int)floor((o.y - p.y) / SCULPT_VOX)); } -// Chaque cellule pleine est un cube biseauté ; la couche violette est un étage plus bas. Hors du -// voisinage 3×3, une borne : une cellule au moins, et jamais plus près que la boîte de la forme. +// Un cube par pixel plein, tous de même hauteur, l'arête à peine cassée pour que chaque cube se +// lise. Hors du voisinage 3×3, une borne : une cellule au moins, et jamais plus près que la boîte +// de la forme. float2 s_voxels(float3 p, int shape) { - float2 o = s_grid_origin(shape); - float2 cell = floor((p.xy - o) / SCULPT_VOX); - float2 bc = shape == 0 ? float2(0.33, -0.5) : float2(0.25, -0.47); - float2 bh = shape == 0 ? float2(0.44, 0.61) : float2(0.48, 0.55); - float2 bq = max(abs(p.xy - bc) - bh, 0.0); + float4 b = PIX_BOX[shape]; + float2 bq = max(abs(p.xy - b.xy) - b.zw, 0.0); float bz = max(abs(p.z - (SCULPT_HOVER + 0.07)) - 0.07, 0.0); - float far = max(SCULPT_VOX, sqrt(dot(bq, bq) + bz * bz)); - float dF = far; - float dB = far; - const float3 half_cell = float3(0.5 * SCULPT_VOX, 0.5 * SCULPT_VOX, 0.04); + float d = max(SCULPT_VOX, sqrt(dot(bq, bq) + bz * bz)); + float2 o = PIX_ORIGIN[shape]; + int2 cell = s_pix_cell(p.xy, shape); + const float3 cube = float3(0.5 * SCULPT_VOX, 0.5 * SCULPT_VOX, 0.07); [loop] for (int j = -1; j <= 1; j++) { [loop] for (int i = -1; i <= 1; i++) { - float2 id = cell + float2(i, j); - float3 q = float3(p.xy - (o + (id + 0.5) * SCULPT_VOX), p.z); - if (s_occ(id, shape)) - { - dF = min(dF, s_round_box(q - float3(0.0, 0.0, SCULPT_HOVER + 0.1), half_cell, 0.009)); - } - if (s_occ_back(id, shape)) + int c = cell.x + i; + int r = cell.y + j; + if (s_pix_body(shape, c, r)) { - dB = min(dB, s_round_box(q - float3(0.0, 0.0, SCULPT_HOVER + 0.04), half_cell, 0.009)); + float3 q = float3(p.x - o.x - (c + 0.5) * SCULPT_VOX, p.y - o.y + (r + 0.5) * SCULPT_VOX, + p.z - SCULPT_HOVER - 0.07); + d = min(d, s_round_box(q, cube, 0.006)); } } } - return dF < dB ? float2(dF, 6.0) : float2(dB, 7.0); + return float2(d, 6.0); } -// Distance, dans le plan de l'écran, au contour de la couche violette de la main. -float s_pixel_hand_dist(float2 p) +// Distance, dans le plan de l'écran, au contour de la forme. +float s_pixel_outline(float2 p, int shape) { float d = 1e9; - [loop] for (int r = 0; r < 17; r++) + [loop] for (int i = PIX_RECT_N[shape]; i < PIX_RECT_N[shape + 1]; i++) { - float2 x = (SCULPT_HANDSPAN[r] + float2(-3.5, -2.5)) * SCULPT_VOX; - float2 y = float2(-r, 1.0 - r) * SCULPT_VOX; - float2 q = max(max(float2(x.x, y.x) - p, p - float2(x.y, y.y)), 0.0); - d = min(d, length(q)); - } - return d; -} - -// Prism Glow est taillé, pas peint : chaque pièce est un polyèdre convexe (le max de ses plans), -// les facettes sont planes et la silhouette polygonale. - -// Une arête d'un contour convexe de gemme (a -> b, sens trigonométrique) : sa paroi, une facette -// de rondiste raide, et deux facettes de couronne qui se rejoignent au milieu de l'arête. Les -// couronnes de toutes les arêtes se rejoignent en arêtes vives au-dessus du centre. -float s_gem_edge(float3 p, float2 a, float2 b, float z0) -{ - float2 e = b - a; - float len = length(e); - float2 d = e / len; - float2 q = p.xy - a; - float dist = dot(q, float2(-d.y, d.x)); - float along = abs(dot(q, d) - 0.5 * len); - float z = p.z - z0; - float girdle = (z - 0.035 - 2.2 * dist) * 0.4138; - float crown = (z - 0.07 - 0.6 * dist + 0.18 * along) * 0.8438; - return max(-dist, max(girdle, crown)); -} - -float s_gem_arrow(float3 p) -{ - float z0 = SCULPT_HOVER + 0.03; - float slab = max(p.z - z0 - 0.2, z0 - 0.03 - p.z); - float head = max(slab, max(s_gem_edge(p, float2(-0.02, 0.03), float2(-0.02, -0.88), z0), - max(s_gem_edge(p, float2(-0.02, -0.88), float2(0.66, -0.61), z0), - s_gem_edge(p, float2(0.66, -0.61), float2(-0.02, 0.03), z0)))); - float tail = max(slab, max(max(s_gem_edge(p, float2(0.215, -0.665), float2(0.37, -1.0), z0), - s_gem_edge(p, float2(0.37, -1.0), float2(0.53, -0.93), z0)), - max(s_gem_edge(p, float2(0.53, -0.93), float2(0.38, -0.60), z0), - s_gem_edge(p, float2(0.38, -0.60), float2(0.215, -0.665), z0)))); - return min(head, tail); -} - -// Capsule taillée a -> b : un prisme hexagonal coiffé en b de deux couronnes de facettes -// décalées et d'une facette sommitale, tous les plans tangents à la sphère du bout. -float s_facet_capsule(float3 p, float3 a, float3 b, float r, float spin) -{ - float3 u = normalize(b - a); - float3 v = normalize(cross(u, float3(0.0, 0.0, 1.0))); - float3 w = cross(u, v); - float3 q = p - b; - float h = dot(q, u); - float2 rad = float2(dot(q, v), dot(q, w)); - float d = max(h - r, -dot(p - a, u) - r); - [loop] for (int k = 0; k < 6; k++) - { - float an = spin + k * 1.0471976; - float s = dot(rad, float2(cos(an), sin(an))); - d = max(d, s - r); - d = max(d, dot(rad, float2(cos(an + 0.5236), sin(an + 0.5236))) * 0.8660 + h * 0.5 - r); - d = max(d, s * 0.5 + h * 0.8660 - r); + float2 q = abs(p - PIX_RECT[i].xy) - PIX_RECT[i].zw; + d = min(d, length(max(q, 0.0)) + min(max(q.x, q.y), 0.0)); } return d; } -// Ellipsoïde taillé : les plans tangents dans dix directions du premier octant, reportées dans -// les huit. Une pierre aux facettes de taille égale. -static const float3 SCULPT_FACETS[10] = { - float3(1.0, 0.0, 0.0), float3(0.0, 1.0, 0.0), float3(0.0, 0.0, 1.0), - float3(0.7071, 0.7071, 0.0), float3(0.7071, 0.0, 0.7071), float3(0.0, 0.7071, 0.7071), - float3(0.5774, 0.5774, 0.5774), float3(0.4472, 0.0, 0.8944), float3(0.0, 0.4472, 0.8944), - float3(0.3015, 0.3015, 0.9045) +// ---- Prism Glow : un cristal en MAILLAGE ---- +// Les facettes tracées sur l'art 2D du thème (`design/cursors/prism-glow/model`), en triangles dans +// le repère du MODÈLE lui-même (unité = côté du sprite 128 px, origine au hotspot, y vers le bas, +// dessus du serti en z = 0). Le serti marine est la silhouette de l'art extrudée sous z = 0, un +// champ de distance comme les autres sculptés : il donne la couverture, l'ombre portée et l'ombre +// de contact. Le cristal est lancé de rayons, triangles rangés par boîtes : réfraction par canal +// (dispersion), réflexions totales internes, sortie par son fond plat vers l'image composée sous +// le curseur, flous de confidentialité compris (sa copie en t5). Tables : miroir de `prism_mesh.rs` ; +// un triangle = quatre +// float4 (v0, drapeaux), (e1, r), (e2, g), (normale sortante, b) ; drapeaux = paroi (1) + 2 × +// arêtes réelles (bit k : l'arête opposée au sommet k) ; une boîte = deux float4 (coin bas, +// premier triangle), (coin haut, nombre de triangles). +// prism mesh: generated by design/cursors/prism-glow/model/export_compositor.py +static const int PRISM_TRI_START[2] = { 0, 31 }; +static const int PRISM_TRI_COUNT[2] = { 31, 130 }; +static const int PRISM_SIL_START[2] = { 0, 25 }; +static const int PRISM_SIL_COUNT[2] = { 25, 42 }; +static const int PRISM_OUT_START[2] = { 67, 76 }; +static const int PRISM_OUT_COUNT[2] = { 9, 34 }; +static const int PRISM_BOX_START[2] = { 0, 4 }; +static const int PRISM_BOX_COUNT[2] = { 4, 18 }; +static const float4 PRISM_TRIS[644] = { + float4(0.252301, 0.266258, -0.042945, 11.0), + float4(-0.221166, -0.203988, 0.0, 0.01033), + float4(-0.221166, -0.203988, 0.042945, 0.045182), + float4(0.677984, -0.735077, 0.0, 0.577584), + float4(0.252301, 0.266258, -0.042945, 3.0), + float4(-0.221166, -0.203988, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(0.677984, -0.735077, 0.0, 0.577584), + float4(0.031135, 0.06227, -0.042945, 7.0), + float4(0.004294, 0.419785, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(-0.999948, 0.01023, 0.0, 0.577584), + float4(0.169632, 0.356442, 0.115951, 14.0), + float4(0.082669, -0.090184, -0.115951, 0.590616), + float4(-0.138497, -0.294172, -0.115951, 0.737911), + float4(0.466385, -0.505659, 0.725806, 0.973449), + float4(0.035429, 0.482055, 0.0, 14.0), + float4(0.134202, -0.125614, 0.115951, 0.000305), + float4(-0.004294, -0.419785, 0.0, 0.938689), + float4(-0.650191, 0.006652, 0.759742, 0.854994), + float4(0.031135, 0.06227, -0.042945, 9.0), + float4(0.004294, 0.419785, 0.0, 0.01033), + float4(0.004294, 0.419785, 0.042945, 0.045182), + float4(-0.999948, 0.01023, 0.0, 0.577584), + float4(0.169632, 0.356442, 0.115951, 14.0), + float4(0.159969, 0.080521, -0.045092, 0.381323), + float4(0.082669, -0.090184, -0.115951, 0.056123), + float4(0.461407, -0.510214, 0.725799, 0.973449), + float4(0.541104, 0.544325, -0.042945, 9.0), + float4(-0.288804, -0.278068, 0.0, 0.01033), + float4(-0.288804, -0.278068, 0.042945, 0.045182), + float4(0.69359, -0.72037, 0.0, 0.577584), + float4(0.329601, 0.436963, 0.070859, 14.0), + float4(0.211503, 0.107362, -0.070859, 0.545724), + float4(-0.077301, -0.170706, -0.070859, 0.064805), + float4(0.49567, -0.514808, 0.699488, 0.973449), + float4(0.268405, 0.533589, 0.081595, 14.0), + float4(0.061196, -0.096626, -0.010736, 0.000916), + float4(-0.098773, -0.177147, 0.034356, 0.046662), + float4(0.247834, 0.049458, 0.967539, 0.904662), + float4(0.541104, 0.544325, -0.042945, 7.0), + float4(-0.288804, -0.278068, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(0.69359, -0.72037, 0.0, 0.577584), + float4(0.199693, 0.595859, 0.0, 14.0), + float4(0.068712, -0.06227, 0.081595, 0.0), + float4(-0.030061, -0.239417, 0.115951, 0.01445), + float4(-0.504461, 0.426837, 0.750553, 0.254154), + float4(0.268405, 0.533589, 0.081595, 14.0), + float4(0.272699, 0.010736, -0.081595, 0.0), + float4(0.061196, -0.096626, -0.010736, 0.025193), + float4(0.283244, 0.073137, 0.956255, 0.527108), + float4(0.035429, 0.482055, 0.0, 14.0), + float4(0.0, 0.258742, 0.0, 0.0), + float4(0.134202, -0.125614, 0.115951, 0.473533), + float4(-0.653777, 0.0, 0.756687, 0.964691), + float4(0.359663, 0.559356, 0.0, 14.0), + float4(0.181442, -0.015031, 0.0, 0.006043), + float4(-0.091258, -0.025767, 0.081595, 0.076188), + float4(0.076465, 0.923048, 0.37701, 0.964691), + float4(0.035429, 0.482055, -0.042945, 3.0), + float4(0.0, 0.258742, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(-1.0, 0.0, 0.0, 0.577584), + float4(0.035429, 0.482055, -0.042945, 11.0), + float4(0.0, 0.258742, 0.0, 0.01033), + float4(0.0, 0.258742, 0.042945, 0.045182), + float4(-1.0, 0.0, 0.0, 0.577584), + float4(0.035429, 0.740798, -0.042945, 7.0), + float4(0.164264, -0.144939, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(0.661622, 0.749838, 0.0, 0.577584), + float4(0.035429, 0.740798, 0.0, 14.0), + float4(0.164264, -0.144939, 0.0, 0.0), + float4(0.134202, -0.384356, 0.115951, 0.010956), + float4(0.332572, 0.376915, 0.864483, 0.215854), + float4(0.035429, 0.740798, -0.042945, 11.0), + float4(0.164264, -0.144939, 0.0, 0.01033), + float4(0.164264, -0.144939, 0.042945, 0.045182), + float4(0.661622, 0.749838, 0.0, 0.577584), + float4(0.199693, 0.595859, -0.042945, 7.0), + float4(0.105215, 0.22546, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(-0.906183, 0.422886, 0.0, 0.577584), + float4(0.199693, 0.595859, 0.0, 14.0), + float4(0.105215, 0.22546, 0.0, 0.0), + float4(0.068712, -0.06227, 0.081595, 0.015991), + float4(-0.613882, 0.286478, 0.735581, 0.473533), + float4(0.199693, 0.595859, -0.042945, 11.0), + float4(0.105215, 0.22546, 0.0, 0.01033), + float4(0.105215, 0.22546, 0.042945, 0.045182), + float4(-0.906183, 0.422886, 0.0, 0.577584), + float4(0.304908, 0.821319, 0.0, 14.0), + float4(0.128834, -0.067638, 0.0, 0.0), + float4(-0.036503, -0.28773, 0.081595, 0.254154), + float4(0.133686, 0.254641, 0.957751, 0.973449), + float4(0.304908, 0.821319, -0.042945, 7.0), + float4(0.128834, -0.067638, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(0.464834, 0.885398, 0.0, 0.577584), + float4(0.433742, 0.753681, 0.0, 14.0), + float4(-0.07408, -0.194325, 0.0, 0.0), + float4(-0.165337, -0.220092, 0.081595, 0.964691), + float4(0.683362, -0.260508, 0.682021, 0.964691), + float4(0.433742, 0.753681, -0.042945, 11.0), + float4(-0.07408, -0.194325, 0.0, 0.01033), + float4(-0.07408, -0.194325, 0.042945, 0.045182), + float4(0.934406, -0.35621, 0.0, 0.577584), + float4(0.304908, 0.821319, -0.042945, 11.0), + float4(0.128834, -0.067638, 0.0, 0.01033), + float4(0.128834, -0.067638, 0.042945, 0.045182), + float4(0.464834, 0.885398, 0.0, 0.577584), + float4(0.433742, 0.753681, -0.042945, 7.0), + float4(-0.07408, -0.194325, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(0.934406, -0.35621, 0.0, 0.577584), + float4(0.359663, 0.559356, -0.042945, 7.0), + float4(0.181442, -0.015031, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(0.082557, 0.996586, 0.0, 0.577584), + float4(0.359663, 0.559356, -0.042945, 11.0), + float4(0.181442, -0.015031, 0.0, 0.01033), + float4(0.181442, -0.015031, 0.042945, 0.045182), + float4(0.082557, 0.996586, 0.0, 0.577584), + float4(0.044345, 0.043263, -0.043263, 11.0), + float4(-0.043263, -0.005408, 0.0, 0.01033), + float4(-0.043263, -0.005408, 0.043263, 0.045182), + float4(0.124035, -0.992278, 0.0, 0.577584), + float4(0.044345, 0.043263, -0.043263, 7.0), + float4(-0.043263, -0.005408, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.124035, -0.992278, 0.0, 0.577584), + float4(0.001082, 0.037855, -0.043263, 7.0), + float4(-0.044345, 0.040019, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.669964, -0.742393, 0.0, 0.577584), + float4(0.044345, 0.043263, 0.0, 10.0), + float4(-0.043263, -0.005408, 0.0, 0.000305), + float4(-0.040019, 0.030284, 0.049753, 0.964691), + float4(0.101434, -0.811469, 0.575525, 0.973449), + float4(0.004326, 0.073548, 0.049753, 14.0), + float4(-0.003245, -0.035692, -0.049753, 0.0), + float4(-0.04759, 0.004326, -0.049753, 0.955978), + float4(-0.580475, -0.643229, 0.499304, 0.896269), + float4(0.001082, 0.037855, -0.043263, 11.0), + float4(-0.044345, 0.040019, 0.0, 0.01033), + float4(-0.044345, 0.040019, 0.043263, 0.045182), + float4(-0.669964, -0.742393, 0.0, 0.577584), + float4(-0.031366, 0.229295, 0.058405, 14.0), + float4(0.035692, -0.155748, -0.008653, 0.000305), + float4(-0.011897, -0.151422, -0.058405, 0.955978), + float4(-0.716533, -0.20131, 0.667874, 0.904662), + float4(-0.043263, 0.077874, -0.043263, 7.0), + float4(-0.001082, 0.154666, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.999976, -0.006993, 0.0, 0.577584), + float4(-0.031366, 0.229295, 0.058405, 14.0), + float4(0.089771, -0.137361, -0.02055, 0.445197), + float4(0.035692, -0.155748, -0.008653, 0.973449), + float4(0.216369, -0.004654, 0.976301, 0.973449), + float4(-0.031366, 0.229295, 0.058405, 14.0), + float4(-0.011897, -0.151422, -0.058405, 0.000305), + float4(-0.012979, 0.003245, -0.058405, 0.955978), + float4(-0.976245, -0.006827, 0.216564, 0.904662), + float4(-0.043263, 0.077874, -0.043263, 9.0), + float4(-0.001082, 0.154666, 0.0, 0.01033), + float4(-0.001082, 0.154666, 0.043263, 0.045182), + float4(-0.999976, -0.006993, 0.0, 0.577584), + float4(-0.045426, 0.564586, 0.052998, 14.0), + float4(0.014061, -0.335291, 0.005408, 0.520989), + float4(0.001082, -0.332046, -0.052998, 0.088655), + float4(-0.975941, -0.037462, 0.214794, 0.973449), + float4(-0.044345, 0.23254, 0.0, 3.0), + float4(0.0, 0.0, -0.043263, 0.01033), + float4(-0.001082, 0.332046, -0.043263, 0.045182), + float4(-0.999995, -0.003257, 0.0, 0.577584), + float4(-0.163319, 0.380717, -0.043263, 7.0), + float4(-0.060569, 0.02704, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.407651, -0.913138, 0.0, 0.577584), + float4(-0.094098, 0.431551, -0.043263, 11.0), + float4(-0.069221, -0.050834, 0.0, 0.01033), + float4(-0.069221, -0.050834, 0.043263, 0.045182), + float4(0.591909, -0.806004, 0.0, 0.577584), + float4(-0.163319, 0.380717, -0.043263, 11.0), + float4(-0.060569, 0.02704, 0.0, 0.01033), + float4(-0.060569, 0.02704, 0.043263, 0.045182), + float4(-0.407651, -0.913138, 0.0, 0.577584), + float4(0.075711, 0.08869, -0.043263, 11.0), + float4(-0.031366, -0.045426, 0.0, 0.01033), + float4(-0.031366, -0.045426, 0.043263, 0.045182), + float4(0.822897, -0.568191, 0.0, 0.577584), + float4(0.044345, 0.043263, 0.0, 6.0), + float4(-0.040019, 0.030284, 0.049753, 0.000305), + float4(0.014061, 0.048671, 0.037855, 0.964691), + float4(0.365607, -0.634949, 0.680566, 0.973449), + float4(0.075711, 0.08869, -0.043263, 7.0), + float4(-0.031366, -0.045426, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.822897, -0.568191, 0.0, 0.577584), + float4(0.058405, 0.091934, 0.037855, 14.0), + float4(0.017305, -0.003245, -0.037855, 0.0), + float4(-0.014061, -0.048671, -0.037855, 0.473533), + float4(0.757369, -0.522945, 0.39105, 0.904662), + float4(-0.031366, 0.229295, 0.058405, 14.0), + float4(0.107077, -0.140606, -0.058405, 0.0), + float4(0.089771, -0.137361, -0.02055, 0.376257), + float4(0.812052, 0.481346, 0.329966, 0.830777), + float4(0.077874, 0.407756, -0.043263, 11.0), + float4(-0.002163, -0.319067, 0.0, 0.01033), + float4(-0.002163, -0.319067, 0.043263, 0.045182), + float4(0.999977, -0.006779, 0.0, 0.577584), + float4(-0.031366, 0.229295, 0.058405, 14.0), + float4(0.10924, 0.178461, -0.058405, 0.0), + float4(0.107077, -0.140606, -0.058405, 0.376257), + float4(0.475583, -0.003224, 0.879665, 0.830777), + float4(0.147095, 0.272559, -0.043263, 7.0), + float4(-0.035692, 0.034611, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.696146, -0.7179, 0.0, 0.577584), + float4(0.147095, 0.272559, -0.043263, 11.0), + float4(-0.035692, 0.034611, 0.0, 0.01033), + float4(-0.035692, 0.034611, 0.043263, 0.045182), + float4(-0.696146, -0.7179, 0.0, 0.577584), + float4(0.144932, 0.307169, 0.049753, 14.0), + float4(0.002163, -0.034611, -0.049753, 0.0), + float4(-0.033529, 0.0, -0.049753, 0.9131), + float4(-0.630237, -0.649931, 0.424725, 0.904662), + float4(0.183869, 0.317985, 0.037855, 6.0), + float4(-0.036774, -0.045426, -0.037855, 0.0), + float4(-0.038937, -0.010816, 0.011897, 0.806958), + float4(0.374433, -0.753477, 0.540438, 0.854994), + float4(0.077874, 0.407756, -0.043263, 7.0), + float4(-0.002163, -0.319067, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.999977, -0.006779, 0.0, 0.577584), + float4(0.111403, 0.307169, -0.043263, 7.0), + float4(-0.002163, 0.131953, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.999866, -0.016391, 0.0, 0.577584), + float4(0.10924, 0.439122, 0.0, 6.0), + float4(0.035692, -0.131953, 0.049753, 0.18117), + float4(0.002163, -0.131953, 0.0, 0.955978), + float4(-0.82919, -0.013593, 0.558802, 0.964691), + float4(0.128708, 0.463999, 0.063813, 14.0), + float4(0.055161, -0.146014, -0.025958, 0.246204), + float4(0.016224, -0.156829, -0.014061, 0.964691), + float4(0.305397, -0.053644, 0.950713, 0.973449), + float4(0.111403, 0.307169, -0.043263, 11.0), + float4(-0.002163, 0.131953, 0.0, 0.01033), + float4(-0.002163, 0.131953, 0.043263, 0.045182), + float4(-0.999866, -0.016391, 0.0, 0.577584), + float4(0.189277, 0.28013, -0.043263, 11.0), + float4(-0.042182, -0.007571, 0.0, 0.01033), + float4(-0.042182, -0.007571, 0.043263, 0.045182), + float4(0.176664, -0.984271, 0.0, 0.577584), + float4(0.183869, 0.317985, 0.037855, 10.0), + float4(0.005408, -0.037855, -0.037855, 0.0), + float4(-0.036774, -0.045426, -0.037855, 0.806958), + float4(0.124328, -0.692684, 0.710445, 0.854994), + float4(0.189277, 0.28013, -0.043263, 7.0), + float4(-0.042182, -0.007571, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.176664, -0.984271, 0.0, 0.577584), + float4(0.183869, 0.317985, 0.037855, 14.0), + float4(0.02704, -0.004326, -0.037855, 0.0), + float4(0.005408, -0.037855, -0.037855, 0.514916), + float4(0.70062, -0.452013, 0.552101, 0.846876), + float4(0.210908, 0.313659, -0.043263, 11.0), + float4(-0.021632, -0.033529, 0.0, 0.01033), + float4(-0.021632, -0.033529, 0.043263, 0.045182), + float4(0.840297, -0.542127, 0.0, 0.577584), + float4(0.210908, 0.445612, 0.0, 10.0), + float4(0.0, -0.131953, 0.0, 0.0), + float4(-0.02704, -0.127627, 0.037855, 0.439658), + float4(0.813734, 0.0, 0.581238, 0.871366), + float4(0.210908, 0.313659, -0.043263, 7.0), + float4(-0.021632, -0.033529, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.840297, -0.542127, 0.0, 0.577584), + float4(0.210908, 0.445612, -0.043263, 11.0), + float4(0.0, -0.131953, 0.0, 0.01033), + float4(0.0, -0.131953, 0.043263, 0.045182), + float4(1.0, 0.0, 0.0, 0.577584), + float4(0.241193, 0.343943, -0.043263, 7.0), + float4(-0.002163, 0.107077, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.999796, -0.020198, 0.0, 0.577584), + float4(0.276885, 0.311496, -0.043263, 11.0), + float4(-0.035692, 0.032447, 0.0, 0.01033), + float4(-0.035692, 0.032447, 0.043263, 0.045182), + float4(-0.672673, -0.73994, 0.0, 0.577584), + float4(0.23903, 0.45102, 0.0, 14.0), + float4(0.042182, -0.108158, 0.046508, 0.000305), + float4(0.002163, -0.107077, 0.0, 0.846876), + float4(-0.758098, -0.015315, 0.651961, 0.896269), + float4(0.276885, 0.311496, -0.043263, 7.0), + float4(-0.035692, 0.032447, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.672673, -0.73994, 0.0, 0.577584), + float4(0.281211, 0.342862, 0.046508, 14.0), + float4(-0.004326, -0.031366, -0.046508, 0.0), + float4(-0.040019, 0.001082, -0.046508, 0.752941), + float4(-0.58651, -0.645161, 0.489667, 0.846876), + float4(0.246601, 0.459672, 0.06165, 14.0), + float4(0.070303, -0.101669, -0.024876, 0.262257), + float4(0.034611, -0.116811, -0.015142, 0.973449), + float4(0.27928, -0.041605, 0.959308, 0.973449), + float4(0.319067, 0.319067, -0.043263, 11.0), + float4(-0.042182, -0.007571, 0.0, 0.01033), + float4(-0.042182, -0.007571, 0.043263, 0.045182), + float4(0.176664, -0.984271, 0.0, 0.577584), + float4(0.319067, 0.319067, 0.0, 10.0), + float4(-0.042182, -0.007571, 0.0, 0.0), + float4(-0.037855, 0.023795, 0.046508, 0.896269), + float4(0.1483, -0.826245, 0.54344, 0.896269), + float4(0.319067, 0.319067, -0.043263, 7.0), + float4(-0.042182, -0.007571, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.176664, -0.984271, 0.0, 0.577584), + float4(0.319067, 0.319067, 0.0, 6.0), + float4(-0.037855, 0.023795, 0.046508, 0.0), + float4(-0.002163, 0.038937, 0.036774, 0.896269), + float4(0.437906, -0.604311, 0.665618, 0.896269), + float4(0.316904, 0.358004, 0.036774, 14.0), + float4(0.024876, -0.004326, -0.036774, 0.0), + float4(0.002163, -0.038937, -0.036774, 0.617212), + float4(0.707338, -0.464191, 0.533104, 0.921584), + float4(0.34178, 0.353677, -0.043263, 11.0), + float4(-0.022713, -0.034611, 0.0, 0.01033), + float4(-0.022713, -0.034611, 0.043263, 0.045182), + float4(0.836048, -0.548657, 0.0, 0.577584), + float4(0.340698, 0.474815, 0.0, 14.0), + float4(0.001082, -0.121137, 0.0, 0.0), + float4(-0.023795, -0.116811, 0.036774, 0.473533), + float4(0.828664, 0.007399, 0.559697, 0.930114), + float4(0.34178, 0.353677, -0.043263, 7.0), + float4(-0.022713, -0.034611, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.836048, -0.548657, 0.0, 0.577584), + float4(0.340698, 0.474815, -0.043263, 11.0), + float4(0.001082, -0.121137, 0.0, 0.01033), + float4(0.001082, -0.121137, 0.043263, 0.045182), + float4(0.99996, 0.008928, 0.0, 0.577584), + float4(0.411001, 0.351514, -0.043263, 11.0), + float4(-0.034611, 0.030284, 0.0, 0.01033), + float4(-0.034611, 0.030284, 0.043263, 0.045182), + float4(-0.658505, -0.752577, 0.0, 0.577584), + float4(0.411001, 0.351514, -0.043263, 7.0), + float4(-0.034611, 0.030284, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.658505, -0.752577, 0.0, 0.577584), + float4(0.412083, 0.38288, 0.043263, 14.0), + float4(-0.001082, -0.031366, -0.043263, 0.0), + float4(-0.035692, -0.001082, -0.043263, 0.745403), + float4(-0.574039, -0.656045, 0.489984, 0.854994), + float4(0.45102, 0.359085, -0.043263, 11.0), + float4(-0.040019, -0.007571, 0.0, 0.01033), + float4(-0.040019, -0.007571, 0.043263, 0.045182), + float4(0.185892, -0.98257, 0.0, 0.577584), + float4(0.45102, 0.359085, 0.0, 10.0), + float4(-0.040019, -0.007571, 0.0, 0.000305), + float4(-0.038937, 0.023795, 0.043263, 0.879622), + float4(0.151736, -0.802035, 0.577682, 0.9131), + float4(0.45102, 0.359085, -0.043263, 7.0), + float4(-0.040019, -0.007571, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.185892, -0.98257, 0.0, 0.577584), + float4(0.45102, 0.359085, 0.0, 6.0), + float4(-0.038937, 0.023795, 0.043263, 0.000305), + float4(0.003245, 0.0411, 0.032447, 0.879622), + float4(0.417852, -0.583049, 0.696744, 0.9131), + float4(0.473733, 0.391533, -0.043263, 11.0), + float4(-0.022713, -0.032447, 0.0, 0.01033), + float4(-0.022713, -0.032447, 0.043263, 0.045182), + float4(0.819232, -0.573462, 0.0, 0.577584), + float4(0.454264, 0.400185, 0.032447, 14.0), + float4(0.019468, -0.008653, -0.032447, 0.0), + float4(-0.003245, -0.0411, -0.032447, 0.527108), + float4(0.688617, -0.482032, 0.541712, 0.887922), + float4(0.473733, 0.391533, -0.043263, 7.0), + float4(-0.022713, -0.032447, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.819232, -0.573462, 0.0, 0.577584), + float4(-0.223888, 0.407756, -0.043263, 7.0), + float4(0.002163, 0.06814, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.999496, 0.03173, 0.0, 0.577584), + float4(-0.223888, 0.407756, -0.043263, 11.0), + float4(0.002163, 0.06814, 0.0, 0.01033), + float4(0.002163, 0.06814, 0.043263, 0.045182), + float4(-0.999496, 0.03173, 0.0, 0.577584), + float4(-0.167645, 0.416409, 0.046508, 14.0), + float4(-0.056242, -0.008653, -0.046508, 0.000305), + float4(-0.054079, 0.059487, -0.046508, 0.955978), + float4(-0.638985, 0.020285, 0.768952, 0.973449), + float4(-0.167645, 0.416409, 0.046508, 14.0), + float4(0.004326, -0.035692, -0.046508, 0.000305), + float4(-0.056242, -0.008653, -0.046508, 0.955978), + float4(-0.339782, -0.761112, 0.552501, 0.973449), + float4(-0.221724, 0.475896, 0.0, 14.0), + float4(0.139524, 0.055161, 0.043263, 0.520989), + float4(0.054079, -0.059487, 0.046508, 0.973449), + float4(-0.39307, 0.317373, 0.863001, 0.973449), + float4(-0.167645, 0.416409, 0.046508, 6.0), + float4(0.073548, 0.015142, -0.046508, 0.0), + float4(0.004326, -0.035692, -0.046508, 0.768154), + float4(0.490917, -0.668482, 0.558688, 0.871366), + float4(-0.094098, 0.431551, -0.043263, 7.0), + float4(-0.069221, -0.050834, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.591909, -0.806004, 0.0, 0.577584), + float4(-0.167645, 0.416409, 0.046508, 10.0), + float4(0.085445, 0.114648, -0.003245, 0.0), + float4(0.073548, 0.015142, -0.046508, 0.768154), + float4(0.548356, -0.38771, 0.740936, 0.871366), + float4(-0.0822, 0.531057, -0.043263, 11.0), + float4(-0.011897, -0.099506, 0.0, 0.01033), + float4(-0.011897, -0.099506, 0.043263, 0.045182), + float4(0.992928, -0.11872, 0.0, 0.577584), + float4(-0.044345, 0.23254, 0.0, 7.0), + float4(-0.001082, 0.332046, -0.043263, 0.01033), + float4(-0.001082, 0.332046, 0.052998, 0.045182), + float4(-0.999995, -0.003257, 0.0, 0.577584), + float4(-0.045426, 0.564586, 0.052998, 14.0), + float4(0.1233, -0.156829, -0.052998, 0.520989), + float4(0.014061, -0.335291, 0.005408, 0.088655), + float4(0.429355, 0.032562, 0.902549, 0.973449), + float4(0.10924, 0.439122, 0.0, 14.0), + float4(-0.031366, -0.031366, 0.0, 0.132876), + float4(-0.154666, 0.125464, 0.052998, 0.028428), + float4(0.182761, -0.182761, 0.966021, 0.955978), + float4(0.10924, 0.439122, -0.043263, 11.0), + float4(-0.031366, -0.031366, 0.0, 0.01033), + float4(-0.031366, -0.031366, 0.043263, 0.045182), + float4(0.707107, -0.707107, 0.0, 0.577584), + float4(0.10924, 0.439122, -0.043263, 7.0), + float4(-0.031366, -0.031366, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.707107, -0.707107, 0.0, 0.577584), + float4(0.10924, 0.439122, 0.0, 10.0), + float4(0.019468, 0.024876, 0.063813, 0.18117), + float4(0.035692, -0.131953, 0.049753, 0.955978), + float4(-0.933936, -0.126584, 0.334276, 0.964691), + float4(0.128708, 0.463999, 0.063813, 14.0), + float4(0.050834, 0.002163, -0.005408, 0.0), + float4(0.0822, -0.018387, -0.063813, 0.439658), + float4(0.07859, -0.92643, 0.368172, 0.871366), + float4(-0.221724, 0.475896, -0.043263, 7.0), + float4(0.122219, 0.207664, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.861819, 0.507216, 0.0, 0.577584), + float4(-0.221724, 0.475896, -0.043263, 9.0), + float4(0.122219, 0.207664, 0.0, 0.01033), + float4(0.122219, 0.207664, 0.043263, 0.045182), + float4(-0.861819, 0.507216, 0.0, 0.577584), + float4(-0.099506, 0.68356, 0.0, 14.0), + float4(0.017305, -0.152503, 0.043263, 0.0), + float4(-0.122219, -0.207664, 0.0, 0.723049), + float4(-0.365879, 0.215335, 0.905408, 0.854994), + float4(-0.0822, 0.531057, -0.043263, 7.0), + float4(-0.011897, -0.099506, 0.043263, 0.01033), + float4(0.0, 0.0, 0.086527, 0.045182), + float4(0.992928, -0.11872, 0.0, 0.577584), + float4(-0.099506, 0.68356, 0.0, 14.0), + float4(0.054079, -0.118974, 0.052998, 0.0), + float4(0.017305, -0.152503, 0.043263, 0.514916), + float4(-0.41958, 0.203352, 0.884647, 0.863157), + float4(-0.099506, 0.68356, -0.043263, 3.0), + float4(0.086527, 0.149258, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.86514, 0.501531, 0.0, 0.577584), + float4(-0.045426, 0.564586, -0.043263, 11.0), + float4(-0.036774, -0.033529, 0.0, 0.01033), + float4(-0.036774, -0.033529, 0.086527, 0.045182), + float4(0.673754, -0.738956, 0.0, 0.577584), + float4(-0.045426, 0.564586, -0.043263, 7.0), + float4(-0.036774, -0.033529, 0.086527, 0.01033), + float4(0.0, 0.0, 0.096261, 0.045182), + float4(0.673754, -0.738956, 0.0, 0.577584), + float4(-0.099506, 0.68356, 0.0, 14.0), + float4(0.086527, 0.149258, 0.0, 0.0), + float4(0.054079, -0.118974, 0.052998, 0.262257), + float4(-0.385563, 0.223515, 0.8952, 0.9131), + float4(-0.099506, 0.68356, -0.043263, 11.0), + float4(0.086527, 0.149258, 0.0, 0.01033), + float4(0.086527, 0.149258, 0.043263, 0.045182), + float4(-0.86514, 0.501531, 0.0, 0.577584), + float4(-0.012979, 0.832818, 0.0, 14.0), + float4(0.214153, -0.194685, 0.0822, 0.000305), + float4(-0.032447, -0.268232, 0.052998, 0.930114), + float4(-0.176863, 0.211325, 0.961281, 0.973449), + float4(0.128708, 0.463999, 0.063813, 14.0), + float4(-0.019468, -0.024876, -0.063813, 0.168276), + float4(-0.174135, 0.100587, -0.010816, 0.038209), + float4(-0.469238, -0.764881, 0.441331, 0.955978), + float4(0.201174, 0.638133, 0.0822, 14.0), + float4(-0.072466, -0.174135, -0.018387, 0.18117), + float4(-0.246601, -0.073548, -0.029203, 0.082277), + float4(-0.098561, -0.063844, 0.993081, 0.973449), + float4(-0.012979, 0.832818, -0.043263, 7.0), + float4(0.339617, -0.002163, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.006369, 0.99998, 0.0, 0.577584), + float4(0.179543, 0.466162, 0.058405, 10.0), + float4(-0.050834, -0.002163, 0.005408, 0.000305), + float4(0.021632, 0.171972, 0.023795, 0.022171), + float4(0.110896, -0.149888, 0.982464, 0.381323), + float4(-0.012979, 0.832818, 0.0, 14.0), + float4(0.339617, -0.002163, 0.0, 0.0), + float4(0.214153, -0.194685, 0.0822, 0.027314), + float4(0.002492, 0.391296, 0.920261, 0.49102), + float4(0.183869, 0.317985, 0.037855, 10.0), + float4(-0.055161, 0.146014, 0.025958, 0.0), + float4(0.02704, 0.127627, -0.037855, 0.439658), + float4(0.623841, 0.097823, 0.775405, 0.871366), + float4(0.210908, 0.445612, -0.043263, 7.0), + float4(0.0, -0.131953, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(1.0, 0.0, 0.0, 0.577584), + float4(0.210908, 0.445612, 0.0, 10.0), + float4(-0.031366, 0.02055, 0.058405, 0.000305), + float4(0.035692, 0.014061, 0.06165, 0.022171), + float4(-0.105864, -0.954447, 0.278971, 0.381323), + float4(0.23903, 0.45102, -0.043263, 11.0), + float4(-0.028121, -0.005408, 0.0, 0.01033), + float4(-0.028121, -0.005408, 0.043263, 0.045182), + float4(0.188847, -0.982006, 0.0, 0.577584), + float4(0.23903, 0.45102, -0.043263, 7.0), + float4(-0.028121, -0.005408, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.188847, -0.982006, 0.0, 0.577584), + float4(0.246601, 0.459672, 0.06165, 11.0), + float4(-0.007571, -0.008653, -0.06165, 0.000305), + float4(-0.035692, -0.014061, -0.06165, 0.022171), + float4(0.187618, -0.975617, 0.113888, 0.381323), + float4(0.241193, 0.343943, -0.043263, 11.0), + float4(-0.002163, 0.107077, 0.0, 0.01033), + float4(-0.002163, 0.107077, 0.043263, 0.045182), + float4(-0.999796, -0.020198, 0.0, 0.577584), + float4(0.23903, 0.45102, 0.0, 15.0), + float4(0.007571, 0.008653, 0.06165, 0.000305), + float4(0.042182, -0.108158, 0.046508, 0.846876), + float4(-0.94107, -0.299262, 0.157572, 0.896269), + float4(0.316904, 0.358004, 0.036774, 14.0), + float4(-0.070303, 0.101669, 0.024876, 0.0), + float4(0.023795, 0.116811, -0.036774, 0.473533), + float4(0.523424, 0.157027, 0.837478, 0.930114), + float4(0.340698, 0.474815, -0.043263, 7.0), + float4(0.001082, -0.121137, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.99996, 0.008928, 0.0, 0.577584), + float4(0.376391, 0.381799, -0.043263, 11.0), + float4(-0.001082, 0.107077, 0.0, 0.01033), + float4(-0.001082, 0.107077, 0.043263, 0.045182), + float4(-0.999949, -0.0101, 0.0, 0.577584), + float4(0.376391, 0.381799, -0.043263, 7.0), + float4(-0.001082, 0.107077, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.999949, -0.0101, 0.0, 0.577584), + float4(0.375309, 0.488875, 0.0, 14.0), + float4(0.036774, -0.105995, 0.043263, 0.000305), + float4(0.001082, -0.107077, 0.0, 0.879622), + float4(-0.771254, -0.00779, 0.63648, 0.896269), + float4(0.375309, 0.488875, 0.0, 14.0), + float4(0.078955, -0.08869, 0.032447, 0.223224), + float4(0.036774, -0.105995, 0.043263, 0.964691), + float4(0.071225, 0.39802, 0.914608, 0.973449), + float4(0.473733, 0.603523, 0.0, 14.0), + float4(0.0, -0.21199, 0.0, 0.158953), + float4(-0.019468, -0.203337, 0.032447, 0.030716), + float4(0.857493, 0.0, 0.514496, 0.964691), + float4(0.473733, 0.603523, -0.043263, 11.0), + float4(0.0, -0.21199, 0.0, 0.01033), + float4(0.0, -0.21199, 0.043263, 0.045182), + float4(1.0, 0.0, 0.0, 0.577584), + float4(0.276885, 0.484549, 0.058405, 10.0), + float4(-0.030284, -0.024876, 0.003245, 0.000305), + float4(-0.097342, -0.018387, 0.0, 0.022171), + float4(-0.03153, 0.166923, 0.985466, 0.381323), + float4(0.246601, 0.459672, 0.06165, 14.0), + float4(0.030284, 0.024876, -0.003245, 0.0), + float4(0.094098, 0.015142, -0.06165, 0.473533), + float4(0.518869, -0.545855, 0.657888, 0.930114), + float4(0.375309, 0.488875, -0.043263, 11.0), + float4(-0.034611, -0.014061, 0.0, 0.01033), + float4(-0.034611, -0.014061, 0.043263, 0.045182), + float4(0.376377, -0.926467, 0.0, 0.577584), + float4(0.340698, 0.474815, 0.0, 14.0), + float4(-0.063813, 0.009734, 0.058405, 0.000305), + float4(0.034611, 0.014061, 0.0, 0.022171), + float4(0.3276, -0.806402, 0.492334, 0.381323), + float4(0.375309, 0.488875, -0.043263, 7.0), + float4(-0.034611, -0.014061, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.376377, -0.926467, 0.0, 0.577584), + float4(0.454264, 0.400185, 0.032447, 12.0), + float4(-0.078955, 0.08869, -0.032447, 0.158953), + float4(0.019468, 0.203337, -0.032447, 0.030716), + float4(-0.201684, 0.173144, 0.964025, 0.964691), + float4(0.201174, 0.638133, 0.0822, 8.0), + float4(0.075711, -0.153585, -0.023795, 0.000305), + float4(-0.021632, -0.171972, -0.023795, 0.022171), + float4(0.026497, -0.140281, 0.989757, 0.381323), + float4(0.473733, 0.603523, -0.043263, 7.0), + float4(0.0, -0.21199, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(1.0, 0.0, 0.0, 0.577584), + float4(0.276885, 0.484549, 0.058405, 14.0), + float4(-0.075711, 0.153585, 0.023795, 0.000305), + float4(0.098424, 0.004326, -0.058405, 0.022171), + float4(0.50316, 0.115345, 0.856461, 0.381323), + float4(0.375309, 0.488875, 0.0, 10.0), + float4(0.090853, 0.12979, 0.0, 0.158953), + float4(0.098424, 0.114648, 0.0, 0.030716), + float4(0.0, 0.0, 1.0, 0.964691), + float4(0.466162, 0.618665, -0.043263, 11.0), + float4(0.007571, -0.015142, 0.0, 0.01033), + float4(0.007571, -0.015142, 0.043263, 0.045182), + float4(0.894427, 0.447214, 0.0, 0.577584), + float4(0.466162, 0.618665, -0.043263, 3.0), + float4(0.007571, -0.015142, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.894427, 0.447214, 0.0, 0.577584), + float4(0.326638, 0.830655, 0.0, 14.0), + float4(0.139524, -0.21199, 0.0, 0.0), + float4(0.048671, -0.34178, 0.0, 0.015213), + float4(0.0, 0.0, 1.0, 0.428687), + float4(0.326638, 0.830655, 0.0, 14.0), + float4(0.048671, -0.34178, 0.0, 0.007492), + float4(-0.125464, -0.192522, 0.0822, 0.054475), + float4(0.472493, 0.067285, 0.878762, 0.964691), + float4(0.326638, 0.830655, -0.043263, 9.0), + float4(0.139524, -0.21199, 0.0, 0.01033), + float4(0.139524, -0.21199, 0.043263, 0.045182), + float4(0.835314, 0.549773, 0.0, 0.577584), + float4(0.326638, 0.830655, -0.043263, 7.0), + float4(0.139524, -0.21199, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.835314, 0.549773, 0.0, 0.577584), + float4(-0.012979, 0.832818, -0.043263, 11.0), + float4(0.339617, -0.002163, 0.0, 0.01033), + float4(0.339617, -0.002163, 0.043263, 0.045182), + float4(0.006369, 0.99998, 0.0, 0.577584) +}; +static const float4 PRISM_BOXES[44] = { + float4(0.030135, 0.06127, -0.043945, 0.0), + float4(0.330601, 0.483055, 0.116951, 7.0), + float4(0.034429, 0.265258, -0.043945, 7.0), + float4(0.542104, 0.741798, 0.116951, 8.0), + float4(0.034429, 0.355442, -0.043945, 15.0), + float4(0.305908, 0.822319, 0.116951, 8.0), + float4(0.267405, 0.532589, -0.043945, 23.0), + float4(0.542104, 0.822319, 0.082595, 8.0), + float4(-0.045345, 0.036855, -0.044263, 31.0), + float4(0.045345, 0.23354, 0.059405, 8.0), + float4(-0.224888, 0.072548, -0.044263, 39.0), + float4(0.059405, 0.565586, 0.059405, 8.0), + float4(-0.032366, 0.042263, -0.044263, 47.0), + float4(0.148095, 0.408756, 0.059405, 8.0), + float4(0.074711, 0.08769, -0.044263, 55.0), + float4(0.184869, 0.464999, 0.064813, 8.0), + float4(0.146095, 0.271559, -0.044263, 63.0), + float4(0.211908, 0.446612, 0.038855, 8.0), + float4(0.23803, 0.310496, -0.044263, 71.0), + float4(0.320067, 0.460672, 0.06265, 8.0), + float4(0.275885, 0.310496, -0.044263, 79.0), + float4(0.412001, 0.475815, 0.047508, 8.0), + float4(0.375391, 0.350514, -0.044263, 87.0), + float4(0.45202, 0.38388, 0.044263, 4.0), + float4(0.410001, 0.350514, -0.044263, 91.0), + float4(0.474733, 0.401185, 0.044263, 5.0), + float4(-0.224888, 0.379717, -0.044263, 96.0), + float4(-0.0812, 0.532057, 0.047508, 8.0), + float4(-0.095098, 0.228295, -0.044263, 104.0), + float4(0.211908, 0.565586, 0.064813, 8.0), + float4(-0.222724, 0.430551, -0.044263, 112.0), + float4(-0.011979, 0.833818, 0.053998, 8.0), + float4(-0.100506, 0.438122, -0.044263, 120.0), + float4(0.327638, 0.833818, 0.0832, 8.0), + float4(0.127708, 0.312659, -0.044263, 128.0), + float4(0.282211, 0.467162, 0.064813, 8.0), + float4(0.245601, 0.352677, -0.044263, 136.0), + float4(0.474733, 0.604523, 0.06265, 8.0), + float4(0.178543, 0.390533, -0.044263, 144.0), + float4(0.474733, 0.639133, 0.0832, 8.0), + float4(0.200174, 0.483549, -0.044263, 152.0), + float4(0.474733, 0.639133, 0.0832, 4.0), + float4(-0.013979, 0.487875, -0.044263, 156.0), + float4(0.467162, 0.833818, 0.0832, 5.0) }; +static const float2 PRISM_POLY[110] = { + float2(0.006442, -0.002147), + float2(-0.001074, 0.001074), + float2(-0.004294, 0.007515), + float2(-0.004294, 0.806288), + float2(-0.001074, 0.809509), + float2(0.004294, 0.809509), + float2(0.022546, 0.796626), + float2(0.185736, 0.655982), + float2(0.281288, 0.86319), + float2(0.286656, 0.870706), + float2(0.293098, 0.871779), + float2(0.465951, 0.781595), + float2(0.478834, 0.773006), + float2(0.478834, 0.763344), + float2(0.407975, 0.592638), + float2(0.45092, 0.586196), + float2(0.601227, 0.573313), + float2(0.606595, 0.569018), + float2(0.606595, 0.564724), + float2(0.601227, 0.556135), + float2(0.50138, 0.453067), + float2(0.425153, 0.37684), + float2(0.289877, 0.246933), + float2(0.081595, 0.057975), + float2(0.013957, 0.0), + float2(-0.011897, 0.0), + float2(-0.055161, 0.037855), + float2(-0.081119, 0.063813), + float2(-0.083282, 0.06814), + float2(-0.084363, 0.38288), + float2(-0.089771, 0.38288), + float2(-0.158993, 0.33529), + float2(-0.166564, 0.336372), + float2(-0.250927, 0.379635), + float2(-0.261743, 0.388288), + float2(-0.263906, 0.461836), + float2(-0.262824, 0.486712), + float2(-0.173053, 0.638133), + float2(-0.044345, 0.860939), + float2(-0.036774, 0.867429), + float2(-0.024876, 0.871755), + float2(0.33529, 0.871755), + float2(0.349351, 0.867429), + float2(0.363412, 0.853368), + float2(0.513752, 0.619747), + float2(0.513752, 0.379635), + float2(0.466162, 0.322311), + float2(0.461836, 0.320148), + float2(0.399104, 0.311496), + float2(0.393696, 0.313659), + float2(0.374227, 0.332046), + float2(0.343943, 0.288782), + float2(0.336372, 0.282293), + float2(0.269314, 0.271477), + float2(0.263906, 0.27364), + float2(0.243356, 0.292027), + float2(0.21199, 0.244438), + float2(0.207664, 0.241193), + float2(0.149258, 0.231459), + float2(0.138443, 0.231459), + float2(0.118974, 0.248764), + float2(0.116811, 0.222806), + float2(0.117892, 0.162237), + float2(0.115729, 0.076792), + float2(0.06814, 0.011897), + float2(0.062732, 0.007571), + float2(0.005408, 0.0), + float2(0.031135, 0.06227), + float2(0.252301, 0.266258), + float2(0.541104, 0.544325), + float2(0.359663, 0.559356), + float2(0.433742, 0.753681), + float2(0.304908, 0.821319), + float2(0.199693, 0.595859), + float2(0.035429, 0.740798), + float2(0.035429, 0.482055), + float2(-0.043263, 0.077874), + float2(0.001082, 0.037855), + float2(0.044345, 0.043263), + float2(0.075711, 0.08869), + float2(0.077874, 0.407756), + float2(0.10924, 0.439122), + float2(0.111403, 0.307169), + float2(0.147095, 0.272559), + float2(0.189277, 0.28013), + float2(0.210909, 0.313659), + float2(0.210909, 0.445612), + float2(0.23903, 0.45102), + float2(0.241193, 0.343943), + float2(0.276885, 0.311496), + float2(0.319067, 0.319067), + float2(0.34178, 0.353677), + float2(0.340698, 0.474815), + float2(0.375309, 0.488875), + float2(0.376391, 0.381799), + float2(0.411001, 0.351514), + float2(0.45102, 0.359085), + float2(0.473733, 0.391533), + float2(0.473733, 0.603523), + float2(0.466162, 0.618665), + float2(0.326638, 0.830655), + float2(-0.012979, 0.832818), + float2(-0.099506, 0.68356), + float2(-0.221724, 0.475896), + float2(-0.223888, 0.407756), + float2(-0.163319, 0.380717), + float2(-0.094098, 0.431551), + float2(-0.0822, 0.531057), + float2(-0.045426, 0.564586), + float2(-0.044345, 0.23254) +}; +static const float PRISM_BEVEL = 0.012883; +// end of the prism mesh + +// Le verre validé sur le prototype (30/09/2026). +static const float PRISM_IOR = 1.61; +static const float PRISM_DISPERSION = 0.035; +static const float PRISM_GLOW = 0.45; +static const float PRISM_FOLD = 0.35; +// Rebonds au plus dans le cristal, par canal ; l'id de matière du cristal. +static const int PRISM_BOUNCES = 6; +static const float PRISM_MAT = 9.0; + +// Le modèle de Prism Glow de ce dessin : 0 flèche, 1 main, -1 un autre curseur. +int prism_shape() +{ + int id = sculpt_id(); + return id == 3 || id == 4 ? id - 3 : -1; +} -float s_facet_ellipsoid(float3 p, float3 r) +// Distance signée au polygone `n` sommets de PRISM_POLY à partir de `i0` (plan xy du modèle), +// négative dedans : la silhouette du serti, ou le contour du cristal. +float prism_poly_dist(float2 p, int i0, int n) { - float3 q = abs(p); - float d = -1e9; - [loop] for (int i = 0; i < 10; i++) + float2 vj = PRISM_POLY[i0 + n - 1]; + float d = 1e9; + float sgn = 1.0; + [loop] for (int i = 0; i < n; i++) { - d = max(d, dot(q, SCULPT_FACETS[i]) - length(r * SCULPT_FACETS[i])); + float2 vi = PRISM_POLY[i0 + i]; + float2 e = vj - vi; + float2 w = p - vi; + float2 b = w - e * saturate(dot(w, e) / dot(e, e)); + d = min(d, dot(b, b)); + bool c0 = p.y >= vi.y; + bool c1 = p.y < vj.y; + bool c2 = e.x * w.y > e.y * w.x; + if ((c0 && c1 && c2) || (!c0 && !c1 && !c2)) + { + sgn = -sgn; + } + vj = vi; } - return d; + return sgn * sqrt(d); } -// Le gant des autres thèmes, mêmes os, taillé en facettes. -float s_crystal_hand(float3 p) +// Le serti : la silhouette extrudée de z = -épaisseur à 0, arêtes arrondies de PRISM_BEVEL. +float prism_rim(float3 p, int shape) { - float zc = SCULPT_HAND_ZC; - float d = s_facet_ellipsoid(p - float3(0.185, -0.70, zc), float3(0.3, 0.235, 0.125)); - d = min(d, s_facet_capsule(p, float3(0.0, -0.52, zc), float3(0.0, -0.10, zc), 0.098, 0.3)); - d = min(d, s_facet_capsule(p, float3(0.17, -0.57, zc + 0.012), float3(0.17, -0.41, zc + 0.045), 0.086, 0.1)); - d = min(d, s_facet_capsule(p, float3(0.31, -0.59, zc + 0.01), float3(0.31, -0.45, zc + 0.04), 0.08, 0.5)); - d = min(d, s_facet_capsule(p, float3(0.435, -0.625, zc + 0.005), float3(0.435, -0.52, zc + 0.03), 0.07, 0.2)); - return min(d, s_facet_capsule(p, float3(0.03, -0.77, zc + 0.03), float3(-0.165, -0.60, zc + 0.065), 0.082, 0.7)); + float half_t = model_thick() * 0.5; + float sil = prism_poly_dist(p.xy, PRISM_SIL_START[shape], PRISM_SIL_COUNT[shape]); + float2 w = float2(sil + PRISM_BEVEL, abs(p.z + half_t) - (half_t - PRISM_BEVEL)); + return min(max(w.x, w.y), 0.0) + length(max(w, 0.0)) - PRISM_BEVEL; } -// Le modèle `shape` du thème `theme`, repère du prototype : (distance, id de matière). -float2 sculpt_proto(float3 p, int theme, int shape) +// Distance le long de (ro, rd) au triangle `k` (Möller–Trumbore), -1 s'il le rate. Un rien de jeu +// sur les bords : un rayon ne passe pas entre deux facettes voisines. +float prism_tri(float3 ro, float3 rd, int k) { - if (theme == 3) + float3 v0 = PRISM_TRIS[4 * k].xyz; + float3 e1 = PRISM_TRIS[4 * k + 1].xyz; + float3 e2 = PRISM_TRIS[4 * k + 2].xyz; + float3 pv = cross(rd, e2); + float det = dot(e1, pv); + if (abs(det) < 1e-10) { - return s_voxels(p, shape); + return -1.0; } - if (theme == 1) + float inv = 1.0 / det; + float3 sv = ro - v0; + float u = dot(sv, pv) * inv; + float3 qv = cross(sv, e1); + float v = dot(rd, qv) * inv; + if (u < -2e-4 || v < -2e-4 || u + v > 1.0004) { - return float2(shape == 0 ? s_gem_arrow(p) : s_crystal_hand(p), 8.0); + return -1.0; } - float2 body; - float3 star; - if (shape == 0) + return dot(e2, qv) * inv; +} + +// Le triangle du cristal `shape` le plus proche devant (ro, rd) : sa distance (1e9 si aucun), son +// indice dans `hit` (-1 si aucun). Les triangles sont rangés par voisinage, une boîte englobante +// par groupe (PRISM_BOXES) : un rayon qui rate une boîte, ou n'y entre qu'au-delà du meilleur +// point, saute tous ses triangles. L'inverse de la direction sert à toutes les boîtes. +float prism_trace(float3 ro, float3 rd, int shape, out int hit) +{ + float3 inv = 1.0 / (abs(rd) > 1e-6 ? rd : 1e-6); + int b0 = PRISM_BOX_START[shape]; + int nb = PRISM_BOX_COUNT[shape]; + float best = 1e9; + hit = -1; + [loop] for (int j = b0; j < b0 + nb; j++) { - float h = 0.075, re = 0.03, dome = 0.0; - if (theme == 2) - { - h = 0.07; - re = 0.06; - dome = 0.035; - } - if (theme == 4) + float4 lo = PRISM_BOXES[2 * j]; + float4 hi = PRISM_BOXES[2 * j + 1]; + float3 t0 = (lo.xyz - ro) * inv; + float3 t1 = (hi.xyz - ro) * inv; + float3 tn = min(t0, t1); + float3 tf = max(t0, t1); + float tmin = max(max(tn.x, tn.y), tn.z); + float tmax = min(min(tf.x, tf.y), tf.z); + if (tmax < max(tmin, 1e-5) || tmin > best) { - h = 0.07; - re = 0.055; - dome = 0.025; + continue; } - float2 r = float2(s_arrow_solid(p, h, re, dome), 1.0); - if (theme == 0) + int k0 = (int)lo.w; + int n = (int)hi.w; + [loop] for (int i = k0; i < k0 + n; i++) { - r = s_opu(r, float2(s_piping(p, SCULPT_HOVER + 2.0 * h - 0.004), 2.0)); + float t = prism_tri(ro, rd, i); + if (t > 1e-5 && t < best) + { + best = t; + hit = i; + } } - body = r; - star = float3(0.57, -0.86, SCULPT_HOVER + 2.0 * h + dome); } - else + return best; +} + +// Le modèle `shape` du thème `theme`, repère du prototype : (distance, id de matière). +float2 sculpt_proto(float3 p, int theme, int shape) +{ + if (theme == 3) { - body = s_opu(float2(s_glove(p, SCULPT_HAND_ZC), 1.0), float2(s_cuff(p, SCULPT_HAND_ZC), 2.0)); - star = float3(0.185, -0.93, SCULPT_HAND_ZC + 0.15); + return s_voxels(p, shape); } - // L'étoile de Star Sprout : un seul appel pour les deux formes (FXC recopie chaque appel). - return theme == 4 ? s_opu(body, s_sprout(p, star)) : body; + // Un seul appel pour les trois thèmes cerclés et leurs deux formes (FXC recopie chaque appel). + return s_rimmed(p, theme, shape); } // Repère du modèle -> prototype. L'écrasement du clic ne raccourcit que z, autour du hotspot. @@ -1005,23 +2052,28 @@ float sculpt_units() // Le curseur sculpté en `q` (repère du modèle) : distance et id de matière. `occ` : ce qui porte // l'ombre sur l'écran. La borne des plans d'une gemme et le champ des voxels sont de mauvaises // distances loin de la surface : la pénombre s'y strie, ou s'arrête net au bord de la boîte des -// voxels. Le cristal porte donc l'ombre de la forme lisse de Pop Coral, les voxels celle de leur -// contour extrudé sur toute leur hauteur. +// voxels. Pixel Candy porte donc l'ombre de sa forme, extrudée sur la hauteur des cubes. Prism +// Glow n'a ici que son serti, dans le repère du modèle : son cristal est tracé à part +// (`prism_primary`), et l'ombre est celle de toute sa silhouette. float2 sculpt_eval(float3 q, bool occ) { int id = sculpt_id() - 1; int theme = id / 2, shape = id % 2; + if (theme == 1) + { + return float2(prism_rim(q, shape), 8.0); + } float3 p = sculpt_point(q); float2 r; if (occ && theme == 3) { - float d2 = shape == 0 ? s_arrow2(p.xy) - 1.2 * SCULPT_VOX : s_pixel_hand_dist(p.xy); + float d2 = s_pixel_outline(p.xy, shape); float dz = abs(p.z - (SCULPT_HOVER + 0.07)) - 0.07; r = float2(length(max(float2(d2, dz), 0.0)) + min(max(d2, dz), 0.0), 7.0); } else { - r = sculpt_proto(p, occ && theme == 1 ? 2 : theme, shape); + r = sculpt_proto(p, theme, shape); } return float2(r.x * sculpt_units(), r.y); } @@ -1138,20 +2190,16 @@ SculptMat s_mat(float3 alb, float rough, float spec, float sss, float refl) return m; } -// Couleur de face du voxel qui porte `p` (prototype) : menthe où le bord regarde en bas à gauche, -// rose pâle où il regarde en haut à droite, rose dedans. +// Couleur du cube qui porte `p` (prototype) : celle de son pixel. float3 s_pixel_colour(float3 p, int shape) { - float2 id = floor((p.xy - s_grid_origin(shape)) / SCULPT_VOX); - if (!s_occ(id + float2(-1.0, 0.0), shape) || !s_occ(id + float2(0.0, -1.0), shape)) - { - return s_lin(0.52, 0.91, 0.77); - } - if (!s_occ(id + float2(1.0, 0.0), shape) || !s_occ(id + float2(0.0, 1.0), shape)) - { - return s_lin(1.0, 0.78, 0.87); - } - return s_lin(1.0, 0.50, 0.71); + int2 cell = s_pix_cell(p.xy, shape); + int4 g = PIX_GRID[shape]; + int row = g.z + clamp(cell.y, 0, g.y - 1); + if (s_pix_bit(PIX_LINE[row], cell.x)) return s_lin(0.29, 0.12, 0.36); + if (s_pix_bit(PIX_HI[row], cell.x)) return s_lin(1.0, 0.78, 0.87); + if (s_pix_bit(PIX_SHADE[row], cell.x)) return s_lin(0.87, 0.27, 0.51); + return s_lin(1.0, 0.435, 0.66); } // La matière `mat` (id de `sculpt_proto`) au point `p` du prototype. @@ -1160,28 +2208,36 @@ SculptMat sculpt_material(float mat, float3 p, int theme, int shape) bool primary = mat < 1.5; if (theme == 0) { - if (shape == 0 && primary) return s_mat(s_lin(0.10, 0.10, 0.115), 0.3, 0.2, 0.0, 0.9); - if (shape == 0) return s_mat(s_lin(0.94, 0.91, 0.84), 0.45, 0.4, 0.2, 0.3); - if (primary) return s_mat(s_lin(0.95, 0.92, 0.85), 0.55, 0.35, 0.35, 0.25); - return s_mat(s_lin(0.17, 0.18, 0.22), 0.35, 0.6, 0.0, 0.6); + // Ivoire (le gant, la bande de la flèche) ; noir satiné, presque sans reflet : un reflet + // large sur le champ plat de la flèche le virait au gris. + if (mat < 2.5) return s_mat(s_lin(0.95, 0.92, 0.85), 0.3, 0.6, 0.35, 0.25); + return s_mat(s_lin(0.1, 0.1, 0.11), 0.55, 0.05, 0.0, 0.1); + } + if (theme == 1) + { + // Le serti de Prism Glow : un émail marine, brillant sur son arrondi seulement (`model_shade`). + return s_mat(s_lin(0.02, 0.05, 0.33), 0.35, 0.5, 0.05, 0.3); } if (theme == 2) { - if (shape == 0) return s_mat(s_lin(1.0, 0.40, 0.30), 0.5, 0.45, 0.4, 0.25); - if (primary) return s_mat(s_lin(1.0, 0.79, 0.16), 0.5, 0.45, 0.4, 0.25); - return s_mat(s_lin(0.18, 0.20, 0.29), 0.4, 0.5, 0.0, 0.4); + // Corail : la flèche, le calque et les tirets de la main ; jaune : le gant, le calque et les + // tirets de la flèche ; le trait marine. + if ((primary && shape == 0) || (mat > 2.5 && mat < 3.5 && shape == 1)) return s_mat(s_lin(1.0, 0.40, 0.30), 0.85, 0.08, 0.15, 0.03); + if (mat < 3.5) return s_mat(s_lin(1.0, 0.80, 0.10), 0.85, 0.08, 0.15, 0.03); + return s_mat(s_lin(0.09, 0.13, 0.45), 0.9, 0.05, 0.05, 0.02); } if (theme == 4) { - if (primary && shape == 0) return s_mat(s_lin(0.62, 0.91, 0.78), 0.22, 0.8, 0.25, 0.6); - if (primary) return s_mat(s_lin(0.96, 0.94, 0.88), 0.5, 0.35, 0.35, 0.25); - if (mat < 2.5) return s_mat(s_lin(0.62, 0.91, 0.78), 0.25, 0.7, 0.25, 0.5); - if (mat < 3.5) return s_mat(s_lin(1.0, 0.80, 0.20), 0.3, 0.6, 0.3, 0.4); - if (mat < 4.5) return s_mat(s_lin(0.38, 0.80, 0.55), 0.35, 0.5, 0.35, 0.3); - return s_mat(s_lin(0.16, 0.12, 0.10), 0.2, 0.8, 0.0, 0.5); + // Coussins laqués ; le marine satiné, un reflet large et sombre plutôt qu'un filet blanc. + if (primary && shape == 0) return s_mat(s_lin(0.68, 0.93, 0.80), 0.25, 0.7, 0.3, 0.3); + if (primary) return s_mat(s_lin(0.97, 0.95, 0.90), 0.3, 0.6, 0.35, 0.25); + if (mat < 2.5) return s_mat(s_lin(0.52, 0.87, 0.78), 0.25, 0.7, 0.3, 0.3); + if (mat < 3.5) return s_mat(s_lin(1.0, 0.75, 0.25), 0.25, 0.7, 0.3, 0.3); + if (mat < 4.5) return s_mat(s_lin(0.62, 0.92, 0.72), 0.3, 0.6, 0.3, 0.35); + return s_mat(s_lin(0.07, 0.15, 0.33), 0.7, 0.15, 0.05, 0.08); } - if (mat < 6.5) return s_mat(s_pixel_colour(p, shape), 0.45, 0.35, 0.15, 0.2); - return s_mat(s_lin(0.36, 0.18, 0.54), 0.45, 0.35, 0.1, 0.2); + // Plastique mat : sur les faces plates des cubes, un reflet de la lampe blanchirait le prune. + return s_mat(s_pixel_colour(p, shape), 0.65, 0.1, 0.12, 0.04); } // L'environnement du studio vu du modèle : l'écran dessous, la pièce au-dessus (z du modèle), une @@ -1197,39 +2253,6 @@ float3 model_env(float3 d, float rough, float3 l, float3 fill) return col; } -// Cyan, bleu, violet, orchidée. -float3 s_gem(float k) -{ - k = saturate(k) * 3.0; - float3 a = s_lin(0.20, 0.95, 1.0); - float3 b = s_lin(0.15, 0.42, 1.0); - float3 c = s_lin(0.45, 0.25, 0.95); - float3 d = s_lin(0.88, 0.50, 1.0); - if (k < 1.0) return lerp(a, b, k); - if (k < 2.0) return lerp(b, c, k - 1.0); - return lerp(c, d, k - 2.0); -} - -// Le cristal : la teinte vient d'où la facette regarde et d'où elle plie la vue (lues y vers le -// haut, comme au prototype) ; la dispersion sépare le décalage par canal. Les facettes tournées -// vers la lampe sont cyan, les autres violettes. -float3 model_shade_crystal(float3 n, float3 rd, float3 L, float fall, float3 l, float3 fill) -{ - float cosi = saturate(dot(-rd, n)); - float F = 0.04 + 0.96 * pow(1.0 - cosi, 5.0); - float3 t = refract(rd, n, 1.0 / 1.6); - float k = 0.42 + 0.9 * dot(float2(n.x, -n.y), float2(0.7557, -0.6549)) - + 0.6 * dot(float2(t.x, -t.y), float2(0.6, -0.8)); - float3 body = float3(s_gem(k - 0.08).r, s_gem(k).g, s_gem(k + 0.08).b); - float3 col = body * (0.1 + 1.8 * fall * pow(max(dot(n, L), 0.0), 2.5)); - // Éclat : la vue rebondit sur le dos plat et allume la facette quand elle trouve la lampe. - col += body * model_env(reflect(t, float3(0.0, 0.0, 1.0)) * float3(1.0, 1.0, -1.0), 0.15, l, fill) * 0.5; - col += pow(max(dot(reflect(rd, n), L), 0.0), 30.0) * 2.0; - col += model_env(reflect(rd, n), 0.05, l, fill) * F; - col += s_lin(0.5, 0.9, 1.0) * pow(1.0 - cosi, 4.0) * 0.6; - return col; -} - // Khronos PBR Neutral : garde les teintes, ne comprime que les hautes lumières ; puis sRGB. float3 model_tonemap(float3 c) { @@ -1260,10 +2283,6 @@ float3 model_shade(float3 q, float3 n, float3 rd, float3 L, float fall, float sh float3 l, float3 fill) { int id = sculpt_id(); - if (id > 0 && (id - 1) / 2 == 1) - { - return model_tonemap(model_shade_crystal(n, rd, L, fall, l, fill)); - } SculptMat m; // Sur un sprite, reflets et brillance sur les arrondis seulement : le dessus plat d'un sprite // sombre virerait au gris sous la lampe. @@ -1273,6 +2292,12 @@ float3 model_shade(float3 q, float3 n, float3 rd, float3 L, float fall, float sh // La matière d'un voxel se lit juste sous la surface : son flanc lit sa propre cellule. float3 p = sculpt_point(q) - float3(n.x, -n.y, n.z) * 0.01; m = sculpt_material(mat, p, (id - 1) / 2, (id - 1) % 2); + if (id == 3 || id == 4) + { + // Le serti de Prism Glow, comme un sprite : son dessus plat, face à la lampe, prendrait + // tout son reflet et virerait au gris. + gloss = 1.0 - smoothstep(0.97, 0.995, abs(n.z)); + } } else { @@ -1298,6 +2323,177 @@ float3 model_shade(float3 q, float3 n, float3 rd, float3 L, float fall, float sh return model_tonemap(col); } +// ---- Le cristal de Prism Glow ---- + +float prism_fresnel(float c, float ior) +{ + float f0 = (ior - 1.0) / (ior + 1.0); + f0 *= f0; + return f0 + (1.0 - f0) * pow(1.0 - c, 5.0); +} + +// Le cristal sur le rayon (ro, rd) : distance au point touché (1e9 si raté ou hors de Prism Glow), +// facette dans `hc`. Une paroi sous le dessus du serti ne compte pas : le rayon a traversé le +// serti avant, c'est lui qu'on voit. +float prism_primary(float3 ro, float3 rd, out int hc) +{ + hc = -1; + int shape = prism_shape(); + if (shape < 0) + { + return 1e9; + } + float t = prism_trace(ro, rd, shape, hc); + if (hc >= 0 && fmod(PRISM_TRIS[4 * hc].w, 2.0) > 0.5 && ro.z + rd.z * t < 0.0) + { + hc = -1; + return 1e9; + } + return t; +} + +// La normale de la facette `h`, tournée vers le rayon qui arrive le long de `rd`. +float3 prism_normal(int h, float3 rd) +{ + float3 n = PRISM_TRIS[4 * h + 3].xyz; + return dot(n, rd) > 0.0 ? -n : n; +} + +// Distance de `p` au segment [a, b]. +float prism_seg(float3 p, float3 a, float3 b) +{ + float3 pa = p - a, ba = b - a; + return length(pa - ba * saturate(dot(pa, ba) / dot(ba, ba))); +} + +// Distance, en pixels (`px` = unités du modèle par pixel au point), de `p` aux arêtes réelles de +// la facette `h` : les plis de l'art et le bord du serti. +float prism_edge_px(float3 p, int h, float px) +{ + float4 r0 = PRISM_TRIS[4 * h]; + float3 a = r0.xyz; + float3 b = a + PRISM_TRIS[4 * h + 1].xyz; + float3 c = a + PRISM_TRIS[4 * h + 2].xyz; + int bits = (int)(r0.w * 0.5); + float d = 1e9; + if ((bits & 1) != 0) d = min(d, prism_seg(p, b, c)); + if ((bits & 2) != 0) d = min(d, prism_seg(p, c, a)); + if ((bits & 4) != 0) d = min(d, prism_seg(p, a, b)); + return d / px; +} + +// La rotation inverse de `world_to_plane` : du repère du plan à celui de la caméra. +float3 plane_to_world(float3 v, ModelFrame f) +{ + float x = v.x * f.c.z - v.y * f.s.z; + float y = v.x * f.s.z + v.y * f.c.z; + float z = -x * f.s.y + v.z * f.c.y; + return float3(x * f.c.y + v.z * f.s.y, y * f.c.x - z * f.s.x, y * f.s.x + z * f.c.x); +} + +// Ce que montre l'image là où le rayon (q, d) du modèle retombe sur le plan : la copie de l'image +// composée faite juste avant le curseur (t5), flous de confidentialité compris, lue au pixel de +// sortie de ce point, en linéaire. Seuls des pixels déjà floutés passent donc à travers le verre. +// L'œil est en (-mb.zw, persp) du repère caméra, le rayon du pixel `local` y va vers +// (local + src.xy, -persp) : on remonte de ce point du plan à son `local`, puis à son uv de sortie. +// Hors de l'image, l'écran sombre du studio. +float3 prism_screen(float3 q, float3 d, float3 nz, float hz, float3 tip, float unit, ModelFrame f) +{ + float denom = dot(d, nz); + if (denom > -1e-4) + { + return SCULPT_SCREEN * 0.2; + } + float3 g = q + d * ((hz - dot(q, nz)) / denom); + float3 w = plane_to_world(tip + unit * model_to_plane(g, f), f); + float k = 1.0 - w.z / src.z; + if (k < 1e-3) + { + return SCULPT_SCREEN * 0.2; + } + float2 uv = dst.xy + ((w.xy + mb.zw) / k - src.xy) / quad_px * dst.zw; + if (any(uv < 0.0) || any(uv > 1.0)) + { + return SCULPT_SCREEN * 0.2; + } + return pow(texDof.SampleLevel(samp, uv, 0.0).rgb, 2.2); +} + +// La couleur (sRGB, comme `model_shade`) du cristal au point `p` de la facette `h`, vu le long de +// `rd` : reflet du studio, l'enregistrement réfracté par canal (la dispersion) à travers les +// facettes et le fond plat, la lueur de la couleur de la facette dans l'art, ses plis en liseré. +float3 prism_shade(float3 p, float3 n, float3 rd, int h, float px, float3 l, float3 fill, float3 nz, + float hz, float3 tip, float unit, ModelFrame f) +{ + int shape = prism_shape(); + float4 r3 = PRISM_TRIS[4 * h + 3]; + float3 facet = float3(PRISM_TRIS[4 * h + 1].w, PRISM_TRIS[4 * h + 2].w, r3.w); + float F = prism_fresnel(saturate(-dot(rd, n)), PRISM_IOR); + float zb = -model_thick(); + float3 trans = 0.0; + [loop] for (int ch = 0; ch < 3; ch++) + { + // Le rouge plie le moins, le bleu le plus. FXC n'indexe pas un vecteur par une variable en + // écriture (il déroulerait la boucle) : le canal passe par un masque. + float3 mask = float3(ch == 0, ch == 1, ch == 2); + float ior = PRISM_IOR + PRISM_DISPERSION * (ch - 1); + float3 d = refract(rd, n, 1.0 / ior); + float3 pos = p; + float thr = 1.0; + float acc = 0.0; + [loop] for (int b = 0; b < PRISM_BOUNCES; b++) + { + int hh; + float th = prism_trace(pos, d, shape, hh); + float tb = d.z < -1e-6 ? (zb - pos.z) / d.z : 1e9; + if (tb <= th) + { + // Le fond plat : le rayon sort vers l'écran, ou s'y réfléchit tout entier. + float3 qb = pos + d * tb; + float3 d2 = refract(d, float3(0.0, 0.0, 1.0), ior); + if (dot(d2, d2) < 1e-8) + { + pos = qb; + d = reflect(d, float3(0.0, 0.0, 1.0)); + continue; + } + acc += thr * dot(prism_screen(qb, d2, nz, hz, tip, unit, f), mask); + thr = 0.0; + break; + } + if (hh < 0) + { + break; + } + pos += d * th; + float4 rn = PRISM_TRIS[4 * hh + 3]; + if (fmod(PRISM_TRIS[4 * hh].w, 2.0) > 0.5 && pos.z < 0.0) + { + // Une paroi sous le dessus du serti : le marine de son flanc intérieur. + acc += thr * dot(s_lin(0.03, 0.06, 0.37) * 0.35, mask); + thr = 0.0; + break; + } + float3 dout = refract(d, -rn.xyz, ior); + if (dot(dout, dout) < 1e-8) + { + d = reflect(d, -rn.xyz); + continue; + } + float fi = prism_fresnel(saturate(dot(dout, rn.xyz)), ior); + acc += thr * (1.0 - fi) * dot(model_env(dout, 0.05, l, fill), mask); + thr *= fi; + d = reflect(d, -rn.xyz); + } + trans += acc * mask; + } + float3 col = F * model_env(reflect(rd, n), 0.05, l, fill) + (1.0 - F) * trans + facet * PRISM_GLOW; + // Les plis de l'art : un liseré clair sur chaque arête réelle, fin d'un pixel. + float fold = 1.0 - smoothstep(0.2, 1.0, prism_edge_px(p, h, px)); + col = lerp(col, s_lin(0.96, 0.99, 1.0) * 1.3, PRISM_FOLD * fold * (0.35 + 0.65 * saturate(dot(n, l)))); + return model_tonemap(col); +} + // Les passes de la boucle unique de `cursor_model`, dans l'ordre. Chaque tour n'évalue le modèle // QU'UNE fois (`model_eval`) : FXC recopie une fonction à chaque appel, et un curseur sculpté est // assez gros pour qu'un appel par passe multiplie par neuf le temps de compilation de ps_main. @@ -1306,7 +2502,12 @@ static const int STAGE_NORMAL = 1; // quatre sondes en tétraèdre autour du poi static const int STAGE_AO = 2; // cinq sondes le long de la normale, jusqu'à 0,08 du prototype static const int STAGE_SELF = 3; // pénombre du modèle sur lui-même, vers la lampe (32 pas) static const int STAGE_PLANE = 4; // le point du plan : son contact, puis sa pénombre (32 pas) -static const int STAGE_DONE = 5; +static const int STAGE_EDGE = 5; // quatre sondes à un demi-pixel du point touché : un bord ? +static const int STAGE_SHADE = 6; // l'ombrage d'un échantillon, puis le rayon suivant s'il en reste +static const int STAGE_DONE = 7; + +// Les trois rayons de plus d'un pixel au bord, en pixels autour de son centre. +static const float2 MODEL_SUBPIXEL[3] = { float2(0.35, 0.2), float2(-0.35, 0.2), float2(0.0, -0.4) }; // Sonde `k` de la normale, sommet d'un tétraèdre. float3 model_tetra(int k) @@ -1347,17 +2548,30 @@ float4 cursor_model(float2 local) float3 lamp = float3(color.rg + sprite_size() * 0.5, 0.0) + l * SCULPT_LAMP_DIST; // Silhouette antialiasée : un rayon qui frôle le modèle à moins d'un pixel le couvre en partie - // (`best`, la plus petite distance rencontrée, en pixels). - float2 tb = ray_box(ro, rd, lo - 0.02, hi + 0.02); + // (`best`, la plus petite distance rencontrée, en pixels). Les bords intérieurs (deux matières, + // une arête, un arrondi serré) sont suréchantillonnés : `STAGE_EDGE` sonde le modèle à un + // demi-pixel du point touché, dans son plan tangent ; si la matière change ou que la surface + // s'écarte, trois rayons de plus (`MODEL_SUBPIXEL`) sont marchés et ombrés, avec l'occlusion et + // les ombres du premier. Chaque échantillon passe par `STAGE_SHADE`, seul appel de + // `model_shade` ; le pixel est leur moyenne. + float3 ray = rd; + float2 tb = ray_box(ro, ray, lo - 0.02, hi + 0.02); int stage = tb.x < tb.y && tb.y > 0.0 ? STAGE_MARCH : STAGE_DONE; + // Prism Glow : le cristal est tracé au départ de chaque rayon ; `tc` est + // son point touché (1e9 : raté), `hc` sa facette. La marche du serti s'arrête à lui. + bool need_tc = stage == STAGE_MARCH; + float tc = 1e9; + int hc = -1; // Le point du plan derrière le pixel est à régler avant le prochain tour. bool plane_next = stage == STAGE_DONE; int k = 0; float t = max(tb.x, 0.0); float best = 1e9; float t_best = t; + float d_best = 0.0; float mat = 0.0; float cov = 0.0; + float cov_s = 0.0; float3 q = 0.0; float3 n = 0.0; float3 L = 0.0; @@ -1371,14 +2585,24 @@ float4 cursor_model(float2 local) float inside = 0.0; float contact = 0.0; float dropped = 0.0; - [loop] for (int it = 0; it < 180; it++) + // Sous-échantillons restants, échantillons ombrés, et leur somme (couleur × couverture, + // couverture). + int sub = 0; + int shaded = 0; + float4 acc = 0.0; + [loop] for (int it = 0; it < 500; it++) { + if (need_tc) + { + need_tc = false; + tc = prism_primary(ro, ray, hc); + } if (plane_next) { // Le plan, là où le modèle ne couvre pas tout le pixel : ombre portée et ombre de // contact, seulement à l'intérieur de l'écran (`mb.xy` = sa demi-taille, px du plan). plane_next = false; - stage = STAGE_DONE; + stage = cov > 0.0 ? STAGE_SHADE : STAGE_DONE; float denom = dot(rd, nz); if (cov < 1.0 && denom < -1e-4) { @@ -1393,15 +2617,63 @@ float4 cursor_model(float2 local) } } } + if (stage == STAGE_SHADE) + { + if (cov_s > 0.0) + { + float3 c; + if (mat > PRISM_MAT - 0.5) + { + c = prism_shade(q, n, ray, hc, t_best / dlen, l, fill, nz, hz, tip, unit, f); + } + else + { + float3 tl = lamp - q; + float fall = SCULPT_LAMP_DIST * SCULPT_LAMP_DIST / dot(tl, tl); + c = model_shade(q, n, ray, normalize(tl), fall, sh, ao, mat, l, fill); + } + acc += float4(c * cov_s, cov_s); + } + shaded++; + stage = STAGE_DONE; + if (sub > 0) + { + // Le rayon suivant, décalé dans le pixel ; il repart de sa propre entrée dans la + // boîte, pour ne pas manquer ce qui passerait devant le premier. + float3 dws = float3(local + MODEL_SUBPIXEL[3 - sub] + src.xy, -persp); + sub--; + dlen = length(dws); + ray = plane_to_model(world_to_plane(dws / dlen, f), f); + tb = ray_box(ro, ray, lo - 0.02, hi + 0.02); + cov_s = 0.0; + stage = STAGE_SHADE; + if (tb.x < tb.y && tb.y > 0.0) + { + stage = STAGE_MARCH; + k = 0; + t = max(tb.x, 0.0); + best = 1e9; + t_best = t; + need_tc = true; + } + } + continue; + } float3 pos; if (stage == STAGE_MARCH) { - pos = ro + rd * t; + pos = ro + ray * t; } else if (stage == STAGE_NORMAL) { pos = q + 0.002 * model_tetra(k); } + else if (stage == STAGE_EDGE) + { + float3 side = normalize(cross(n, ray)); + float3 e = (k >= 2 ? cross(n, side) : side) * (k % 2 == 1 ? -1.0 : 1.0); + pos = q + e * 0.5 * t_best / dlen; + } else if (stage == STAGE_AO) { pos = q + (0.01 + 0.0175 * k) * SCULPT_SCALE * n; @@ -1428,22 +2700,68 @@ float4 cursor_model(float2 local) { best = hit ? 0.0 : d / fp; t_best = t; + d_best = d; mat = m.y; } t += d * stride; k++; + if (!hit && t >= tc) + { + // Le pas dépasse le cristal : c'est lui que le rayon touche, avant le serti. + hit = true; + best = 0.0; + t_best = tc; + d_best = 0.0; + mat = PRISM_MAT; + } if (hit || t > tb.y || k == 96) { - cov = saturate(1.0 - best); - if (cov > 0.0) + cov_s = saturate(1.0 - best); + if (shaded > 0) { - q = ro + rd * t_best; - stage = STAGE_NORMAL; - k = 0; + stage = STAGE_SHADE; + if (cov_s > 0.0) + { + q = ro + ray * t_best; + n = 0.0; + stage = STAGE_NORMAL; + k = 0; + } } else { - plane_next = true; + cov = cov_s; + if (cov > 0.0) + { + q = ro + ray * t_best; + stage = STAGE_NORMAL; + k = 0; + } + else + { + plane_next = true; + } + } + if (stage == STAGE_NORMAL && mat > PRISM_MAT - 0.5) + { + // Le cristal : la normale de sa facette, sans sondes ; ni occlusion ni ombre + // propre. Un bord réel à moins d'un pixel (le serti, un pli) appelle les trois + // rayons de plus, comme les bords intérieurs des autres modèles. + n = prism_normal(hc, ray); + k = 0; + if (shaded > 0) + { + stage = STAGE_SHADE; + } + else + { + if (prism_edge_px(q, hc, t_best / dlen) < 0.75) + { + sub = 3; + } + stage = STAGE_DONE; + plane_next = true; + } } } } @@ -1454,6 +2772,34 @@ float4 cursor_model(float2 local) if (k == 4) { n = normalize(n); + k = 0; + stage = shaded > 0 ? STAGE_SHADE : STAGE_EDGE; + } + } + else if (stage == STAGE_EDGE) + { + // Autre matière, ou surface qui s'écarte du plan tangent de plus de 2 % d'un pixel + // (le point touché flotte déjà de `d_best` au-dessus d'elle). + if (m.y != mat || abs(d - d_best) > 0.02 * t_best / dlen) + { + sub = 3; + } + if (k == 0 && mat > 7.5) + { + // Le serti de Prism Glow au bord du cristal : son contour à moins d'un pixel. + int ps = prism_shape(); + if (ps >= 0) + { + float dout = prism_poly_dist(q.xy, PRISM_OUT_START[ps], PRISM_OUT_COUNT[ps]); + if (abs(dout) < 0.75 * t_best / dlen) + { + sub = 3; + } + } + } + k++; + if (k == 4) + { stage = STAGE_AO; k = 0; ao = 0.0; @@ -1504,7 +2850,7 @@ float4 cursor_model(float2 local) k = 1; if (!(tbp.x < tbp.y && tbp.y > 0.0)) { - stage = STAGE_DONE; + stage = cov > 0.0 ? STAGE_SHADE : STAGE_DONE; } } else @@ -1516,21 +2862,15 @@ float4 cursor_model(float2 local) { res = saturate(res); dropped = 1.0 - res * res * (3.0 - 2.0 * res); - stage = STAGE_DONE; + stage = cov > 0.0 ? STAGE_SHADE : STAGE_DONE; } } } - float3 rgb = 0.0; - if (cov > 0.0) - { - float3 tl = lamp - q; - float fall = SCULPT_LAMP_DIST * SCULPT_LAMP_DIST / dot(tl, tl); - rgb = model_shade(q, n, rd, normalize(tl), fall, sh, ao, mat, l, fill); - } + float w = 1.0 / max(shaded, 1); float shadow = inside * max(dropped * MODEL_SHADOW_ALPHA, contact * MODEL_CONTACT_ALPHA); - float a = cov * color.a; - return float4(rgb * a, a + (1.0 - a) * shadow * color.a); // prémultiplié, ombre noire + float a = acc.a * w * color.a; + return float4(acc.rgb * w * color.a, a + (1.0 - a) * shadow * color.a); // prémultiplié, ombre noire } // ============ Impact du clic (mode 16) ============ @@ -2154,7 +3494,8 @@ float4 device_frame(float2 local) return float4(rgb * a, a); // prémultiplié } -float4 ps_main(VSOut i) : SV_Target +// The body of `ps_main`, which only adds the layer's transparency on top of it. +float4 ps_layer(VSOut i) { // mode 18 : l'écran CADRÉ (ombre, cadre, métrage, appareil) flouté comme UN objet rigide // (`FrameGeometry::screen_trail`). t2 = son rendu isolé, prémultiplié, à la taille de la @@ -2213,6 +3554,7 @@ float4 ps_main(VSOut i) : SV_Target return acc / max(n, 1.0); } +#ifdef LAYER_MODELS // mode 17 : CADRE D'APPAREIL MODELÉ (cf. `device_frame`). Testé en premier, comme le 16. if (mode > 16.5) { @@ -2236,6 +3578,14 @@ float4 ps_main(VSOut i) : SV_Target } return cursor_model(i.local); } +#else + // Les modèles 3D n'existent que dans `ps_main_models` ; un draw qui les enverrait ici ne + // peint rien plutôt qu'une ombre faite de leurs emplacements. + if (mode > 14.5) + { + return float4(0.0, 0.0, 0.0, 0.0); + } +#endif // mode 14 : CADRE DE FENÊTRE autour de l'écran (barre de titre, trois pastilles, filet), // dessiné SOUS lui. Testé avant le mode 13, dont la branche n'a pas de borne haute. @@ -2490,8 +3840,12 @@ float4 ps_main(VSOut i) : SV_Target float2 plane_px = dst_prev.xy; float2 p = float2(r.x, r.y) * plane_px - plane_px * 0.5; float rad = max(radius_px, 0.0); - float d = (dst_prev.z > 0.5) ? sd_screen_under_bar(p, plane_px * 0.5, rad, color.z) - : sd_round_rect(p, plane_px * 0.5, rad); + // Une branche et non `?:`, qui évalue ses DEUX opérandes (trois SDF par pixel). + float d; + if (dst_prev.z > 0.5) + d = sd_screen_under_bar(p, plane_px * 0.5, rad, color.z); + else + d = sd_round_rect(p, plane_px * 0.5, rad); float tilt_a = 1.0 - smoothstep(0.0, 1.5, d); // Slot d'un layout en bloc (`color.w` = rayon de ses coins, px ; 0 ailleurs, sans effet) : // `dst` EST le slot, dont le rect arrondi rogne le plan. Le conteneur masque, le métrage @@ -2653,6 +4007,8 @@ float4 ps_main(VSOut i) : SV_Target float3 rgb; // 1 sauf en mode detourage, ou il porte le masque du sujet (cf. la branche fx.z ci-dessous). float alpha_mask = 1.0; + // 0 where a camera homography finds no camera point (`persp_camera`), 1 everywhere else. + float in_picture = 1.0; if (mode < 0.5) { // flou de mouvement par vélocité (§8) : pour CE pixel sortie, uv à la frame @@ -2661,6 +4017,17 @@ float4 ps_main(VSOut i) : SV_Target float2 uv_now = i.uv; float2 localp = (i.pout - dst_prev.xy) / dst_prev.zw; float2 uv_prev = src_prev.xy + localp * (src_prev.zw - src_prev.xy); + if (layer_fx.y > 0.5) + { + // Camera homography: the texture uv comes from the fragment's place in the quad + // (`src` is not used), scaled to the valid part of the decoder texture. The previous + // frame's uv is the same map at the quad's previous place. + float3 cam = persp_camera((i.pout - dst.xy) / dst.zw); + in_picture = cam.z; + uv_now = cam.xy * fx.xy; + float3 cam_prev = persp_camera(localp); + uv_prev = (cam_prev.z > 0.5) ? cam_prev.xy * fx.xy : uv_now; + } float2 duv = uv_now - uv_prev; float mb_scale = saturate(mb.y); float2 duv_blur = duv * mb_scale; @@ -2701,13 +4068,30 @@ float4 ps_main(VSOut i) : SV_Target } else if (effect > 1.5) { - rgb = lerp(blur_webcam_bg(uv_now, fx.w, quad_px, i.local), rgb, person); + rgb = lerp(blur_webcam_bg(uv_now, fx.w, quad_px, i.local, fx.xy), rgb, person); } else { alpha_mask = person; } } + + // Desk-view cover: the camera is being tilted, so the whole picture is blurred and + // dimmed (cover.x = strength, cover.y = radius in quad px, cover.z = dim at full cover). + if (cover.x > 0.001) + { + float3 hidden = blur_webcam_radius(uv_now, cover.y, quad_px, i.local, fx.xy); + rgb = lerp(rgb, hidden, cover.x) * (1.0 - cover.z * cover.x); + } + + // Past the camera picture the layer is black, as opaque as the rest of it (spec: a + // corrected crop that reaches past the camera image has a black edge, never the + // wallpaper). The mask and effects have no picture to act on there. + if (in_picture < 0.5) + { + rgb = float3(0.0, 0.0, 0.0); + alpha_mask = 1.0; + } } else { @@ -2726,13 +4110,24 @@ float4 ps_main(VSOut i) : SV_Target // cadre (`sd_screen_under_bar`, `mb.z` = la remontée du contour intérieur, px). float2 halfsz = quad_px * 0.5; float2 p = i.local - quad_px * 0.5; - float d = (mb.w > 0.5) ? sd_screen_under_bar(p, halfsz, radius_px, mb.z) - : sd_round_rect(p, halfsz, radius_px); + // Une branche et non `?:`, qui évalue ses DEUX opérandes : trois SDF par pixel de l'écran + // au lieu d'une -- 0,6 ms par frame 1080p sur une Radeon 610M (mesuré en WGSL). + float d; + if (mb.w > 0.5) + d = sd_screen_under_bar(p, halfsz, radius_px, mb.z); + else + d = sd_round_rect(p, halfsz, radius_px); alpha *= 1.0 - smoothstep(0.0, 1.5, d); // ~1.5px feather (§7 fwidth-like) } return float4(rgb * alpha, alpha); // prémultiplié } +float4 ps_main(VSOut i) : SV_Target +{ + // Premultiplied, so the transparency scales all four channels; 0 leaves them as they are. + return ps_layer(i) * (1.0 - layer_fx.x); +} + // ============ RGB -> NV12 (§5) : deux passes vers les plans d'une texture NV12 ============ // VS plein écran (triangle unique) qui expose l'UV. struct FSOut { float4 pos : SV_Position; float2 uv : TEXCOORD0; }; diff --git a/crates/compositor/src/shaders.metal b/crates/compositor/src/shaders.metal index 6c019c3ec8..f2e5bee75d 100644 --- a/crates/compositor/src/shaders.metal +++ b/crates/compositor/src/shaders.metal @@ -33,6 +33,12 @@ // `compositor_macos::tests::every_shader_entry_point_compiles` le compile sur le device // système au `cargo test`, pour qu'une faute de syntaxe MSL ne se découvre pas à // l'ouverture de l'éditeur chez un utilisateur. +// +// Il est compilé DEUX fois (`compositor_macos.rs::new_sized`) : tel quel pour tous les calques, +// et avec `LAYER_MODELS` défini pour les seuls modèles 3D (modes 15 à 17, +// `LayerCB::needs_models`). Un shader alloue à chaque draw les registres de sa branche la plus +// lourde, et ce sont ces modèles ray-tracés : le même `ps_main` les portait pour le fond, +// l'ombre et la vidéo, qui couvrent toute la sortie (cf. le haut de `shaders.hlsl`). #include using namespace metal; @@ -42,7 +48,7 @@ using namespace metal; // ================================================================================= // // Le moteur côté CPU upload ce buffer via `setVertexBytes` (vertex stage) et -// `setFragmentBytes` (fragment stage) avant chaque draw — la copie est de 176 octets, +// `setFragmentBytes` (fragment stage) avant chaque draw — la copie est de 256 octets, // ce qui est sous le seuil d'alignement 4K de Metal pour le mode « immediate ». struct Layer @@ -60,6 +66,9 @@ struct Layer float4 trail_a; // mode 8 : coins TL, TR du plan à la frame précédente (px locaux, comme fx) ; mode 18 incliné : en fractions de sortie float4 trail_b; // mode 8 : coins BR, BL du plan à la frame précédente (comme src_prev) ; mode 18 incliné : en fractions de sortie float4 trail_mb; // mode 8 : x = taps, y = force du flou de mouvement (ceux du mode 0) ; 0 ailleurs + float4 cover; // x = desk-view cover 0..1, y = blur radius (quad px), z = dim, w unused + float4 persp[3]; // mode 0: rows of the homography quad point (0..1 in dst) -> camera uv (0..1 of the valid frame), xyz; read when layer_fx.y = 1 + float4 layer_fx; // x = transparency 0..1 (0 = opaque), every mode; y = 1 when persp applies; z, w unused }; // Mode 15 (curseur modélisé) : le détail des emplacements est dans `frame_geometry.rs`, en tête // de la section « Curseur modélisé » (`cursor_model_cb`). Mode 17 (appareil modelé) : en tête de @@ -170,9 +179,13 @@ inline float sd_segment(float2 p, float2 a, float2 b) inline float sd_round_rect(float2 p, float2 halfsz, float r) { float hmin = min(halfsz.x, halfsz.y); - if (r <= 0.0 || hmin <= 0.0) + float2 q0 = abs(p) - halfsz; + // Hors des coins, la distance est celle du rectangle vif quel que soit l'exposant : un seul + // axe déborde de l'étendue E, qui s'y ajoute puis s'en retranche. E ne dépasse pas 1,42 r + // (n = 3) : sous 1,5 r du bord sur un axe, on rend la boîte sans calculer `n` ni E, deux + // transcendantes de moins sur presque tous les pixels d'un calque arrondi. + if (r <= 0.0 || hmin <= 0.0 || min(q0.x, q0.y) <= -min(1.5 * r, hmin)) { - float2 q0 = abs(p) - halfsz; return length(max(q0, 0.0)) + min(max(q0.x, q0.y), 0.0); } float n = 3.0 - smoothstep(0.5, 1.0, r / hmin); @@ -414,12 +427,17 @@ constant float3 VOGEL_TAPS[21] = { float3(-0.633036, -0.758588, 0.087119) }; -inline float3 blur_webcam_bg(float2 uv, float intensity, float2 qpx, float2 local_px, - texture2d texY, - texture2d texUV) +// Vogel-disc blur of the camera texture at a radius in quad pixels. `valid` is the part of the +// texture the picture fills (fx.xy): decoders allocate aligned textures (a 1080-line camera in a +// 1088-line texture), so each tap is clamped half a chroma texel inside it, never into padding. +inline float3 blur_webcam_radius(float2 uv, float max_r_px, float2 qpx, float2 local_px, + float2 valid, + texture2d texY, + texture2d texUV) { - float max_r_px = max(intensity, 0.0) * 22.0 + 1.5; float2 step = max_r_px / max(qpx, float2(1.0)); + float2 chroma = max(float2(float(texUV.get_width()), float(texUV.get_height())), float2(1.0)); + float2 hi = max(valid - 0.5 / chroma, float2(0.0)); float noise = fract(52.9829189 * fract(0.06711056 * local_px.x + 0.00583715 * local_px.y)); float angle = noise * 6.2831853; float s = sin(angle); @@ -431,12 +449,34 @@ inline float3 blur_webcam_bg(float2 uv, float intensity, float2 qpx, float2 loca float2 p = VOGEL_TAPS[k].xy; float w = VOGEL_TAPS[k].z; float2 rot_p = float2(p.x * c - p.y * s, p.x * s + p.y * c); - sum += sample_yuv(saturate(uv + rot_p * step), texY, texUV) * w; + sum += sample_yuv(clamp(uv + rot_p * step, float2(0.0), hi), texY, texUV) * w; total += w; } return sum / max(total, 1e-4); } +// Camera homography (`persp`, read when `layer_fx.y` = 1): the camera point (0..1 of its valid +// frame) seen at `local` (0..1 in the layer's dst quad), and in z whether there is one -- 0 when +// the point is behind the projection (q.z <= 0) or outside the camera frame, where the layer is +// black. Mirror of the HLSL `persp_camera`. +inline float3 persp_camera(constant Layer &layer, float2 local) +{ + float3 p = float3(local, 1.0); + float3 q = float3(dot(layer.persp[0].xyz, p), dot(layer.persp[1].xyz, p), dot(layer.persp[2].xyz, p)); + float2 cam = q.xy / max(q.z, 1e-6); + bool inside = q.z > 0.0 && all(cam >= 0.0) && all(cam <= 1.0); + return float3(cam, inside ? 1.0 : 0.0); +} + +inline float3 blur_webcam_bg(float2 uv, float intensity, float2 qpx, float2 local_px, + float2 valid, + texture2d texY, + texture2d texUV) +{ + return blur_webcam_radius(uv, max(intensity, 0.0) * 22.0 + 1.5, qpx, local_px, valid, texY, + texUV); +} + // Hash 2D -> [0,1) sans sin(). Miroir de `hash12` côté HLSL. inline float hash12(float2 p) { @@ -545,7 +585,9 @@ inline float3 image_motion(float2 q, float time, float motion, float aspect) // ============ Curseur MODÉLISÉ (mode 15) ============ // Port ligne pour ligne de `cursor_model` (HLSL), dont les commentaires font foi ; seules // différences : `layer` et les textures arrivent en paramètres (le sprite en texture(2), son -// champ R16F en texture(4)), `lerp` s'écrit `mix`, `SampleLevel` s'écrit `sample(…, level(0.0))`, +// champ R16F en texture(4), la copie de l'image composée en texture(5) pour le cristal de Prism +// Glow), +// `lerp` s'écrit `mix`, `SampleLevel` s'écrit `sample(…, level(0.0))`, un `out` s'écrit `thread &`, // et `pow` veut un exposant du type de sa base. // Constantes : miroir exact de `frame_geometry.rs` (MODEL_*) et de `sculpt.rs` (SCULPT_*). constant float MODEL_BEVEL = 0.045; @@ -590,8 +632,6 @@ inline int sculpt_id(constant Layer &layer) constant float SCULPT_SCALE = 0.85; constant float SCULPT_HOVER = 0.05; constant float SCULPT_VOX = 0.0625; -constant float SCULPT_AR_ROUND = 0.03; -constant float SCULPT_HAND_ZC = 0.185; constant float SCULPT_ZREF_ARROW = 0.2; constant float SCULPT_ZREF_HAND = 0.185; constant float SCULPT_LAMP_DIST = 1.9; @@ -613,13 +653,6 @@ inline float2 s_opu(float2 a, float2 b) return a.x < b.x ? a : b; } -static float s_capsule(float3 p, float3 a, float3 b, float r) -{ - float3 pa = p - a, ba = b - a; - float h = saturate(dot(pa, ba) / dot(ba, ba)); - return length(pa - ba * h) - r; -} - static float s_round_box(float3 p, float3 b, float r) { float3 q = abs(p) - b + r; @@ -646,24 +679,76 @@ inline float2 s_rot(float2 v, float a) return float2(c * v.x - s * v.y, s * v.x + c * v.y); } -constant float2 SCULPT_ARROW[7] = { - float2(0.0, 0.0), float2(0.0, -0.86), float2(0.215, -0.665), float2(0.37, -1.0), - float2(0.53, -0.93), float2(0.38, -0.60), float2(0.64, -0.60) +static float s_star5(float2 p, float r, float rf) +{ + const float2 k1 = float2(0.809016994375, -0.587785252292); + const float2 k2 = float2(-0.809016994375, -0.587785252292); + p.x = abs(p.x); + p -= 2.0 * max(dot(k1, p), 0.0) * k1; + p -= 2.0 * max(dot(k2, p), 0.0) * k2; + p.x = abs(p.x); + p.y -= r; + float2 ba = rf * float2(-k1.y, k1.x) - float2(0.0, 1.0); + float h = clamp(dot(p, ba) / dot(ba, ba), 0.0, r); + return length(p - ba * h) * sign(p.y * ba.x - p.x * ba.y); +} + +static float s_vesica(float2 p, float r, float d) +{ + p = abs(p); + float b = sqrt(r * r - d * d); + return (p.y - b) * d > p.x * b ? length(p - float2(0.0, b)) : length(p + float2(d, 0.0)) - r; +} + +// Curseurs cercles (cf. HLSL) : Studio Ink, Pop Coral, Star Sprout. +constant float2 RIM_POLY[28] = { + float2(0.0, -0.06), float2(0.0, -0.7712), float2(0.165, -0.6325), float2(0.313, -0.9318), + float2(0.4234, -0.8557), float2(0.2672, -0.5686), float2(0.456, -0.5393), + float2(-0.173, -0.5237), float2(-0.016, -0.75), float2(0.329, -0.75), float2(0.411, -0.592), + float2(0.411, -0.43), float2(0.19, -0.435), float2(-0.03, -0.44), + float2(-0.1825, -0.6215), float2(0.0037, -0.9195), float2(0.3462, -0.9195), float2(0.467, -0.6906), + float2(0.467, -0.49), float2(0.2, -0.49), float2(-0.03, -0.49), + float2(-0.1807, -0.5534), float2(0.0194, -0.872), float2(0.3485, -0.872), float2(0.47, -0.6795), + float2(0.47, -0.49), float2(0.2, -0.49), float2(-0.03, -0.49) +}; + +constant float4 RIM_GLOVE[27] = { + float4(0.0, -0.1077, 0.0, -0.6), float4(0.1493, -0.3279, 0.1493, -0.6), + float4(0.2863, -0.3654, 0.2863, -0.6), float4(0.4193, -0.4043, 0.4193, -0.6), + float4(-0.2, -0.478, -0.075, -0.625), float4(0.0788, -0.25, 0.0788, -0.3992), + float4(0.2188, -0.3, 0.2188, -0.4234), float4(0.3552, -0.33, 0.3552, -0.4534), + float4(-0.0845, -0.5386, 0.762, 0.648), + float4(0.0, -0.1186, 0.0, -0.65), float4(0.1614, -0.3838, 0.1614, -0.65), + float4(0.3177, -0.427, 0.3177, -0.65), float4(0.4645, -0.4848, 0.4645, -0.65), + float4(-0.215, -0.535, -0.1208, -0.6864), float4(0.0832, -0.3, 0.0832, -0.48), + float4(0.2414, -0.35, 0.2414, -0.482), float4(0.3959, -0.41, 0.3959, -0.533), + float4(-0.122, -0.5718, 0.867, 0.498), + float4(0.0, -0.122, 0.0, -0.65), float4(0.172, -0.387, 0.172, -0.65), + float4(0.3267, -0.427, 0.3267, -0.65), float4(0.4727, -0.4867, 0.4727, -0.65), + float4(-0.199, -0.483, -0.102, -0.645), float4(0.0927, -0.3, 0.0927, -0.476), + float4(0.2493, -0.35, 0.2493, -0.478), float4(0.402, -0.41, 0.402, -0.52), + float4(-0.0993, -0.5267, 0.858, 0.514) +}; + +constant float4 RIM_GLOVE_R[9] = { + float4(0.0552, 0.047, 0.045, 0.0375), float4(0.052, 0.0235, 0.0225, 0.0225), float4(0.0552, -0.548, 0.0525, 0.0), + float4(0.064, 0.0585, 0.0585, 0.0515), float4(0.06, 0.0197, 0.0178, 0.0172), float4(0.0648, -0.6, 0.058, 0.0), + float4(0.066, 0.0553, 0.0553, 0.0507), float4(0.063, 0.024, 0.022, 0.02), float4(0.0647, -0.555, 0.056, 0.0) }; -static float s_arrow2(float2 p) +static float s_rim_poly(float2 p, int base) { - float d = dot(p - SCULPT_ARROW[0], p - SCULPT_ARROW[0]); + float d = dot(p - RIM_POLY[base], p - RIM_POLY[base]); float s = 1.0; - int j = 6; - for (int i = 0; i < 7; i++) + int j = base + 6; + for (int i = base; i < base + 7; i++) { - float2 e = SCULPT_ARROW[j] - SCULPT_ARROW[i]; - float2 w = p - SCULPT_ARROW[i]; + float2 e = RIM_POLY[j] - RIM_POLY[i]; + float2 w = p - RIM_POLY[i]; float2 b = w - e * saturate(dot(w, e) / dot(e, e)); d = min(d, dot(b, b)); - bool c0 = p.y >= SCULPT_ARROW[i].y; - bool c1 = p.y < SCULPT_ARROW[j].y; + bool c0 = p.y >= RIM_POLY[i].y; + bool c1 = p.y < RIM_POLY[j].y; bool c2 = e.x * w.y > e.y * w.x; if ((c0 && c1 && c2) || (!c0 && !c1 && !c2)) { @@ -674,276 +759,1190 @@ static float s_arrow2(float2 p) return s * sqrt(d); } -static float s_star5(float2 p, float r, float rf) +static float2 s_rim_glove(float2 p, int g, float palm) { - const float2 k1 = float2(0.809016994375, -0.587785252292); - const float2 k2 = float2(-0.809016994375, -0.587785252292); - p.x = abs(p.x); - p -= 2.0 * max(dot(k1, p), 0.0) * k1; - p -= 2.0 * max(dot(k2, p), 0.0) * k2; - p.x = abs(p.x); - p.y -= r; - float2 ba = rf * float2(-k1.y, k1.x) - float2(0.0, 1.0); - float h = clamp(dot(p, ba) / dot(ba, ba), 0.0, r); - return length(p - ba * h) * sign(p.y * ba.x - p.x * ba.y); + float4 r0 = RIM_GLOVE_R[3 * g]; + float4 r1 = RIM_GLOVE_R[3 * g + 1]; + float4 r2 = RIM_GLOVE_R[3 * g + 2]; + int i = 9 * g; + float f = min(sd_segment(p, RIM_GLOVE[i].xy, RIM_GLOVE[i].zw) - r0.x, + sd_segment(p, RIM_GLOVE[i + 1].xy, RIM_GLOVE[i + 1].zw) - r0.y); + f = min(f, sd_segment(p, RIM_GLOVE[i + 2].xy, RIM_GLOVE[i + 2].zw) - r0.z); + f = min(f, sd_segment(p, RIM_GLOVE[i + 3].xy, RIM_GLOVE[i + 3].zw) - r0.w); + f = min(f, sd_segment(p, RIM_GLOVE[i + 4].xy, RIM_GLOVE[i + 4].zw) - r1.x); + float grooves = min(sd_segment(p, RIM_GLOVE[i + 5].xy, RIM_GLOVE[i + 5].zw) - r1.y, + sd_segment(p, RIM_GLOVE[i + 6].xy, RIM_GLOVE[i + 6].zw) - r1.z); + grooves = min(grooves, sd_segment(p, RIM_GLOVE[i + 7].xy, RIM_GLOVE[i + 7].zw) - r1.w); + float sil = s_smin(palm, f, 0.03); + float4 v = RIM_GLOVE[i + 8]; + float wedge = max(max(-dot(p - v.xy, v.zw), p.x + r2.x), r2.y - p.y); + return float2(sil, max(sil, -min(grooves, wedge))); } -static float s_arrow_solid(float3 p, float h, float re, float dome) +static float2 s_piece(float d, float dc, float sil, float z, float w, float zt, float h, float bump, float mat) { - float d2 = s_arrow2(p.xy) - SCULPT_AR_ROUND; - float hh = h + dome * smoothstep(0.0, 0.07, -d2); - return s_extrude(d2, p.z - (SCULPT_HOVER + h + dome), hh, re); + float tray = s_extrude(sil - w, z - 0.5 * (SCULPT_HOVER + zt), 0.5 * (zt - SCULPT_HOVER), 0.012); + float bead = length(float2(d - 0.5 * w, z - zt)) - 0.5 * w; + float u = saturate(-dc / 0.045); + float cushion = 0.8 * s_extrude(dc, z - zt, h + 0.022 * u * (2.0 - u) + bump, 0.75 * h); + return s_opu(float2(min(tray, bead), 5.0), float2(cushion, mat)); } -static float s_piping(float3 p, float ztop) +static float2 s_paper(float d, float sil, float z, float w) { - float d2 = s_arrow2(p.xy) - SCULPT_AR_ROUND; - return length(float2(d2 + 0.075, p.z - ztop)) - 0.017; + float navy = s_extrude(sil - w, z - 0.5 * (SCULPT_HOVER + 0.15), 0.5 * (0.15 - SCULPT_HOVER), 0.006); + float sheet = s_extrude(d, z - 0.16, 0.01, 0.004); + return s_opu(float2(navy, 5.0), float2(sheet, 1.0)); } -static float s_glove(float3 p, float zc) +inline float s_bump(float2 p, float2 c, float r) { - float index = s_capsule(p, float3(0.0, -0.10, zc), float3(0.0, -0.52, zc), 0.098); - float palm = s_round_box(p - float3(0.185, -0.70, zc), float3(0.255, 0.19, 0.105), 0.1); - float f1 = s_capsule(p, float3(0.17, -0.57, zc + 0.012), float3(0.17, -0.41, zc + 0.045), 0.086); - float f2 = s_capsule(p, float3(0.31, -0.59, zc + 0.01), float3(0.31, -0.45, zc + 0.04), 0.08); - float f3 = s_capsule(p, float3(0.435, -0.625, zc + 0.005), float3(0.435, -0.52, zc + 0.03), 0.07); - float thumb = s_capsule(p, float3(0.03, -0.77, zc + 0.03), float3(-0.165, -0.60, zc + 0.065), 0.082); - float d = s_smin(palm, min(f1, min(f2, f3)), 0.035); - d = s_smin(d, index, 0.05); - return s_smin(d, thumb, 0.05); + float k = saturate(1.0 - dot(p - c, p - c) / (r * r)); + return k * k; } -static float s_cuff(float3 p, float zc) +static float s_dash(float2 p, float2 a, float2 b, float ra, float rb) { - float3 q = p - float3(0.185, -0.925, zc); - float2 ab = float2(0.272, 0.12); - float e = (length(q.xz / ab) - 1.0) * min(ab.x, ab.y); - return s_extrude(e, q.y, 0.07, 0.06); + p -= a; + b -= a; + float hb = dot(b, b); + float2 q = float2(abs(dot(p, float2(b.y, -b.x))), dot(p, b)) / hb; + float2 c = float2(sqrt(hb - (ra - rb) * (ra - rb)), ra - rb); + float k = c.x * q.y - c.y * q.x; + if (k < 0.0) + { + return sqrt(hb * dot(q, q)) - ra; + } + if (k > c.x) + { + return sqrt(hb * (dot(q, q) + 1.0 - 2.0 * q.y)) - rb; + } + return dot(c, q) - ra; } -static float2 s_sprout(float3 p, float3 c) +static float2 s_rimmed(float3 p, int theme, int shape) { - float3 q = p - c; - float st2 = s_star5(q.xy, 0.125, 0.52) - 0.022; - float2 r = float2(s_extrude(st2, q.z, 0.038, 0.034), 3.0); - float3 e = float3(abs(q.x) - 0.032, q.y + 0.005, q.z - 0.036); - r = s_opu(r, float2(length(e) - 0.0135, 5.0)); - float3 l1 = float3(s_rot(q.xy - float2(-0.04, 0.15), -0.6), q.z); - float3 l2 = float3(s_rot(q.xy - float2(0.045, 0.155), 0.7), q.z); - float leaves = min(s_ellipsoid(l1, float3(0.03, 0.058, 0.02)), s_ellipsoid(l2, float3(0.03, 0.058, 0.02))); - return s_opu(r, float2(leaves, 4.0)); -} - -// Pixel Candy : une ligne par entier, bit c = colonne c (cf. HLSL et `sculpt.rs`). -constant int SCULPT_ARROWPIX[16] = { 0, 1, 3, 7, 31, 63, 127, 255, 511, 63, 55, 115, 113, 224, 224, 64 }; -constant int SCULPT_ARROWBACK[18] = { - 3, 7, 15, 31, 63, 127, 255, 511, 1023, 2047, 4095, 255, 511, 511, 999, 995, 960, 192 + bool arrow = shape == 0; + int g = theme == 4 ? 0 : (theme == 0 ? 1 : 2); + float poly = s_rim_poly(p.xy, arrow ? 0 : 7 * g + 7); + float2 body = float2(poly, poly); + float w = 0.06, zt = 0.17, h = 0.03; + float bump = 0.022 * s_bump(p.xy, float2(0.15, -0.45), 0.3); + float back = 1e9; + if (!arrow) + { + w = RIM_GLOVE_R[3 * g + 2].z; + for (int i = 0; i < 2; i++) + { + float2 q = p.xy + (i == 1 ? float2(0.03, 0.04) : float2(0.0, 0.0)); + float2 v = s_rim_glove(q, g, (i == 1 ? s_rim_poly(q, 7 * g + 7) : poly) - 0.05); + if (i == 1) + { + back = v.x - w; + } + else + { + body = v; + if (theme != 2 || v.x - w <= 0.0) + { + break; + } + } + } + zt = 0.185; + h = 0.028; + bump = 0.03 * s_bump(p.xy, float2(0.19, -0.62), 0.28); + } + float dc = body.y; + if (arrow && theme == 0) + { + body -= 0.015; + w = 0.045; + dc = 1.0; + } + float2 r = theme == 2 ? s_paper(body.y, body.x, p.z, w) + : s_piece(body.y, dc, body.x, p.z, w, zt, h, bump, 1.0); + if (theme == 0) + { + if (arrow) + { + float band = s_extrude(abs(poly + 0.0075) - 0.0225, p.z - zt, 0.028, 0.012); + r = s_opu(r, float2(band, 2.0)); + } + return r; + } + if (theme == 2) + { + float dash; + if (arrow) + { + float back = s_rim_poly(p.xy + float2(0.025, 0.05), 0) - 0.06; + r = s_opu(r, float2(s_extrude(back, p.z - 0.08, 0.03, 0.006), 2.0)); + dash = min(s_dash(p.xy, float2(-0.1179, -0.1374), float2(-0.1799, -0.0443), 0.028, 0.045), + s_dash(p.xy, float2(-0.1347, -0.2493), float2(-0.248, -0.2056), 0.026, 0.042)); + } + else + { + r = s_opu(r, float2(s_extrude(back, p.z - 0.08, 0.03, 0.006), 3.0)); + dash = min(s_dash(p.xy, float2(0.216, -0.1695), float2(0.2593, -0.0685), 0.028, 0.042), + s_dash(p.xy, float2(0.3054, -0.2339), float2(0.3949, -0.1652), 0.0275, 0.041)); + } + return s_opu(r, float2(s_extrude(dash, p.z - 0.11, 0.06, 0.006), 3.0)); + } + if (!arrow) + { + float2 cq = p.xy - float2(0.1541, -0.9047); + float2 bq = abs(float2(cq.x, cq.y - 0.2066 * cq.x * cq.x)) - float2(0.214, 0.0361); + float cuff = length(max(bq, 0.0)) + min(max(bq.x, bq.y), 0.0) - 0.03; + r = s_opu(r, s_piece(cuff, cuff, cuff, p.z, 0.0477, 0.225, 0.025, 0.0, 2.0)); + } + float2 c = arrow ? float2(0.6118, -0.8079) : float2(0.15, -0.885); + float rs = arrow ? 0.1678 : 0.128; + float zs = arrow ? 0.23 : 0.285; + float2 q = s_rot(p.xy - c, arrow ? 0.2443 : 0.0) / rs; + float star = (s_star5(q, 0.82, 0.55) - 0.18) * rs; + float leaves = min(s_vesica(s_rot(q - float2(-0.33, 1.38), -0.925), 0.4296, 0.2626), + s_vesica(s_rot(q - float2(0.53, 1.37), 0.873), 0.4296, 0.2626)) * rs; + r = s_opu(r, s_piece(leaves, leaves, leaves, p.z, 0.038, zs - 0.02, 0.018, 0.0, 4.0)); + r = s_opu(r, s_piece(star, star, star, p.z, 0.038, zs, 0.022, 0.018 * s_bump(q, float2(0.0), 1.0), 3.0)); + float3 e = float3(abs(q.x) - 0.25, q.y - 0.08, (p.z - zs - 0.056) / rs); + return s_opu(r, float2(s_ellipsoid(e, float3(0.075, 0.13, 0.1)) * rs, 5.0)); +} + +// Pixel Candy : tables générées par scripts/generate-pixel-candy-voxels.mjs (cf. HLSL). +// +constant int PIX_BODY[32] = { 1, 3, 7, 15, 31, 63, 127, 255, 511, 1023, 2047, 127, 247, 243, 480, 192, 48, 120, 120, 120, 504, 4088, 32760, 65534, 65535, 65535, 65534, 32766, 32764, 16380, 16376, 16376 }; +constant int PIX_LINE[32] = { 1, 3, 5, 9, 17, 33, 65, 129, 257, 513, 1985, 73, 149, 147, 288, 192, 48, 72, 72, 72, 456, 3656, 29256, 37454, 32777, 32769, 32770, 16386, 16388, 8196, 8200, 16376 }; +constant int PIX_HI[32] = { 0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 18, 34, 32, 64, 0, 0, 16, 16, 16, 16, 16, 16, 16, 22, 18, 4, 4, 8, 8, 16, 0 }; +constant int PIX_SHADE[32] = { 0, 0, 0, 4, 8, 16, 32, 64, 128, 448, 40, 36, 64, 64, 128, 0, 0, 32, 32, 32, 32, 288, 2336, 18720, 16384, 16384, 16384, 8192, 8192, 4096, 8160, 0 }; +constant int PIX_RECT_N[3] = { 0, 17, 29 }; +constant int4 PIX_GRID[2] = { + int4(11, 16, 0, 0), + int4(16, 16, 16, 0) }; -constant int SCULPT_HANDPIX[15] = { 4, 14, 14, 14, 110, 878, 7022, 7022, 8190, 8191, 8191, 8191, 8190, 4092, 4092 }; -constant int SCULPT_HANDBACK[17] = { - 28, 62, 62, 62, 510, 4094, 32766, 32766, 32766, 32767, 32767, 32767, 32767, 32767, 32766, 16380, 16380 +constant float2 PIX_ORIGIN[2] = { + float2(0.0, 0.0), + float2(-0.3125, 0.0) }; -constant float2 SCULPT_HANDSPAN[17] = { - float2(2.0, 4.0), float2(1.0, 5.0), float2(1.0, 5.0), float2(1.0, 5.0), float2(1.0, 8.0), - float2(1.0, 11.0), float2(1.0, 14.0), float2(1.0, 14.0), float2(1.0, 14.0), float2(0.0, 14.0), - float2(0.0, 14.0), float2(0.0, 14.0), float2(0.0, 14.0), float2(0.0, 14.0), float2(1.0, 14.0), - float2(2.0, 13.0), float2(2.0, 13.0) +constant float4 PIX_BOX[2] = { + float4(0.3438, -0.5, 0.3438, 0.5), + float4(0.1875, -0.5, 0.5, 0.5) }; +constant float4 PIX_RECT[29] = { + float4(0.0313, -0.0313, 0.0313, 0.0313), + float4(0.0625, -0.0938, 0.0625, 0.0313), + float4(0.0938, -0.1563, 0.0938, 0.0313), + float4(0.125, -0.2188, 0.125, 0.0313), + float4(0.1563, -0.2813, 0.1563, 0.0313), + float4(0.1875, -0.3438, 0.1875, 0.0313), + float4(0.2188, -0.4063, 0.2188, 0.0313), + float4(0.25, -0.4688, 0.25, 0.0313), + float4(0.2813, -0.5313, 0.2813, 0.0313), + float4(0.3125, -0.5938, 0.3125, 0.0313), + float4(0.3438, -0.6563, 0.3438, 0.0313), + float4(0.2188, -0.7188, 0.2188, 0.0313), + float4(0.0938, -0.7813, 0.0938, 0.0313), + float4(0.375, -0.8125, 0.125, 0.0625), + float4(0.0625, -0.8438, 0.0625, 0.0313), + float4(0.4375, -0.9063, 0.125, 0.0313), + float4(0.4375, -0.9688, 0.0625, 0.0313), + float4(0.0, -0.0313, 0.0625, 0.0313), + float4(0.0, -0.1563, 0.125, 0.0938), + float4(0.0625, -0.2813, 0.1875, 0.0313), + float4(0.1563, -0.3438, 0.2813, 0.0313), + float4(0.25, -0.4063, 0.375, 0.0313), + float4(0.2188, -0.4688, 0.4688, 0.0313), + float4(0.1875, -0.5625, 0.5, 0.0625), + float4(0.2188, -0.6563, 0.4688, 0.0313), + float4(0.1875, -0.7188, 0.4375, 0.0313), + float4(0.2188, -0.7813, 0.4063, 0.0313), + float4(0.1875, -0.8438, 0.375, 0.0313), + float4(0.2188, -0.9375, 0.3438, 0.0625) +}; +// -inline float2 s_grid_origin(int shape) +inline bool s_pix_bit(int m, int c) { - return shape == 0 ? float2(0.0, 0.0) : float2(-2.5 * SCULPT_VOX, 0.0); + return ((m >> clamp(c, 0, 31)) & 1) == 1; } -inline bool s_bit(int row, int r, int c, int rows, int cols) +static bool s_pix_body(int shape, int c, int r) { - return r >= 0 && r < rows && c >= 0 && c < cols && ((row >> clamp(c, 0, 31)) & 1) == 1; + int4 g = PIX_GRID[shape]; + return c >= 0 && c < g.x && r >= 0 && r < g.y && s_pix_bit(PIX_BODY[g.z + clamp(r, 0, g.y - 1)], c); } -static bool s_occ(float2 id, int shape) +inline int2 s_pix_cell(float2 p, int shape) { - int c = int(id.x); - int r = -int(id.y) - 1; - if (shape == 0) - { - return s_bit(SCULPT_ARROWPIX[clamp(r, 0, 15)], r, c, 16, 16); - } - return s_bit(SCULPT_HANDPIX[clamp(r, 0, 14)], r, c, 15, 13); -} - -static bool s_occ_back(float2 id, int shape) -{ - int c = int(id.x) + 1; - int r = -int(id.y); - if (shape == 0) - { - return s_bit(SCULPT_ARROWBACK[clamp(r, 0, 17)], r, c, 18, 17); - } - return s_bit(SCULPT_HANDBACK[clamp(r, 0, 16)], r, c, 17, 15); + float2 o = PIX_ORIGIN[shape]; + return int2(int(floor((p.x - o.x) / SCULPT_VOX)), int(floor((o.y - p.y) / SCULPT_VOX))); } static float2 s_voxels(float3 p, int shape) { - float2 o = s_grid_origin(shape); - float2 cell = floor((p.xy - o) / SCULPT_VOX); - float2 bc = shape == 0 ? float2(0.33, -0.5) : float2(0.25, -0.47); - float2 bh = shape == 0 ? float2(0.44, 0.61) : float2(0.48, 0.55); - float2 bq = max(abs(p.xy - bc) - bh, 0.0); + float4 b = PIX_BOX[shape]; + float2 bq = max(abs(p.xy - b.xy) - b.zw, 0.0); float bz = max(abs(p.z - (SCULPT_HOVER + 0.07)) - 0.07, 0.0); - float far = max(SCULPT_VOX, sqrt(dot(bq, bq) + bz * bz)); - float dF = far; - float dB = far; - const float3 half_cell = float3(0.5 * SCULPT_VOX, 0.5 * SCULPT_VOX, 0.04); + float d = max(SCULPT_VOX, sqrt(dot(bq, bq) + bz * bz)); + float2 o = PIX_ORIGIN[shape]; + int2 cell = s_pix_cell(p.xy, shape); + const float3 cube = float3(0.5 * SCULPT_VOX, 0.5 * SCULPT_VOX, 0.07); for (int j = -1; j <= 1; j++) { for (int i = -1; i <= 1; i++) { - float2 id = cell + float2(float(i), float(j)); - float3 q = float3(p.xy - (o + (id + 0.5) * SCULPT_VOX), p.z); - if (s_occ(id, shape)) - { - dF = min(dF, s_round_box(q - float3(0.0, 0.0, SCULPT_HOVER + 0.1), half_cell, 0.009)); - } - if (s_occ_back(id, shape)) + int c = cell.x + i; + int r = cell.y + j; + if (s_pix_body(shape, c, r)) { - dB = min(dB, s_round_box(q - float3(0.0, 0.0, SCULPT_HOVER + 0.04), half_cell, 0.009)); + float3 q = float3(p.x - o.x - (float(c) + 0.5) * SCULPT_VOX, p.y - o.y + (float(r) + 0.5) * SCULPT_VOX, + p.z - SCULPT_HOVER - 0.07); + d = min(d, s_round_box(q, cube, 0.006)); } } } - return dF < dB ? float2(dF, 6.0) : float2(dB, 7.0); + return float2(d, 6.0); } -static float s_pixel_hand_dist(float2 p) +static float s_pixel_outline(float2 p, int shape) { float d = 1e9; - for (int r = 0; r < 17; r++) + for (int i = PIX_RECT_N[shape]; i < PIX_RECT_N[shape + 1]; i++) { - float2 x = (SCULPT_HANDSPAN[r] + float2(-3.5, -2.5)) * SCULPT_VOX; - float2 y = float2(-float(r), 1.0 - float(r)) * SCULPT_VOX; - float2 q = max(max(float2(x.x, y.x) - p, p - float2(x.y, y.y)), 0.0); - d = min(d, length(q)); + float2 q = abs(p - PIX_RECT[i].xy) - PIX_RECT[i].zw; + d = min(d, length(max(q, 0.0)) + min(max(q.x, q.y), 0.0)); } return d; } -static float s_gem_edge(float3 p, float2 a, float2 b, float z0) -{ - float2 e = b - a; - float len = length(e); - float2 d = e / len; - float2 q = p.xy - a; - float dist = dot(q, float2(-d.y, d.x)); - float along = abs(dot(q, d) - 0.5 * len); - float z = p.z - z0; - float girdle = (z - 0.035 - 2.2 * dist) * 0.4138; - float crown = (z - 0.07 - 0.6 * dist + 0.18 * along) * 0.8438; - return max(-dist, max(girdle, crown)); -} +// ---- Prism Glow : un cristal en MAILLAGE ---- (cf. HLSL) +// prism mesh: generated by design/cursors/prism-glow/model/export_compositor.py +constant int PRISM_TRI_START[2] = { 0, 31 }; +constant int PRISM_TRI_COUNT[2] = { 31, 130 }; +constant int PRISM_SIL_START[2] = { 0, 25 }; +constant int PRISM_SIL_COUNT[2] = { 25, 42 }; +constant int PRISM_OUT_START[2] = { 67, 76 }; +constant int PRISM_OUT_COUNT[2] = { 9, 34 }; +constant int PRISM_BOX_START[2] = { 0, 4 }; +constant int PRISM_BOX_COUNT[2] = { 4, 18 }; +constant float4 PRISM_TRIS[644] = { + float4(0.252301, 0.266258, -0.042945, 11.0), + float4(-0.221166, -0.203988, 0.0, 0.01033), + float4(-0.221166, -0.203988, 0.042945, 0.045182), + float4(0.677984, -0.735077, 0.0, 0.577584), + float4(0.252301, 0.266258, -0.042945, 3.0), + float4(-0.221166, -0.203988, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(0.677984, -0.735077, 0.0, 0.577584), + float4(0.031135, 0.06227, -0.042945, 7.0), + float4(0.004294, 0.419785, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(-0.999948, 0.01023, 0.0, 0.577584), + float4(0.169632, 0.356442, 0.115951, 14.0), + float4(0.082669, -0.090184, -0.115951, 0.590616), + float4(-0.138497, -0.294172, -0.115951, 0.737911), + float4(0.466385, -0.505659, 0.725806, 0.973449), + float4(0.035429, 0.482055, 0.0, 14.0), + float4(0.134202, -0.125614, 0.115951, 0.000305), + float4(-0.004294, -0.419785, 0.0, 0.938689), + float4(-0.650191, 0.006652, 0.759742, 0.854994), + float4(0.031135, 0.06227, -0.042945, 9.0), + float4(0.004294, 0.419785, 0.0, 0.01033), + float4(0.004294, 0.419785, 0.042945, 0.045182), + float4(-0.999948, 0.01023, 0.0, 0.577584), + float4(0.169632, 0.356442, 0.115951, 14.0), + float4(0.159969, 0.080521, -0.045092, 0.381323), + float4(0.082669, -0.090184, -0.115951, 0.056123), + float4(0.461407, -0.510214, 0.725799, 0.973449), + float4(0.541104, 0.544325, -0.042945, 9.0), + float4(-0.288804, -0.278068, 0.0, 0.01033), + float4(-0.288804, -0.278068, 0.042945, 0.045182), + float4(0.69359, -0.72037, 0.0, 0.577584), + float4(0.329601, 0.436963, 0.070859, 14.0), + float4(0.211503, 0.107362, -0.070859, 0.545724), + float4(-0.077301, -0.170706, -0.070859, 0.064805), + float4(0.49567, -0.514808, 0.699488, 0.973449), + float4(0.268405, 0.533589, 0.081595, 14.0), + float4(0.061196, -0.096626, -0.010736, 0.000916), + float4(-0.098773, -0.177147, 0.034356, 0.046662), + float4(0.247834, 0.049458, 0.967539, 0.904662), + float4(0.541104, 0.544325, -0.042945, 7.0), + float4(-0.288804, -0.278068, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(0.69359, -0.72037, 0.0, 0.577584), + float4(0.199693, 0.595859, 0.0, 14.0), + float4(0.068712, -0.06227, 0.081595, 0.0), + float4(-0.030061, -0.239417, 0.115951, 0.01445), + float4(-0.504461, 0.426837, 0.750553, 0.254154), + float4(0.268405, 0.533589, 0.081595, 14.0), + float4(0.272699, 0.010736, -0.081595, 0.0), + float4(0.061196, -0.096626, -0.010736, 0.025193), + float4(0.283244, 0.073137, 0.956255, 0.527108), + float4(0.035429, 0.482055, 0.0, 14.0), + float4(0.0, 0.258742, 0.0, 0.0), + float4(0.134202, -0.125614, 0.115951, 0.473533), + float4(-0.653777, 0.0, 0.756687, 0.964691), + float4(0.359663, 0.559356, 0.0, 14.0), + float4(0.181442, -0.015031, 0.0, 0.006043), + float4(-0.091258, -0.025767, 0.081595, 0.076188), + float4(0.076465, 0.923048, 0.37701, 0.964691), + float4(0.035429, 0.482055, -0.042945, 3.0), + float4(0.0, 0.258742, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(-1.0, 0.0, 0.0, 0.577584), + float4(0.035429, 0.482055, -0.042945, 11.0), + float4(0.0, 0.258742, 0.0, 0.01033), + float4(0.0, 0.258742, 0.042945, 0.045182), + float4(-1.0, 0.0, 0.0, 0.577584), + float4(0.035429, 0.740798, -0.042945, 7.0), + float4(0.164264, -0.144939, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(0.661622, 0.749838, 0.0, 0.577584), + float4(0.035429, 0.740798, 0.0, 14.0), + float4(0.164264, -0.144939, 0.0, 0.0), + float4(0.134202, -0.384356, 0.115951, 0.010956), + float4(0.332572, 0.376915, 0.864483, 0.215854), + float4(0.035429, 0.740798, -0.042945, 11.0), + float4(0.164264, -0.144939, 0.0, 0.01033), + float4(0.164264, -0.144939, 0.042945, 0.045182), + float4(0.661622, 0.749838, 0.0, 0.577584), + float4(0.199693, 0.595859, -0.042945, 7.0), + float4(0.105215, 0.22546, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(-0.906183, 0.422886, 0.0, 0.577584), + float4(0.199693, 0.595859, 0.0, 14.0), + float4(0.105215, 0.22546, 0.0, 0.0), + float4(0.068712, -0.06227, 0.081595, 0.015991), + float4(-0.613882, 0.286478, 0.735581, 0.473533), + float4(0.199693, 0.595859, -0.042945, 11.0), + float4(0.105215, 0.22546, 0.0, 0.01033), + float4(0.105215, 0.22546, 0.042945, 0.045182), + float4(-0.906183, 0.422886, 0.0, 0.577584), + float4(0.304908, 0.821319, 0.0, 14.0), + float4(0.128834, -0.067638, 0.0, 0.0), + float4(-0.036503, -0.28773, 0.081595, 0.254154), + float4(0.133686, 0.254641, 0.957751, 0.973449), + float4(0.304908, 0.821319, -0.042945, 7.0), + float4(0.128834, -0.067638, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(0.464834, 0.885398, 0.0, 0.577584), + float4(0.433742, 0.753681, 0.0, 14.0), + float4(-0.07408, -0.194325, 0.0, 0.0), + float4(-0.165337, -0.220092, 0.081595, 0.964691), + float4(0.683362, -0.260508, 0.682021, 0.964691), + float4(0.433742, 0.753681, -0.042945, 11.0), + float4(-0.07408, -0.194325, 0.0, 0.01033), + float4(-0.07408, -0.194325, 0.042945, 0.045182), + float4(0.934406, -0.35621, 0.0, 0.577584), + float4(0.304908, 0.821319, -0.042945, 11.0), + float4(0.128834, -0.067638, 0.0, 0.01033), + float4(0.128834, -0.067638, 0.042945, 0.045182), + float4(0.464834, 0.885398, 0.0, 0.577584), + float4(0.433742, 0.753681, -0.042945, 7.0), + float4(-0.07408, -0.194325, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(0.934406, -0.35621, 0.0, 0.577584), + float4(0.359663, 0.559356, -0.042945, 7.0), + float4(0.181442, -0.015031, 0.042945, 0.01033), + float4(0.0, 0.0, 0.042945, 0.045182), + float4(0.082557, 0.996586, 0.0, 0.577584), + float4(0.359663, 0.559356, -0.042945, 11.0), + float4(0.181442, -0.015031, 0.0, 0.01033), + float4(0.181442, -0.015031, 0.042945, 0.045182), + float4(0.082557, 0.996586, 0.0, 0.577584), + float4(0.044345, 0.043263, -0.043263, 11.0), + float4(-0.043263, -0.005408, 0.0, 0.01033), + float4(-0.043263, -0.005408, 0.043263, 0.045182), + float4(0.124035, -0.992278, 0.0, 0.577584), + float4(0.044345, 0.043263, -0.043263, 7.0), + float4(-0.043263, -0.005408, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.124035, -0.992278, 0.0, 0.577584), + float4(0.001082, 0.037855, -0.043263, 7.0), + float4(-0.044345, 0.040019, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.669964, -0.742393, 0.0, 0.577584), + float4(0.044345, 0.043263, 0.0, 10.0), + float4(-0.043263, -0.005408, 0.0, 0.000305), + float4(-0.040019, 0.030284, 0.049753, 0.964691), + float4(0.101434, -0.811469, 0.575525, 0.973449), + float4(0.004326, 0.073548, 0.049753, 14.0), + float4(-0.003245, -0.035692, -0.049753, 0.0), + float4(-0.04759, 0.004326, -0.049753, 0.955978), + float4(-0.580475, -0.643229, 0.499304, 0.896269), + float4(0.001082, 0.037855, -0.043263, 11.0), + float4(-0.044345, 0.040019, 0.0, 0.01033), + float4(-0.044345, 0.040019, 0.043263, 0.045182), + float4(-0.669964, -0.742393, 0.0, 0.577584), + float4(-0.031366, 0.229295, 0.058405, 14.0), + float4(0.035692, -0.155748, -0.008653, 0.000305), + float4(-0.011897, -0.151422, -0.058405, 0.955978), + float4(-0.716533, -0.20131, 0.667874, 0.904662), + float4(-0.043263, 0.077874, -0.043263, 7.0), + float4(-0.001082, 0.154666, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.999976, -0.006993, 0.0, 0.577584), + float4(-0.031366, 0.229295, 0.058405, 14.0), + float4(0.089771, -0.137361, -0.02055, 0.445197), + float4(0.035692, -0.155748, -0.008653, 0.973449), + float4(0.216369, -0.004654, 0.976301, 0.973449), + float4(-0.031366, 0.229295, 0.058405, 14.0), + float4(-0.011897, -0.151422, -0.058405, 0.000305), + float4(-0.012979, 0.003245, -0.058405, 0.955978), + float4(-0.976245, -0.006827, 0.216564, 0.904662), + float4(-0.043263, 0.077874, -0.043263, 9.0), + float4(-0.001082, 0.154666, 0.0, 0.01033), + float4(-0.001082, 0.154666, 0.043263, 0.045182), + float4(-0.999976, -0.006993, 0.0, 0.577584), + float4(-0.045426, 0.564586, 0.052998, 14.0), + float4(0.014061, -0.335291, 0.005408, 0.520989), + float4(0.001082, -0.332046, -0.052998, 0.088655), + float4(-0.975941, -0.037462, 0.214794, 0.973449), + float4(-0.044345, 0.23254, 0.0, 3.0), + float4(0.0, 0.0, -0.043263, 0.01033), + float4(-0.001082, 0.332046, -0.043263, 0.045182), + float4(-0.999995, -0.003257, 0.0, 0.577584), + float4(-0.163319, 0.380717, -0.043263, 7.0), + float4(-0.060569, 0.02704, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.407651, -0.913138, 0.0, 0.577584), + float4(-0.094098, 0.431551, -0.043263, 11.0), + float4(-0.069221, -0.050834, 0.0, 0.01033), + float4(-0.069221, -0.050834, 0.043263, 0.045182), + float4(0.591909, -0.806004, 0.0, 0.577584), + float4(-0.163319, 0.380717, -0.043263, 11.0), + float4(-0.060569, 0.02704, 0.0, 0.01033), + float4(-0.060569, 0.02704, 0.043263, 0.045182), + float4(-0.407651, -0.913138, 0.0, 0.577584), + float4(0.075711, 0.08869, -0.043263, 11.0), + float4(-0.031366, -0.045426, 0.0, 0.01033), + float4(-0.031366, -0.045426, 0.043263, 0.045182), + float4(0.822897, -0.568191, 0.0, 0.577584), + float4(0.044345, 0.043263, 0.0, 6.0), + float4(-0.040019, 0.030284, 0.049753, 0.000305), + float4(0.014061, 0.048671, 0.037855, 0.964691), + float4(0.365607, -0.634949, 0.680566, 0.973449), + float4(0.075711, 0.08869, -0.043263, 7.0), + float4(-0.031366, -0.045426, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.822897, -0.568191, 0.0, 0.577584), + float4(0.058405, 0.091934, 0.037855, 14.0), + float4(0.017305, -0.003245, -0.037855, 0.0), + float4(-0.014061, -0.048671, -0.037855, 0.473533), + float4(0.757369, -0.522945, 0.39105, 0.904662), + float4(-0.031366, 0.229295, 0.058405, 14.0), + float4(0.107077, -0.140606, -0.058405, 0.0), + float4(0.089771, -0.137361, -0.02055, 0.376257), + float4(0.812052, 0.481346, 0.329966, 0.830777), + float4(0.077874, 0.407756, -0.043263, 11.0), + float4(-0.002163, -0.319067, 0.0, 0.01033), + float4(-0.002163, -0.319067, 0.043263, 0.045182), + float4(0.999977, -0.006779, 0.0, 0.577584), + float4(-0.031366, 0.229295, 0.058405, 14.0), + float4(0.10924, 0.178461, -0.058405, 0.0), + float4(0.107077, -0.140606, -0.058405, 0.376257), + float4(0.475583, -0.003224, 0.879665, 0.830777), + float4(0.147095, 0.272559, -0.043263, 7.0), + float4(-0.035692, 0.034611, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.696146, -0.7179, 0.0, 0.577584), + float4(0.147095, 0.272559, -0.043263, 11.0), + float4(-0.035692, 0.034611, 0.0, 0.01033), + float4(-0.035692, 0.034611, 0.043263, 0.045182), + float4(-0.696146, -0.7179, 0.0, 0.577584), + float4(0.144932, 0.307169, 0.049753, 14.0), + float4(0.002163, -0.034611, -0.049753, 0.0), + float4(-0.033529, 0.0, -0.049753, 0.9131), + float4(-0.630237, -0.649931, 0.424725, 0.904662), + float4(0.183869, 0.317985, 0.037855, 6.0), + float4(-0.036774, -0.045426, -0.037855, 0.0), + float4(-0.038937, -0.010816, 0.011897, 0.806958), + float4(0.374433, -0.753477, 0.540438, 0.854994), + float4(0.077874, 0.407756, -0.043263, 7.0), + float4(-0.002163, -0.319067, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.999977, -0.006779, 0.0, 0.577584), + float4(0.111403, 0.307169, -0.043263, 7.0), + float4(-0.002163, 0.131953, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.999866, -0.016391, 0.0, 0.577584), + float4(0.10924, 0.439122, 0.0, 6.0), + float4(0.035692, -0.131953, 0.049753, 0.18117), + float4(0.002163, -0.131953, 0.0, 0.955978), + float4(-0.82919, -0.013593, 0.558802, 0.964691), + float4(0.128708, 0.463999, 0.063813, 14.0), + float4(0.055161, -0.146014, -0.025958, 0.246204), + float4(0.016224, -0.156829, -0.014061, 0.964691), + float4(0.305397, -0.053644, 0.950713, 0.973449), + float4(0.111403, 0.307169, -0.043263, 11.0), + float4(-0.002163, 0.131953, 0.0, 0.01033), + float4(-0.002163, 0.131953, 0.043263, 0.045182), + float4(-0.999866, -0.016391, 0.0, 0.577584), + float4(0.189277, 0.28013, -0.043263, 11.0), + float4(-0.042182, -0.007571, 0.0, 0.01033), + float4(-0.042182, -0.007571, 0.043263, 0.045182), + float4(0.176664, -0.984271, 0.0, 0.577584), + float4(0.183869, 0.317985, 0.037855, 10.0), + float4(0.005408, -0.037855, -0.037855, 0.0), + float4(-0.036774, -0.045426, -0.037855, 0.806958), + float4(0.124328, -0.692684, 0.710445, 0.854994), + float4(0.189277, 0.28013, -0.043263, 7.0), + float4(-0.042182, -0.007571, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.176664, -0.984271, 0.0, 0.577584), + float4(0.183869, 0.317985, 0.037855, 14.0), + float4(0.02704, -0.004326, -0.037855, 0.0), + float4(0.005408, -0.037855, -0.037855, 0.514916), + float4(0.70062, -0.452013, 0.552101, 0.846876), + float4(0.210908, 0.313659, -0.043263, 11.0), + float4(-0.021632, -0.033529, 0.0, 0.01033), + float4(-0.021632, -0.033529, 0.043263, 0.045182), + float4(0.840297, -0.542127, 0.0, 0.577584), + float4(0.210908, 0.445612, 0.0, 10.0), + float4(0.0, -0.131953, 0.0, 0.0), + float4(-0.02704, -0.127627, 0.037855, 0.439658), + float4(0.813734, 0.0, 0.581238, 0.871366), + float4(0.210908, 0.313659, -0.043263, 7.0), + float4(-0.021632, -0.033529, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.840297, -0.542127, 0.0, 0.577584), + float4(0.210908, 0.445612, -0.043263, 11.0), + float4(0.0, -0.131953, 0.0, 0.01033), + float4(0.0, -0.131953, 0.043263, 0.045182), + float4(1.0, 0.0, 0.0, 0.577584), + float4(0.241193, 0.343943, -0.043263, 7.0), + float4(-0.002163, 0.107077, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.999796, -0.020198, 0.0, 0.577584), + float4(0.276885, 0.311496, -0.043263, 11.0), + float4(-0.035692, 0.032447, 0.0, 0.01033), + float4(-0.035692, 0.032447, 0.043263, 0.045182), + float4(-0.672673, -0.73994, 0.0, 0.577584), + float4(0.23903, 0.45102, 0.0, 14.0), + float4(0.042182, -0.108158, 0.046508, 0.000305), + float4(0.002163, -0.107077, 0.0, 0.846876), + float4(-0.758098, -0.015315, 0.651961, 0.896269), + float4(0.276885, 0.311496, -0.043263, 7.0), + float4(-0.035692, 0.032447, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.672673, -0.73994, 0.0, 0.577584), + float4(0.281211, 0.342862, 0.046508, 14.0), + float4(-0.004326, -0.031366, -0.046508, 0.0), + float4(-0.040019, 0.001082, -0.046508, 0.752941), + float4(-0.58651, -0.645161, 0.489667, 0.846876), + float4(0.246601, 0.459672, 0.06165, 14.0), + float4(0.070303, -0.101669, -0.024876, 0.262257), + float4(0.034611, -0.116811, -0.015142, 0.973449), + float4(0.27928, -0.041605, 0.959308, 0.973449), + float4(0.319067, 0.319067, -0.043263, 11.0), + float4(-0.042182, -0.007571, 0.0, 0.01033), + float4(-0.042182, -0.007571, 0.043263, 0.045182), + float4(0.176664, -0.984271, 0.0, 0.577584), + float4(0.319067, 0.319067, 0.0, 10.0), + float4(-0.042182, -0.007571, 0.0, 0.0), + float4(-0.037855, 0.023795, 0.046508, 0.896269), + float4(0.1483, -0.826245, 0.54344, 0.896269), + float4(0.319067, 0.319067, -0.043263, 7.0), + float4(-0.042182, -0.007571, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.176664, -0.984271, 0.0, 0.577584), + float4(0.319067, 0.319067, 0.0, 6.0), + float4(-0.037855, 0.023795, 0.046508, 0.0), + float4(-0.002163, 0.038937, 0.036774, 0.896269), + float4(0.437906, -0.604311, 0.665618, 0.896269), + float4(0.316904, 0.358004, 0.036774, 14.0), + float4(0.024876, -0.004326, -0.036774, 0.0), + float4(0.002163, -0.038937, -0.036774, 0.617212), + float4(0.707338, -0.464191, 0.533104, 0.921584), + float4(0.34178, 0.353677, -0.043263, 11.0), + float4(-0.022713, -0.034611, 0.0, 0.01033), + float4(-0.022713, -0.034611, 0.043263, 0.045182), + float4(0.836048, -0.548657, 0.0, 0.577584), + float4(0.340698, 0.474815, 0.0, 14.0), + float4(0.001082, -0.121137, 0.0, 0.0), + float4(-0.023795, -0.116811, 0.036774, 0.473533), + float4(0.828664, 0.007399, 0.559697, 0.930114), + float4(0.34178, 0.353677, -0.043263, 7.0), + float4(-0.022713, -0.034611, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.836048, -0.548657, 0.0, 0.577584), + float4(0.340698, 0.474815, -0.043263, 11.0), + float4(0.001082, -0.121137, 0.0, 0.01033), + float4(0.001082, -0.121137, 0.043263, 0.045182), + float4(0.99996, 0.008928, 0.0, 0.577584), + float4(0.411001, 0.351514, -0.043263, 11.0), + float4(-0.034611, 0.030284, 0.0, 0.01033), + float4(-0.034611, 0.030284, 0.043263, 0.045182), + float4(-0.658505, -0.752577, 0.0, 0.577584), + float4(0.411001, 0.351514, -0.043263, 7.0), + float4(-0.034611, 0.030284, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.658505, -0.752577, 0.0, 0.577584), + float4(0.412083, 0.38288, 0.043263, 14.0), + float4(-0.001082, -0.031366, -0.043263, 0.0), + float4(-0.035692, -0.001082, -0.043263, 0.745403), + float4(-0.574039, -0.656045, 0.489984, 0.854994), + float4(0.45102, 0.359085, -0.043263, 11.0), + float4(-0.040019, -0.007571, 0.0, 0.01033), + float4(-0.040019, -0.007571, 0.043263, 0.045182), + float4(0.185892, -0.98257, 0.0, 0.577584), + float4(0.45102, 0.359085, 0.0, 10.0), + float4(-0.040019, -0.007571, 0.0, 0.000305), + float4(-0.038937, 0.023795, 0.043263, 0.879622), + float4(0.151736, -0.802035, 0.577682, 0.9131), + float4(0.45102, 0.359085, -0.043263, 7.0), + float4(-0.040019, -0.007571, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.185892, -0.98257, 0.0, 0.577584), + float4(0.45102, 0.359085, 0.0, 6.0), + float4(-0.038937, 0.023795, 0.043263, 0.000305), + float4(0.003245, 0.0411, 0.032447, 0.879622), + float4(0.417852, -0.583049, 0.696744, 0.9131), + float4(0.473733, 0.391533, -0.043263, 11.0), + float4(-0.022713, -0.032447, 0.0, 0.01033), + float4(-0.022713, -0.032447, 0.043263, 0.045182), + float4(0.819232, -0.573462, 0.0, 0.577584), + float4(0.454264, 0.400185, 0.032447, 14.0), + float4(0.019468, -0.008653, -0.032447, 0.0), + float4(-0.003245, -0.0411, -0.032447, 0.527108), + float4(0.688617, -0.482032, 0.541712, 0.887922), + float4(0.473733, 0.391533, -0.043263, 7.0), + float4(-0.022713, -0.032447, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.819232, -0.573462, 0.0, 0.577584), + float4(-0.223888, 0.407756, -0.043263, 7.0), + float4(0.002163, 0.06814, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.999496, 0.03173, 0.0, 0.577584), + float4(-0.223888, 0.407756, -0.043263, 11.0), + float4(0.002163, 0.06814, 0.0, 0.01033), + float4(0.002163, 0.06814, 0.043263, 0.045182), + float4(-0.999496, 0.03173, 0.0, 0.577584), + float4(-0.167645, 0.416409, 0.046508, 14.0), + float4(-0.056242, -0.008653, -0.046508, 0.000305), + float4(-0.054079, 0.059487, -0.046508, 0.955978), + float4(-0.638985, 0.020285, 0.768952, 0.973449), + float4(-0.167645, 0.416409, 0.046508, 14.0), + float4(0.004326, -0.035692, -0.046508, 0.000305), + float4(-0.056242, -0.008653, -0.046508, 0.955978), + float4(-0.339782, -0.761112, 0.552501, 0.973449), + float4(-0.221724, 0.475896, 0.0, 14.0), + float4(0.139524, 0.055161, 0.043263, 0.520989), + float4(0.054079, -0.059487, 0.046508, 0.973449), + float4(-0.39307, 0.317373, 0.863001, 0.973449), + float4(-0.167645, 0.416409, 0.046508, 6.0), + float4(0.073548, 0.015142, -0.046508, 0.0), + float4(0.004326, -0.035692, -0.046508, 0.768154), + float4(0.490917, -0.668482, 0.558688, 0.871366), + float4(-0.094098, 0.431551, -0.043263, 7.0), + float4(-0.069221, -0.050834, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.591909, -0.806004, 0.0, 0.577584), + float4(-0.167645, 0.416409, 0.046508, 10.0), + float4(0.085445, 0.114648, -0.003245, 0.0), + float4(0.073548, 0.015142, -0.046508, 0.768154), + float4(0.548356, -0.38771, 0.740936, 0.871366), + float4(-0.0822, 0.531057, -0.043263, 11.0), + float4(-0.011897, -0.099506, 0.0, 0.01033), + float4(-0.011897, -0.099506, 0.043263, 0.045182), + float4(0.992928, -0.11872, 0.0, 0.577584), + float4(-0.044345, 0.23254, 0.0, 7.0), + float4(-0.001082, 0.332046, -0.043263, 0.01033), + float4(-0.001082, 0.332046, 0.052998, 0.045182), + float4(-0.999995, -0.003257, 0.0, 0.577584), + float4(-0.045426, 0.564586, 0.052998, 14.0), + float4(0.1233, -0.156829, -0.052998, 0.520989), + float4(0.014061, -0.335291, 0.005408, 0.088655), + float4(0.429355, 0.032562, 0.902549, 0.973449), + float4(0.10924, 0.439122, 0.0, 14.0), + float4(-0.031366, -0.031366, 0.0, 0.132876), + float4(-0.154666, 0.125464, 0.052998, 0.028428), + float4(0.182761, -0.182761, 0.966021, 0.955978), + float4(0.10924, 0.439122, -0.043263, 11.0), + float4(-0.031366, -0.031366, 0.0, 0.01033), + float4(-0.031366, -0.031366, 0.043263, 0.045182), + float4(0.707107, -0.707107, 0.0, 0.577584), + float4(0.10924, 0.439122, -0.043263, 7.0), + float4(-0.031366, -0.031366, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.707107, -0.707107, 0.0, 0.577584), + float4(0.10924, 0.439122, 0.0, 10.0), + float4(0.019468, 0.024876, 0.063813, 0.18117), + float4(0.035692, -0.131953, 0.049753, 0.955978), + float4(-0.933936, -0.126584, 0.334276, 0.964691), + float4(0.128708, 0.463999, 0.063813, 14.0), + float4(0.050834, 0.002163, -0.005408, 0.0), + float4(0.0822, -0.018387, -0.063813, 0.439658), + float4(0.07859, -0.92643, 0.368172, 0.871366), + float4(-0.221724, 0.475896, -0.043263, 7.0), + float4(0.122219, 0.207664, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.861819, 0.507216, 0.0, 0.577584), + float4(-0.221724, 0.475896, -0.043263, 9.0), + float4(0.122219, 0.207664, 0.0, 0.01033), + float4(0.122219, 0.207664, 0.043263, 0.045182), + float4(-0.861819, 0.507216, 0.0, 0.577584), + float4(-0.099506, 0.68356, 0.0, 14.0), + float4(0.017305, -0.152503, 0.043263, 0.0), + float4(-0.122219, -0.207664, 0.0, 0.723049), + float4(-0.365879, 0.215335, 0.905408, 0.854994), + float4(-0.0822, 0.531057, -0.043263, 7.0), + float4(-0.011897, -0.099506, 0.043263, 0.01033), + float4(0.0, 0.0, 0.086527, 0.045182), + float4(0.992928, -0.11872, 0.0, 0.577584), + float4(-0.099506, 0.68356, 0.0, 14.0), + float4(0.054079, -0.118974, 0.052998, 0.0), + float4(0.017305, -0.152503, 0.043263, 0.514916), + float4(-0.41958, 0.203352, 0.884647, 0.863157), + float4(-0.099506, 0.68356, -0.043263, 3.0), + float4(0.086527, 0.149258, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.86514, 0.501531, 0.0, 0.577584), + float4(-0.045426, 0.564586, -0.043263, 11.0), + float4(-0.036774, -0.033529, 0.0, 0.01033), + float4(-0.036774, -0.033529, 0.086527, 0.045182), + float4(0.673754, -0.738956, 0.0, 0.577584), + float4(-0.045426, 0.564586, -0.043263, 7.0), + float4(-0.036774, -0.033529, 0.086527, 0.01033), + float4(0.0, 0.0, 0.096261, 0.045182), + float4(0.673754, -0.738956, 0.0, 0.577584), + float4(-0.099506, 0.68356, 0.0, 14.0), + float4(0.086527, 0.149258, 0.0, 0.0), + float4(0.054079, -0.118974, 0.052998, 0.262257), + float4(-0.385563, 0.223515, 0.8952, 0.9131), + float4(-0.099506, 0.68356, -0.043263, 11.0), + float4(0.086527, 0.149258, 0.0, 0.01033), + float4(0.086527, 0.149258, 0.043263, 0.045182), + float4(-0.86514, 0.501531, 0.0, 0.577584), + float4(-0.012979, 0.832818, 0.0, 14.0), + float4(0.214153, -0.194685, 0.0822, 0.000305), + float4(-0.032447, -0.268232, 0.052998, 0.930114), + float4(-0.176863, 0.211325, 0.961281, 0.973449), + float4(0.128708, 0.463999, 0.063813, 14.0), + float4(-0.019468, -0.024876, -0.063813, 0.168276), + float4(-0.174135, 0.100587, -0.010816, 0.038209), + float4(-0.469238, -0.764881, 0.441331, 0.955978), + float4(0.201174, 0.638133, 0.0822, 14.0), + float4(-0.072466, -0.174135, -0.018387, 0.18117), + float4(-0.246601, -0.073548, -0.029203, 0.082277), + float4(-0.098561, -0.063844, 0.993081, 0.973449), + float4(-0.012979, 0.832818, -0.043263, 7.0), + float4(0.339617, -0.002163, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.006369, 0.99998, 0.0, 0.577584), + float4(0.179543, 0.466162, 0.058405, 10.0), + float4(-0.050834, -0.002163, 0.005408, 0.000305), + float4(0.021632, 0.171972, 0.023795, 0.022171), + float4(0.110896, -0.149888, 0.982464, 0.381323), + float4(-0.012979, 0.832818, 0.0, 14.0), + float4(0.339617, -0.002163, 0.0, 0.0), + float4(0.214153, -0.194685, 0.0822, 0.027314), + float4(0.002492, 0.391296, 0.920261, 0.49102), + float4(0.183869, 0.317985, 0.037855, 10.0), + float4(-0.055161, 0.146014, 0.025958, 0.0), + float4(0.02704, 0.127627, -0.037855, 0.439658), + float4(0.623841, 0.097823, 0.775405, 0.871366), + float4(0.210908, 0.445612, -0.043263, 7.0), + float4(0.0, -0.131953, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(1.0, 0.0, 0.0, 0.577584), + float4(0.210908, 0.445612, 0.0, 10.0), + float4(-0.031366, 0.02055, 0.058405, 0.000305), + float4(0.035692, 0.014061, 0.06165, 0.022171), + float4(-0.105864, -0.954447, 0.278971, 0.381323), + float4(0.23903, 0.45102, -0.043263, 11.0), + float4(-0.028121, -0.005408, 0.0, 0.01033), + float4(-0.028121, -0.005408, 0.043263, 0.045182), + float4(0.188847, -0.982006, 0.0, 0.577584), + float4(0.23903, 0.45102, -0.043263, 7.0), + float4(-0.028121, -0.005408, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.188847, -0.982006, 0.0, 0.577584), + float4(0.246601, 0.459672, 0.06165, 11.0), + float4(-0.007571, -0.008653, -0.06165, 0.000305), + float4(-0.035692, -0.014061, -0.06165, 0.022171), + float4(0.187618, -0.975617, 0.113888, 0.381323), + float4(0.241193, 0.343943, -0.043263, 11.0), + float4(-0.002163, 0.107077, 0.0, 0.01033), + float4(-0.002163, 0.107077, 0.043263, 0.045182), + float4(-0.999796, -0.020198, 0.0, 0.577584), + float4(0.23903, 0.45102, 0.0, 15.0), + float4(0.007571, 0.008653, 0.06165, 0.000305), + float4(0.042182, -0.108158, 0.046508, 0.846876), + float4(-0.94107, -0.299262, 0.157572, 0.896269), + float4(0.316904, 0.358004, 0.036774, 14.0), + float4(-0.070303, 0.101669, 0.024876, 0.0), + float4(0.023795, 0.116811, -0.036774, 0.473533), + float4(0.523424, 0.157027, 0.837478, 0.930114), + float4(0.340698, 0.474815, -0.043263, 7.0), + float4(0.001082, -0.121137, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.99996, 0.008928, 0.0, 0.577584), + float4(0.376391, 0.381799, -0.043263, 11.0), + float4(-0.001082, 0.107077, 0.0, 0.01033), + float4(-0.001082, 0.107077, 0.043263, 0.045182), + float4(-0.999949, -0.0101, 0.0, 0.577584), + float4(0.376391, 0.381799, -0.043263, 7.0), + float4(-0.001082, 0.107077, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(-0.999949, -0.0101, 0.0, 0.577584), + float4(0.375309, 0.488875, 0.0, 14.0), + float4(0.036774, -0.105995, 0.043263, 0.000305), + float4(0.001082, -0.107077, 0.0, 0.879622), + float4(-0.771254, -0.00779, 0.63648, 0.896269), + float4(0.375309, 0.488875, 0.0, 14.0), + float4(0.078955, -0.08869, 0.032447, 0.223224), + float4(0.036774, -0.105995, 0.043263, 0.964691), + float4(0.071225, 0.39802, 0.914608, 0.973449), + float4(0.473733, 0.603523, 0.0, 14.0), + float4(0.0, -0.21199, 0.0, 0.158953), + float4(-0.019468, -0.203337, 0.032447, 0.030716), + float4(0.857493, 0.0, 0.514496, 0.964691), + float4(0.473733, 0.603523, -0.043263, 11.0), + float4(0.0, -0.21199, 0.0, 0.01033), + float4(0.0, -0.21199, 0.043263, 0.045182), + float4(1.0, 0.0, 0.0, 0.577584), + float4(0.276885, 0.484549, 0.058405, 10.0), + float4(-0.030284, -0.024876, 0.003245, 0.000305), + float4(-0.097342, -0.018387, 0.0, 0.022171), + float4(-0.03153, 0.166923, 0.985466, 0.381323), + float4(0.246601, 0.459672, 0.06165, 14.0), + float4(0.030284, 0.024876, -0.003245, 0.0), + float4(0.094098, 0.015142, -0.06165, 0.473533), + float4(0.518869, -0.545855, 0.657888, 0.930114), + float4(0.375309, 0.488875, -0.043263, 11.0), + float4(-0.034611, -0.014061, 0.0, 0.01033), + float4(-0.034611, -0.014061, 0.043263, 0.045182), + float4(0.376377, -0.926467, 0.0, 0.577584), + float4(0.340698, 0.474815, 0.0, 14.0), + float4(-0.063813, 0.009734, 0.058405, 0.000305), + float4(0.034611, 0.014061, 0.0, 0.022171), + float4(0.3276, -0.806402, 0.492334, 0.381323), + float4(0.375309, 0.488875, -0.043263, 7.0), + float4(-0.034611, -0.014061, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.376377, -0.926467, 0.0, 0.577584), + float4(0.454264, 0.400185, 0.032447, 12.0), + float4(-0.078955, 0.08869, -0.032447, 0.158953), + float4(0.019468, 0.203337, -0.032447, 0.030716), + float4(-0.201684, 0.173144, 0.964025, 0.964691), + float4(0.201174, 0.638133, 0.0822, 8.0), + float4(0.075711, -0.153585, -0.023795, 0.000305), + float4(-0.021632, -0.171972, -0.023795, 0.022171), + float4(0.026497, -0.140281, 0.989757, 0.381323), + float4(0.473733, 0.603523, -0.043263, 7.0), + float4(0.0, -0.21199, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(1.0, 0.0, 0.0, 0.577584), + float4(0.276885, 0.484549, 0.058405, 14.0), + float4(-0.075711, 0.153585, 0.023795, 0.000305), + float4(0.098424, 0.004326, -0.058405, 0.022171), + float4(0.50316, 0.115345, 0.856461, 0.381323), + float4(0.375309, 0.488875, 0.0, 10.0), + float4(0.090853, 0.12979, 0.0, 0.158953), + float4(0.098424, 0.114648, 0.0, 0.030716), + float4(0.0, 0.0, 1.0, 0.964691), + float4(0.466162, 0.618665, -0.043263, 11.0), + float4(0.007571, -0.015142, 0.0, 0.01033), + float4(0.007571, -0.015142, 0.043263, 0.045182), + float4(0.894427, 0.447214, 0.0, 0.577584), + float4(0.466162, 0.618665, -0.043263, 3.0), + float4(0.007571, -0.015142, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.894427, 0.447214, 0.0, 0.577584), + float4(0.326638, 0.830655, 0.0, 14.0), + float4(0.139524, -0.21199, 0.0, 0.0), + float4(0.048671, -0.34178, 0.0, 0.015213), + float4(0.0, 0.0, 1.0, 0.428687), + float4(0.326638, 0.830655, 0.0, 14.0), + float4(0.048671, -0.34178, 0.0, 0.007492), + float4(-0.125464, -0.192522, 0.0822, 0.054475), + float4(0.472493, 0.067285, 0.878762, 0.964691), + float4(0.326638, 0.830655, -0.043263, 9.0), + float4(0.139524, -0.21199, 0.0, 0.01033), + float4(0.139524, -0.21199, 0.043263, 0.045182), + float4(0.835314, 0.549773, 0.0, 0.577584), + float4(0.326638, 0.830655, -0.043263, 7.0), + float4(0.139524, -0.21199, 0.043263, 0.01033), + float4(0.0, 0.0, 0.043263, 0.045182), + float4(0.835314, 0.549773, 0.0, 0.577584), + float4(-0.012979, 0.832818, -0.043263, 11.0), + float4(0.339617, -0.002163, 0.0, 0.01033), + float4(0.339617, -0.002163, 0.043263, 0.045182), + float4(0.006369, 0.99998, 0.0, 0.577584) +}; +constant float4 PRISM_BOXES[44] = { + float4(0.030135, 0.06127, -0.043945, 0.0), + float4(0.330601, 0.483055, 0.116951, 7.0), + float4(0.034429, 0.265258, -0.043945, 7.0), + float4(0.542104, 0.741798, 0.116951, 8.0), + float4(0.034429, 0.355442, -0.043945, 15.0), + float4(0.305908, 0.822319, 0.116951, 8.0), + float4(0.267405, 0.532589, -0.043945, 23.0), + float4(0.542104, 0.822319, 0.082595, 8.0), + float4(-0.045345, 0.036855, -0.044263, 31.0), + float4(0.045345, 0.23354, 0.059405, 8.0), + float4(-0.224888, 0.072548, -0.044263, 39.0), + float4(0.059405, 0.565586, 0.059405, 8.0), + float4(-0.032366, 0.042263, -0.044263, 47.0), + float4(0.148095, 0.408756, 0.059405, 8.0), + float4(0.074711, 0.08769, -0.044263, 55.0), + float4(0.184869, 0.464999, 0.064813, 8.0), + float4(0.146095, 0.271559, -0.044263, 63.0), + float4(0.211908, 0.446612, 0.038855, 8.0), + float4(0.23803, 0.310496, -0.044263, 71.0), + float4(0.320067, 0.460672, 0.06265, 8.0), + float4(0.275885, 0.310496, -0.044263, 79.0), + float4(0.412001, 0.475815, 0.047508, 8.0), + float4(0.375391, 0.350514, -0.044263, 87.0), + float4(0.45202, 0.38388, 0.044263, 4.0), + float4(0.410001, 0.350514, -0.044263, 91.0), + float4(0.474733, 0.401185, 0.044263, 5.0), + float4(-0.224888, 0.379717, -0.044263, 96.0), + float4(-0.0812, 0.532057, 0.047508, 8.0), + float4(-0.095098, 0.228295, -0.044263, 104.0), + float4(0.211908, 0.565586, 0.064813, 8.0), + float4(-0.222724, 0.430551, -0.044263, 112.0), + float4(-0.011979, 0.833818, 0.053998, 8.0), + float4(-0.100506, 0.438122, -0.044263, 120.0), + float4(0.327638, 0.833818, 0.0832, 8.0), + float4(0.127708, 0.312659, -0.044263, 128.0), + float4(0.282211, 0.467162, 0.064813, 8.0), + float4(0.245601, 0.352677, -0.044263, 136.0), + float4(0.474733, 0.604523, 0.06265, 8.0), + float4(0.178543, 0.390533, -0.044263, 144.0), + float4(0.474733, 0.639133, 0.0832, 8.0), + float4(0.200174, 0.483549, -0.044263, 152.0), + float4(0.474733, 0.639133, 0.0832, 4.0), + float4(-0.013979, 0.487875, -0.044263, 156.0), + float4(0.467162, 0.833818, 0.0832, 5.0) +}; +constant float2 PRISM_POLY[110] = { + float2(0.006442, -0.002147), + float2(-0.001074, 0.001074), + float2(-0.004294, 0.007515), + float2(-0.004294, 0.806288), + float2(-0.001074, 0.809509), + float2(0.004294, 0.809509), + float2(0.022546, 0.796626), + float2(0.185736, 0.655982), + float2(0.281288, 0.86319), + float2(0.286656, 0.870706), + float2(0.293098, 0.871779), + float2(0.465951, 0.781595), + float2(0.478834, 0.773006), + float2(0.478834, 0.763344), + float2(0.407975, 0.592638), + float2(0.45092, 0.586196), + float2(0.601227, 0.573313), + float2(0.606595, 0.569018), + float2(0.606595, 0.564724), + float2(0.601227, 0.556135), + float2(0.50138, 0.453067), + float2(0.425153, 0.37684), + float2(0.289877, 0.246933), + float2(0.081595, 0.057975), + float2(0.013957, 0.0), + float2(-0.011897, 0.0), + float2(-0.055161, 0.037855), + float2(-0.081119, 0.063813), + float2(-0.083282, 0.06814), + float2(-0.084363, 0.38288), + float2(-0.089771, 0.38288), + float2(-0.158993, 0.33529), + float2(-0.166564, 0.336372), + float2(-0.250927, 0.379635), + float2(-0.261743, 0.388288), + float2(-0.263906, 0.461836), + float2(-0.262824, 0.486712), + float2(-0.173053, 0.638133), + float2(-0.044345, 0.860939), + float2(-0.036774, 0.867429), + float2(-0.024876, 0.871755), + float2(0.33529, 0.871755), + float2(0.349351, 0.867429), + float2(0.363412, 0.853368), + float2(0.513752, 0.619747), + float2(0.513752, 0.379635), + float2(0.466162, 0.322311), + float2(0.461836, 0.320148), + float2(0.399104, 0.311496), + float2(0.393696, 0.313659), + float2(0.374227, 0.332046), + float2(0.343943, 0.288782), + float2(0.336372, 0.282293), + float2(0.269314, 0.271477), + float2(0.263906, 0.27364), + float2(0.243356, 0.292027), + float2(0.21199, 0.244438), + float2(0.207664, 0.241193), + float2(0.149258, 0.231459), + float2(0.138443, 0.231459), + float2(0.118974, 0.248764), + float2(0.116811, 0.222806), + float2(0.117892, 0.162237), + float2(0.115729, 0.076792), + float2(0.06814, 0.011897), + float2(0.062732, 0.007571), + float2(0.005408, 0.0), + float2(0.031135, 0.06227), + float2(0.252301, 0.266258), + float2(0.541104, 0.544325), + float2(0.359663, 0.559356), + float2(0.433742, 0.753681), + float2(0.304908, 0.821319), + float2(0.199693, 0.595859), + float2(0.035429, 0.740798), + float2(0.035429, 0.482055), + float2(-0.043263, 0.077874), + float2(0.001082, 0.037855), + float2(0.044345, 0.043263), + float2(0.075711, 0.08869), + float2(0.077874, 0.407756), + float2(0.10924, 0.439122), + float2(0.111403, 0.307169), + float2(0.147095, 0.272559), + float2(0.189277, 0.28013), + float2(0.210909, 0.313659), + float2(0.210909, 0.445612), + float2(0.23903, 0.45102), + float2(0.241193, 0.343943), + float2(0.276885, 0.311496), + float2(0.319067, 0.319067), + float2(0.34178, 0.353677), + float2(0.340698, 0.474815), + float2(0.375309, 0.488875), + float2(0.376391, 0.381799), + float2(0.411001, 0.351514), + float2(0.45102, 0.359085), + float2(0.473733, 0.391533), + float2(0.473733, 0.603523), + float2(0.466162, 0.618665), + float2(0.326638, 0.830655), + float2(-0.012979, 0.832818), + float2(-0.099506, 0.68356), + float2(-0.221724, 0.475896), + float2(-0.223888, 0.407756), + float2(-0.163319, 0.380717), + float2(-0.094098, 0.431551), + float2(-0.0822, 0.531057), + float2(-0.045426, 0.564586), + float2(-0.044345, 0.23254) +}; +constant float PRISM_BEVEL = 0.012883; +// end of the prism mesh -static float s_gem_arrow(float3 p) -{ - float z0 = SCULPT_HOVER + 0.03; - float slab = max(p.z - z0 - 0.2, z0 - 0.03 - p.z); - float head = max(slab, max(s_gem_edge(p, float2(-0.02, 0.03), float2(-0.02, -0.88), z0), - max(s_gem_edge(p, float2(-0.02, -0.88), float2(0.66, -0.61), z0), - s_gem_edge(p, float2(0.66, -0.61), float2(-0.02, 0.03), z0)))); - float tail = max(slab, max(max(s_gem_edge(p, float2(0.215, -0.665), float2(0.37, -1.0), z0), - s_gem_edge(p, float2(0.37, -1.0), float2(0.53, -0.93), z0)), - max(s_gem_edge(p, float2(0.53, -0.93), float2(0.38, -0.60), z0), - s_gem_edge(p, float2(0.38, -0.60), float2(0.215, -0.665), z0)))); - return min(head, tail); -} +constant float PRISM_IOR = 1.61; +constant float PRISM_DISPERSION = 0.035; +constant float PRISM_GLOW = 0.45; +constant float PRISM_FOLD = 0.35; +constant int PRISM_BOUNCES = 6; +constant float PRISM_MAT = 9.0; -static float s_facet_capsule(float3 p, float3 a, float3 b, float r, float spin) +inline int prism_shape(constant Layer &layer) { - float3 u = normalize(b - a); - float3 v = normalize(cross(u, float3(0.0, 0.0, 1.0))); - float3 w = cross(u, v); - float3 q = p - b; - float h = dot(q, u); - float2 rad = float2(dot(q, v), dot(q, w)); - float d = max(h - r, -dot(p - a, u) - r); - for (int k = 0; k < 6; k++) - { - float an = spin + float(k) * 1.0471976; - float s = dot(rad, float2(cos(an), sin(an))); - d = max(d, s - r); - d = max(d, dot(rad, float2(cos(an + 0.5236), sin(an + 0.5236))) * 0.8660 + h * 0.5 - r); - d = max(d, s * 0.5 + h * 0.8660 - r); - } - return d; + int id = sculpt_id(layer); + return id == 3 || id == 4 ? id - 3 : -1; } -constant float3 SCULPT_FACETS[10] = { - float3(1.0, 0.0, 0.0), float3(0.0, 1.0, 0.0), float3(0.0, 0.0, 1.0), - float3(0.7071, 0.7071, 0.0), float3(0.7071, 0.0, 0.7071), float3(0.0, 0.7071, 0.7071), - float3(0.5774, 0.5774, 0.5774), float3(0.4472, 0.0, 0.8944), float3(0.0, 0.4472, 0.8944), - float3(0.3015, 0.3015, 0.9045) -}; - -static float s_facet_ellipsoid(float3 p, float3 r) +static float prism_poly_dist(float2 p, int i0, int n) { - float3 q = abs(p); - float d = -1e9; - for (int i = 0; i < 10; i++) + float2 vj = PRISM_POLY[i0 + n - 1]; + float d = 1e9; + float sgn = 1.0; + for (int i = 0; i < n; i++) { - d = max(d, dot(q, SCULPT_FACETS[i]) - length(r * SCULPT_FACETS[i])); + float2 vi = PRISM_POLY[i0 + i]; + float2 e = vj - vi; + float2 w = p - vi; + float2 b = w - e * saturate(dot(w, e) / dot(e, e)); + d = min(d, dot(b, b)); + bool c0 = p.y >= vi.y; + bool c1 = p.y < vj.y; + bool c2 = e.x * w.y > e.y * w.x; + if ((c0 && c1 && c2) || (!c0 && !c1 && !c2)) + { + sgn = -sgn; + } + vj = vi; } - return d; + return sgn * sqrt(d); } -static float s_crystal_hand(float3 p) +static float prism_rim(float3 p, int shape, constant Layer &layer) { - float zc = SCULPT_HAND_ZC; - float d = s_facet_ellipsoid(p - float3(0.185, -0.70, zc), float3(0.3, 0.235, 0.125)); - d = min(d, s_facet_capsule(p, float3(0.0, -0.52, zc), float3(0.0, -0.10, zc), 0.098, 0.3)); - d = min(d, s_facet_capsule(p, float3(0.17, -0.57, zc + 0.012), float3(0.17, -0.41, zc + 0.045), 0.086, 0.1)); - d = min(d, s_facet_capsule(p, float3(0.31, -0.59, zc + 0.01), float3(0.31, -0.45, zc + 0.04), 0.08, 0.5)); - d = min(d, s_facet_capsule(p, float3(0.435, -0.625, zc + 0.005), float3(0.435, -0.52, zc + 0.03), 0.07, 0.2)); - return min(d, s_facet_capsule(p, float3(0.03, -0.77, zc + 0.03), float3(-0.165, -0.60, zc + 0.065), 0.082, 0.7)); + float half_t = model_thick(layer) * 0.5; + float sil = prism_poly_dist(p.xy, PRISM_SIL_START[shape], PRISM_SIL_COUNT[shape]); + float2 w = float2(sil + PRISM_BEVEL, abs(p.z + half_t) - (half_t - PRISM_BEVEL)); + return min(max(w.x, w.y), 0.0) + length(max(w, 0.0)) - PRISM_BEVEL; } -static float2 sculpt_proto(float3 p, int theme, int shape) +static float prism_tri(float3 ro, float3 rd, int k) { - if (theme == 3) + float3 v0 = PRISM_TRIS[4 * k].xyz; + float3 e1 = PRISM_TRIS[4 * k + 1].xyz; + float3 e2 = PRISM_TRIS[4 * k + 2].xyz; + float3 pv = cross(rd, e2); + float det = dot(e1, pv); + if (abs(det) < 1e-10) { - return s_voxels(p, shape); + return -1.0; } - if (theme == 1) + float inv = 1.0 / det; + float3 sv = ro - v0; + float u = dot(sv, pv) * inv; + float3 qv = cross(sv, e1); + float v = dot(rd, qv) * inv; + if (u < -2e-4 || v < -2e-4 || u + v > 1.0004) { - return float2(shape == 0 ? s_gem_arrow(p) : s_crystal_hand(p), 8.0); + return -1.0; } - float2 body; - float3 star; - if (shape == 0) - { - float h = 0.075, re = 0.03, dome = 0.0; - if (theme == 2) - { - h = 0.07; - re = 0.06; - dome = 0.035; - } - if (theme == 4) + return dot(e2, qv) * inv; +} + +static float prism_trace(float3 ro, float3 rd, int shape, thread int &hit) +{ + float3 inv = 1.0 / select(float3(1e-6), rd, abs(rd) > 1e-6); + int b0 = PRISM_BOX_START[shape]; + int nb = PRISM_BOX_COUNT[shape]; + float best = 1e9; + hit = -1; + for (int j = b0; j < b0 + nb; j++) + { + float4 lo = PRISM_BOXES[2 * j]; + float4 hi = PRISM_BOXES[2 * j + 1]; + float3 t0 = (lo.xyz - ro) * inv; + float3 t1 = (hi.xyz - ro) * inv; + float3 tn = min(t0, t1); + float3 tf = max(t0, t1); + float tmin = max(max(tn.x, tn.y), tn.z); + float tmax = min(min(tf.x, tf.y), tf.z); + if (tmax < max(tmin, 1e-5) || tmin > best) { - h = 0.07; - re = 0.055; - dome = 0.025; + continue; } - body = float2(s_arrow_solid(p, h, re, dome), 1.0); - if (theme == 0) + int k0 = int(lo.w); + int n = int(hi.w); + for (int i = k0; i < k0 + n; i++) { - body = s_opu(body, float2(s_piping(p, SCULPT_HOVER + 2.0 * h - 0.004), 2.0)); + float t = prism_tri(ro, rd, i); + if (t > 1e-5 && t < best) + { + best = t; + hit = i; + } } - star = float3(0.57, -0.86, SCULPT_HOVER + 2.0 * h + dome); } - else + return best; +} + +static float2 sculpt_proto(float3 p, int theme, int shape) +{ + if (theme == 3) { - body = s_opu(float2(s_glove(p, SCULPT_HAND_ZC), 1.0), float2(s_cuff(p, SCULPT_HAND_ZC), 2.0)); - star = float3(0.185, -0.93, SCULPT_HAND_ZC + 0.15); + return s_voxels(p, shape); } - return theme == 4 ? s_opu(body, s_sprout(p, star)) : body; + return s_rimmed(p, theme, shape); } inline float3 sculpt_point(float3 q, constant Layer &layer) @@ -961,17 +1960,21 @@ static float2 sculpt_eval(float3 q, bool occ, constant Layer &layer) { int id = sculpt_id(layer) - 1; int theme = id / 2, shape = id % 2; + if (theme == 1) + { + return float2(prism_rim(q, shape, layer), 8.0); + } float3 p = sculpt_point(q, layer); float2 r; if (occ && theme == 3) { - float d2 = shape == 0 ? s_arrow2(p.xy) - 1.2 * SCULPT_VOX : s_pixel_hand_dist(p.xy); + float d2 = s_pixel_outline(p.xy, shape); float dz = abs(p.z - (SCULPT_HOVER + 0.07)) - 0.07; r = float2(length(max(float2(d2, dz), 0.0)) + min(max(d2, dz), 0.0), 7.0); } else { - r = sculpt_proto(p, occ && theme == 1 ? 2 : theme, shape); + r = sculpt_proto(p, theme, shape); } return float2(r.x * sculpt_units(layer), r.y); } @@ -1076,16 +2079,13 @@ inline SculptMat s_mat(float3 alb, float rough, float spec, float sss, float ref static float3 s_pixel_colour(float3 p, int shape) { - float2 id = floor((p.xy - s_grid_origin(shape)) / SCULPT_VOX); - if (!s_occ(id + float2(-1.0, 0.0), shape) || !s_occ(id + float2(0.0, -1.0), shape)) - { - return s_lin(0.52, 0.91, 0.77); - } - if (!s_occ(id + float2(1.0, 0.0), shape) || !s_occ(id + float2(0.0, 1.0), shape)) - { - return s_lin(1.0, 0.78, 0.87); - } - return s_lin(1.0, 0.50, 0.71); + int2 cell = s_pix_cell(p.xy, shape); + int4 g = PIX_GRID[shape]; + int row = g.z + clamp(cell.y, 0, g.y - 1); + if (s_pix_bit(PIX_LINE[row], cell.x)) return s_lin(0.29, 0.12, 0.36); + if (s_pix_bit(PIX_HI[row], cell.x)) return s_lin(1.0, 0.78, 0.87); + if (s_pix_bit(PIX_SHADE[row], cell.x)) return s_lin(0.87, 0.27, 0.51); + return s_lin(1.0, 0.435, 0.66); } static SculptMat sculpt_material(float mat, float3 p, int theme, int shape) @@ -1093,28 +2093,26 @@ static SculptMat sculpt_material(float mat, float3 p, int theme, int shape) bool primary = mat < 1.5; if (theme == 0) { - if (shape == 0 && primary) return s_mat(s_lin(0.10, 0.10, 0.115), 0.3, 0.2, 0.0, 0.9); - if (shape == 0) return s_mat(s_lin(0.94, 0.91, 0.84), 0.45, 0.4, 0.2, 0.3); - if (primary) return s_mat(s_lin(0.95, 0.92, 0.85), 0.55, 0.35, 0.35, 0.25); - return s_mat(s_lin(0.17, 0.18, 0.22), 0.35, 0.6, 0.0, 0.6); + if (mat < 2.5) return s_mat(s_lin(0.95, 0.92, 0.85), 0.3, 0.6, 0.35, 0.25); + return s_mat(s_lin(0.1, 0.1, 0.11), 0.55, 0.05, 0.0, 0.1); } + if (theme == 1) return s_mat(s_lin(0.02, 0.05, 0.33), 0.35, 0.5, 0.05, 0.3); if (theme == 2) { - if (shape == 0) return s_mat(s_lin(1.0, 0.40, 0.30), 0.5, 0.45, 0.4, 0.25); - if (primary) return s_mat(s_lin(1.0, 0.79, 0.16), 0.5, 0.45, 0.4, 0.25); - return s_mat(s_lin(0.18, 0.20, 0.29), 0.4, 0.5, 0.0, 0.4); + if ((primary && shape == 0) || (mat > 2.5 && mat < 3.5 && shape == 1)) return s_mat(s_lin(1.0, 0.40, 0.30), 0.85, 0.08, 0.15, 0.03); + if (mat < 3.5) return s_mat(s_lin(1.0, 0.80, 0.10), 0.85, 0.08, 0.15, 0.03); + return s_mat(s_lin(0.09, 0.13, 0.45), 0.9, 0.05, 0.05, 0.02); } if (theme == 4) { - if (primary && shape == 0) return s_mat(s_lin(0.62, 0.91, 0.78), 0.22, 0.8, 0.25, 0.6); - if (primary) return s_mat(s_lin(0.96, 0.94, 0.88), 0.5, 0.35, 0.35, 0.25); - if (mat < 2.5) return s_mat(s_lin(0.62, 0.91, 0.78), 0.25, 0.7, 0.25, 0.5); - if (mat < 3.5) return s_mat(s_lin(1.0, 0.80, 0.20), 0.3, 0.6, 0.3, 0.4); - if (mat < 4.5) return s_mat(s_lin(0.38, 0.80, 0.55), 0.35, 0.5, 0.35, 0.3); - return s_mat(s_lin(0.16, 0.12, 0.10), 0.2, 0.8, 0.0, 0.5); + if (primary && shape == 0) return s_mat(s_lin(0.68, 0.93, 0.80), 0.25, 0.7, 0.3, 0.3); + if (primary) return s_mat(s_lin(0.97, 0.95, 0.90), 0.3, 0.6, 0.35, 0.25); + if (mat < 2.5) return s_mat(s_lin(0.52, 0.87, 0.78), 0.25, 0.7, 0.3, 0.3); + if (mat < 3.5) return s_mat(s_lin(1.0, 0.75, 0.25), 0.25, 0.7, 0.3, 0.3); + if (mat < 4.5) return s_mat(s_lin(0.62, 0.92, 0.72), 0.3, 0.6, 0.3, 0.35); + return s_mat(s_lin(0.07, 0.15, 0.33), 0.7, 0.15, 0.05, 0.08); } - if (mat < 6.5) return s_mat(s_pixel_colour(p, shape), 0.45, 0.35, 0.15, 0.2); - return s_mat(s_lin(0.36, 0.18, 0.54), 0.45, 0.35, 0.1, 0.2); + return s_mat(s_pixel_colour(p, shape), 0.65, 0.1, 0.12, 0.04); } static float3 model_env(float3 d, float rough, float3 l, float3 fill) @@ -1127,34 +2125,6 @@ static float3 model_env(float3 d, float rough, float3 l, float3 fill) return col; } -static float3 s_gem(float k) -{ - k = saturate(k) * 3.0; - float3 a = s_lin(0.20, 0.95, 1.0); - float3 b = s_lin(0.15, 0.42, 1.0); - float3 c = s_lin(0.45, 0.25, 0.95); - float3 d = s_lin(0.88, 0.50, 1.0); - if (k < 1.0) return mix(a, b, k); - if (k < 2.0) return mix(b, c, k - 1.0); - return mix(c, d, k - 2.0); -} - -static float3 model_shade_crystal(float3 n, float3 rd, float3 L, float fall, float3 l, float3 fill) -{ - float cosi = saturate(dot(-rd, n)); - float F = 0.04 + 0.96 * pow(1.0 - cosi, 5.0); - float3 t = refract(rd, n, 1.0 / 1.6); - float k = 0.42 + 0.9 * dot(float2(n.x, -n.y), float2(0.7557, -0.6549)) - + 0.6 * dot(float2(t.x, -t.y), float2(0.6, -0.8)); - float3 body = float3(s_gem(k - 0.08).r, s_gem(k).g, s_gem(k + 0.08).b); - float3 col = body * (0.1 + 1.8 * fall * pow(max(dot(n, L), 0.0), 2.5)); - col += body * model_env(reflect(t, float3(0.0, 0.0, 1.0)) * float3(1.0, 1.0, -1.0), 0.15, l, fill) * 0.5; - col += pow(max(dot(reflect(rd, n), L), 0.0), 30.0) * 2.0; - col += model_env(reflect(rd, n), 0.05, l, fill) * F; - col += s_lin(0.5, 0.9, 1.0) * pow(1.0 - cosi, 4.0) * 0.6; - return col; -} - static float3 model_tonemap(float3 c) { const float start = 0.76; @@ -1178,16 +2148,16 @@ static float3 model_shade(float3 q, float3 n, float3 rd, float3 L, float fall, f texture2d texImg) { int id = sculpt_id(layer); - if (id > 0 && (id - 1) / 2 == 1) - { - return model_tonemap(model_shade_crystal(n, rd, L, fall, l, fill)); - } SculptMat m; float gloss = 1.0; if (id > 0) { float3 p = sculpt_point(q, layer) - float3(n.x, -n.y, n.z) * 0.01; m = sculpt_material(mat, p, (id - 1) / 2, (id - 1) % 2); + if (id == 3 || id == 4) + { + gloss = 1.0 - smoothstep(0.97, 0.995, abs(n.z)); + } } else { @@ -1211,13 +2181,168 @@ static float3 model_shade(float3 q, float3 n, float3 rd, float3 L, float fall, f return model_tonemap(col); } +// ---- Le cristal de Prism Glow ---- (cf. HLSL) + +inline float prism_fresnel(float c, float ior) +{ + float f0 = (ior - 1.0) / (ior + 1.0); + f0 *= f0; + return f0 + (1.0 - f0) * pow(1.0 - c, 5.0); +} + +static float prism_primary(float3 ro, float3 rd, thread int &hc, constant Layer &layer) +{ + hc = -1; + int shape = prism_shape(layer); + if (shape < 0) + { + return 1e9; + } + float t = prism_trace(ro, rd, shape, hc); + if (hc >= 0 && fmod(PRISM_TRIS[4 * hc].w, 2.0) > 0.5 && ro.z + rd.z * t < 0.0) + { + hc = -1; + return 1e9; + } + return t; +} + +inline float3 prism_normal(int h, float3 rd) +{ + float3 n = PRISM_TRIS[4 * h + 3].xyz; + return dot(n, rd) > 0.0 ? -n : n; +} + +inline float prism_seg(float3 p, float3 a, float3 b) +{ + float3 pa = p - a, ba = b - a; + return length(pa - ba * saturate(dot(pa, ba) / dot(ba, ba))); +} + +static float prism_edge_px(float3 p, int h, float px) +{ + float4 r0 = PRISM_TRIS[4 * h]; + float3 a = r0.xyz; + float3 b = a + PRISM_TRIS[4 * h + 1].xyz; + float3 c = a + PRISM_TRIS[4 * h + 2].xyz; + int bits = int(r0.w * 0.5); + float d = 1e9; + if ((bits & 1) != 0) d = min(d, prism_seg(p, b, c)); + if ((bits & 2) != 0) d = min(d, prism_seg(p, c, a)); + if ((bits & 4) != 0) d = min(d, prism_seg(p, a, b)); + return d / px; +} + +static float3 plane_to_world(float3 v, ModelFrame f) +{ + float x = v.x * f.c.z - v.y * f.s.z; + float y = v.x * f.s.z + v.y * f.c.z; + float z = -x * f.s.y + v.z * f.c.y; + return float3(x * f.c.y + v.z * f.s.y, y * f.c.x - z * f.s.x, y * f.s.x + z * f.c.x); +} + +static float3 prism_screen(float3 q, float3 d, float3 nz, float hz, float3 tip, float unit, ModelFrame f, + constant Layer &layer, texture2d texFrame) +{ + float denom = dot(d, nz); + if (denom > -1e-4) + { + return SCULPT_SCREEN * 0.2; + } + float3 g = q + d * ((hz - dot(q, nz)) / denom); + float3 w = plane_to_world(tip + unit * model_to_plane(g, f), f); + float k = 1.0 - w.z / layer.src.z; + if (k < 1e-3) + { + return SCULPT_SCREEN * 0.2; + } + float2 uv = layer.dst.xy + ((w.xy + layer.mb.zw) / k - layer.src.xy) / layer.quad_px * layer.dst.zw; + if (any(uv < 0.0) || any(uv > 1.0)) + { + return SCULPT_SCREEN * 0.2; + } + return pow(texFrame.sample(samp, uv, level(0.0)).rgb, float3(2.2)); +} + +static float3 prism_shade(float3 p, float3 n, float3 rd, int h, float px, float3 l, float3 fill, float3 nz, + float hz, float3 tip, float unit, ModelFrame f, constant Layer &layer, + texture2d texFrame) +{ + int shape = prism_shape(layer); + float4 r3 = PRISM_TRIS[4 * h + 3]; + float3 facet = float3(PRISM_TRIS[4 * h + 1].w, PRISM_TRIS[4 * h + 2].w, r3.w); + float F = prism_fresnel(saturate(-dot(rd, n)), PRISM_IOR); + float zb = -model_thick(layer); + float3 trans = float3(0.0); + for (int ch = 0; ch < 3; ch++) + { + float3 mask = float3(float(ch == 0), float(ch == 1), float(ch == 2)); + float ior = PRISM_IOR + PRISM_DISPERSION * float(ch - 1); + float3 d = refract(rd, n, 1.0 / ior); + float3 pos = p; + float thr = 1.0; + float acc = 0.0; + for (int b = 0; b < PRISM_BOUNCES; b++) + { + int hh; + float th = prism_trace(pos, d, shape, hh); + float tb = d.z < -1e-6 ? (zb - pos.z) / d.z : 1e9; + if (tb <= th) + { + float3 qb = pos + d * tb; + float3 d2 = refract(d, float3(0.0, 0.0, 1.0), ior); + if (dot(d2, d2) < 1e-8) + { + pos = qb; + d = reflect(d, float3(0.0, 0.0, 1.0)); + continue; + } + acc += thr * dot(prism_screen(qb, d2, nz, hz, tip, unit, f, layer, texFrame), mask); + thr = 0.0; + break; + } + if (hh < 0) + { + break; + } + pos += d * th; + float4 rn = PRISM_TRIS[4 * hh + 3]; + if (fmod(PRISM_TRIS[4 * hh].w, 2.0) > 0.5 && pos.z < 0.0) + { + acc += thr * dot(s_lin(0.03, 0.06, 0.37) * 0.35, mask); + thr = 0.0; + break; + } + float3 dout = refract(d, -rn.xyz, ior); + if (dot(dout, dout) < 1e-8) + { + d = reflect(d, -rn.xyz); + continue; + } + float fi = prism_fresnel(saturate(dot(dout, rn.xyz)), ior); + acc += thr * (1.0 - fi) * dot(model_env(dout, 0.05, l, fill), mask); + thr *= fi; + d = reflect(d, -rn.xyz); + } + trans += acc * mask; + } + float3 col = F * model_env(reflect(rd, n), 0.05, l, fill) + (1.0 - F) * trans + facet * PRISM_GLOW; + float fold = 1.0 - smoothstep(0.2, 1.0, prism_edge_px(p, h, px)); + col = mix(col, s_lin(0.96, 0.99, 1.0) * 1.3, PRISM_FOLD * fold * (0.35 + 0.65 * saturate(dot(n, l)))); + return model_tonemap(col); +} + // Les passes de la boucle unique de `cursor_model` (cf. HLSL : un seul appel de `model_eval`). constant int STAGE_MARCH = 0; constant int STAGE_NORMAL = 1; constant int STAGE_AO = 2; constant int STAGE_SELF = 3; constant int STAGE_PLANE = 4; -constant int STAGE_DONE = 5; +constant int STAGE_EDGE = 5; +constant int STAGE_SHADE = 6; +constant int STAGE_DONE = 7; + +constant float2 MODEL_SUBPIXEL[3] = { float2(0.35, 0.2), float2(-0.35, 0.2), float2(0.0, -0.4) }; inline float3 model_tetra(int k) { @@ -1226,7 +2351,8 @@ inline float3 model_tetra(int k) static float4 cursor_model(float2 local, constant Layer &layer, texture2d texSdf, - texture2d texImg) + texture2d texImg, + texture2d texFrame) { ModelFrame f; f.c = cos(layer.fx.xyz); @@ -1253,15 +2379,21 @@ static float4 cursor_model(float2 local, constant Layer &layer, float stride = sculpt_id(layer) > 0 ? 0.85 : 1.0; float3 lamp = float3(layer.color.rg + sprite_size(layer) * 0.5, 0.0) + l * SCULPT_LAMP_DIST; - float2 tb = ray_box(ro, rd, lo - 0.02, hi + 0.02); + float3 ray = rd; + float2 tb = ray_box(ro, ray, lo - 0.02, hi + 0.02); int stage = tb.x < tb.y && tb.y > 0.0 ? STAGE_MARCH : STAGE_DONE; bool plane_next = stage == STAGE_DONE; + bool need_tc = stage == STAGE_MARCH; + float tc = 1e9; + int hc = -1; int k = 0; float t = max(tb.x, 0.0); float best = 1e9; float t_best = t; + float d_best = 0.0; float mat = 0.0; float cov = 0.0; + float cov_s = 0.0; float3 q = float3(0.0); float3 n = float3(0.0); float3 L = float3(0.0); @@ -1273,12 +2405,20 @@ static float4 cursor_model(float2 local, constant Layer &layer, float inside = 0.0; float contact = 0.0; float dropped = 0.0; - for (int it = 0; it < 180; it++) + int sub = 0; + int shaded = 0; + float4 acc = float4(0.0); + for (int it = 0; it < 500; it++) { + if (need_tc) + { + need_tc = false; + tc = prism_primary(ro, ray, hc, layer); + } if (plane_next) { plane_next = false; - stage = STAGE_DONE; + stage = cov > 0.0 ? STAGE_SHADE : STAGE_DONE; float denom = dot(rd, nz); if (cov < 1.0 && denom < -1e-4) { @@ -1293,18 +2433,64 @@ static float4 cursor_model(float2 local, constant Layer &layer, } } } + if (stage == STAGE_SHADE) + { + if (cov_s > 0.0) + { + float3 c; + if (mat > PRISM_MAT - 0.5) + { + c = prism_shade(q, n, ray, hc, t_best / dlen, l, fill, nz, hz, tip, unit, f, layer, texFrame); + } + else + { + float3 tl = lamp - q; + float fall = SCULPT_LAMP_DIST * SCULPT_LAMP_DIST / dot(tl, tl); + c = model_shade(q, n, ray, normalize(tl), fall, sh, ao, mat, l, fill, layer, texSdf, texImg); + } + acc += float4(c * cov_s, cov_s); + } + shaded++; + stage = STAGE_DONE; + if (sub > 0) + { + float3 dws = float3(local + MODEL_SUBPIXEL[3 - sub] + layer.src.xy, -persp); + sub--; + dlen = length(dws); + ray = plane_to_model(world_to_plane(dws / dlen, f), f); + tb = ray_box(ro, ray, lo - 0.02, hi + 0.02); + cov_s = 0.0; + stage = STAGE_SHADE; + if (tb.x < tb.y && tb.y > 0.0) + { + stage = STAGE_MARCH; + k = 0; + t = max(tb.x, 0.0); + best = 1e9; + t_best = t; + need_tc = true; + } + } + continue; + } float3 pos; if (stage == STAGE_MARCH) { - pos = ro + rd * t; + pos = ro + ray * t; } else if (stage == STAGE_NORMAL) { pos = q + 0.002 * model_tetra(k); } + else if (stage == STAGE_EDGE) + { + float3 side = normalize(cross(n, ray)); + float3 e = (k >= 2 ? cross(n, side) : side) * (k % 2 == 1 ? -1.0 : 1.0); + pos = q + e * 0.5 * t_best / dlen; + } else if (stage == STAGE_AO) { - pos = q + (0.01 + 0.0175 * float(k)) * SCULPT_SCALE * n; + pos = q + (0.01 + 0.0175 * k) * SCULPT_SCALE * n; } else if (stage == STAGE_SELF) { @@ -1328,22 +2514,64 @@ static float4 cursor_model(float2 local, constant Layer &layer, { best = hit ? 0.0 : d / fp; t_best = t; + d_best = d; mat = m.y; } t += d * stride; k++; + if (!hit && t >= tc) + { + hit = true; + best = 0.0; + t_best = tc; + d_best = 0.0; + mat = PRISM_MAT; + } if (hit || t > tb.y || k == 96) { - cov = saturate(1.0 - best); - if (cov > 0.0) + cov_s = saturate(1.0 - best); + if (shaded > 0) { - q = ro + rd * t_best; - stage = STAGE_NORMAL; - k = 0; + stage = STAGE_SHADE; + if (cov_s > 0.0) + { + q = ro + ray * t_best; + n = float3(0.0); + stage = STAGE_NORMAL; + k = 0; + } } else { - plane_next = true; + cov = cov_s; + if (cov > 0.0) + { + q = ro + ray * t_best; + stage = STAGE_NORMAL; + k = 0; + } + else + { + plane_next = true; + } + } + if (stage == STAGE_NORMAL && mat > PRISM_MAT - 0.5) + { + n = prism_normal(hc, ray); + k = 0; + if (shaded > 0) + { + stage = STAGE_SHADE; + } + else + { + if (prism_edge_px(q, hc, t_best / dlen) < 0.75) + { + sub = 3; + } + stage = STAGE_DONE; + plane_next = true; + } } } } @@ -1354,6 +2582,31 @@ static float4 cursor_model(float2 local, constant Layer &layer, if (k == 4) { n = normalize(n); + k = 0; + stage = shaded > 0 ? STAGE_SHADE : STAGE_EDGE; + } + } + else if (stage == STAGE_EDGE) + { + if (m.y != mat || abs(d - d_best) > 0.02 * t_best / dlen) + { + sub = 3; + } + if (k == 0 && mat > 7.5) + { + int ps = prism_shape(layer); + if (ps >= 0) + { + float dout = prism_poly_dist(q.xy, PRISM_OUT_START[ps], PRISM_OUT_COUNT[ps]); + if (abs(dout) < 0.75 * t_best / dlen) + { + sub = 3; + } + } + } + k++; + if (k == 4) + { stage = STAGE_AO; k = 0; ao = 0.0; @@ -1361,7 +2614,7 @@ static float4 cursor_model(float2 local, constant Layer &layer, } else if (stage == STAGE_AO) { - ao += ((0.01 + 0.0175 * float(k)) * SCULPT_SCALE - d) * pow(0.85, float(k)); + ao += ((0.01 + 0.0175 * k) * SCULPT_SCALE - d) * pow(0.85, float(k)); k++; if (k == 5) { @@ -1402,7 +2655,7 @@ static float4 cursor_model(float2 local, constant Layer &layer, k = 1; if (!(tbp.x < tbp.y && tbp.y > 0.0)) { - stage = STAGE_DONE; + stage = cov > 0.0 ? STAGE_SHADE : STAGE_DONE; } } else @@ -1414,21 +2667,15 @@ static float4 cursor_model(float2 local, constant Layer &layer, { res = saturate(res); dropped = 1.0 - res * res * (3.0 - 2.0 * res); - stage = STAGE_DONE; + stage = cov > 0.0 ? STAGE_SHADE : STAGE_DONE; } } } - float3 rgb = float3(0.0); - if (cov > 0.0) - { - float3 tl = lamp - q; - float fall = SCULPT_LAMP_DIST * SCULPT_LAMP_DIST / dot(tl, tl); - rgb = model_shade(q, n, rd, normalize(tl), fall, sh, ao, mat, l, fill, layer, texSdf, texImg); - } + float w = 1.0 / max(shaded, 1); float shadow = inside * max(dropped * MODEL_SHADOW_ALPHA, contact * MODEL_CONTACT_ALPHA); - float a = cov * layer.color.a; - return float4(rgb * a, a + (1.0 - a) * shadow * layer.color.a); // prémultiplié, ombre noire + float a = acc.a * w * layer.color.a; + return float4(acc.rgb * w * layer.color.a, a + (1.0 - a) * shadow * layer.color.a); // prémultiplié, ombre noire } // ============ Impact du clic (mode 16) ============ @@ -2009,22 +3256,16 @@ static float4 device_frame(float2 local, constant Layer &layer) return float4(rgb * a, a); // prémultiplié } -fragment float4 ps_main(VSOut i [[stage_in]], - constant Layer &layer [[buffer(0)]], - texture2d texY [[texture(0)]], - texture2d texUV [[texture(1)]], - texture2d texImg [[texture(2)]], - // Masque de segmentation du sujet webcam. Non lie tant qu'aucun - // masque n'existe : Metal rend alors 0, ce qui est sans effet - // puisque la branche n'est prise que si layer.fx.z > 0.5. - texture2d texMask [[texture(3)]], - // Champ de distance du sprite de curseur (mode 15 seulement), R16F, cf. - // `cursor_sdf.rs`. Le sprite lui-même est en texture(2), comme aux - // modes 7 et 13. - texture2d texSdf [[texture(4)]], - // Pyramide de profondeur de champ (`tilted_sample`), lue par le mode 8 et - // par le repli du mode 18, qui garde texture(2) pour son rendu isolé. - texture2d texDof [[texture(5)]]) +// The body of `ps_main`, which only adds the layer's transparency on top of it. Same resources +// as `ps_main`, without their bindings (MSL only allows those on an entry point). +inline float4 ps_layer(VSOut i, + constant Layer &layer, + texture2d texY, + texture2d texUV, + texture2d texImg, + texture2d texMask, + texture2d texSdf, + texture2d texDof) { // mode 18 : l'écran CADRÉ (ombre, cadre, métrage, appareil) flouté comme UN objet rigide // (`FrameGeometry::screen_trail`), port 1:1 du HLSL. texImg = son rendu isolé, prémultiplié, @@ -2082,6 +3323,7 @@ fragment float4 ps_main(VSOut i [[stage_in]], return acc / max(n, 1.0); } +#ifdef LAYER_MODELS // mode 17 : CADRE D'APPAREIL MODELÉ (`device_frame`). Testé en premier, comme le 16. if (layer.mode > 16.5) { @@ -2103,8 +3345,16 @@ fragment float4 ps_main(VSOut i [[stage_in]], { return float4(0.0, 0.0, 0.0, 0.0); } - return cursor_model(i.local, layer, texSdf, texImg); + return cursor_model(i.local, layer, texSdf, texImg, texDof); + } +#else + // Les modèles 3D n'existent que dans la variante `LAYER_MODELS` ; un draw qui les enverrait + // ici ne peint rien plutôt qu'une ombre faite de leurs emplacements. + if (layer.mode > 14.5) + { + return float4(0.0, 0.0, 0.0, 0.0); } +#endif // mode 14 : CADRE DE FENÊTRE autour de l'écran, dessiné SOUS lui. Cf. commentaires HLSL. // Testé avant le mode 13, dont la branche n'a pas de borne haute. @@ -2248,8 +3498,12 @@ fragment float4 ps_main(VSOut i [[stage_in]], float2 plane_px = layer.dst_prev.xy; float2 p = float2(r.x, r.y) * plane_px - plane_px * 0.5; float rad = max(layer.radius_px, 0.0); - float d = (layer.dst_prev.z > 0.5) ? sd_screen_under_bar(p, plane_px * 0.5, rad, layer.color.z) - : sd_round_rect(p, plane_px * 0.5, rad); + // Une branche et non `?:`, pour ne jamais évaluer les DEUX distances (trois SDF par pixel). + float d; + if (layer.dst_prev.z > 0.5) + d = sd_screen_under_bar(p, plane_px * 0.5, rad, layer.color.z); + else + d = sd_round_rect(p, plane_px * 0.5, rad); float tilt_a = 1.0 - smoothstep(0.0, 1.5, d); // Slot d'un layout en bloc (`color.w` = rayon de ses coins, px ; 0 ailleurs, sans effet) : // `dst` EST le slot, dont le rect arrondi rogne le plan (`ScreenMask`, cf. HLSL). @@ -2502,12 +3756,25 @@ fragment float4 ps_main(VSOut i [[stage_in]], float3 rgb; // 1 sauf en detourage, ou il porte le masque du sujet. Cf. la branche fx.z plus bas. float alpha_mask = 1.0; + // 0 where a camera homography finds no camera point (`persp_camera`), 1 everywhere else. + float in_picture = 1.0; if (layer.mode < 0.5) { // flou de mouvement par vélocité (§8) float2 uv_now = i.uv; float2 localp = (i.pout - layer.dst_prev.xy) / layer.dst_prev.zw; float2 uv_prev = layer.src_prev.xy + localp * (layer.src_prev.zw - layer.src_prev.xy); + if (layer.layer_fx.y > 0.5) + { + // Camera homography: the texture uv comes from the fragment's place in the quad + // (`src` is not used), scaled to the valid part of the decoder texture. The previous + // frame's uv is the same map at the quad's previous place. + float3 cam = persp_camera(layer, (i.pout - layer.dst.xy) / layer.dst.zw); + in_picture = cam.z; + uv_now = cam.xy * layer.fx.xy; + float3 cam_prev = persp_camera(layer, localp); + uv_prev = (cam_prev.z > 0.5) ? cam_prev.xy * layer.fx.xy : uv_now; + } float2 duv = uv_now - uv_prev; float mb_scale = saturate(layer.mb.y); float2 duv_blur = duv * mb_scale; @@ -2542,13 +3809,31 @@ fragment float4 ps_main(VSOut i [[stage_in]], } else if (effect > 1.5) { - rgb = mix(blur_webcam_bg(uv_now, layer.fx.w, layer.quad_px, i.local, texY, texUV), rgb, person); + rgb = mix(blur_webcam_bg(uv_now, layer.fx.w, layer.quad_px, i.local, layer.fx.xy, texY, texUV), rgb, person); } else { alpha_mask = person; } } + + // Desk-view cover: the camera is being tilted, so the whole picture is blurred and + // dimmed (cover.x = strength, cover.y = radius in quad px, cover.z = dim at full cover). + if (layer.cover.x > 0.001) + { + float3 hidden = blur_webcam_radius(uv_now, layer.cover.y, layer.quad_px, i.local, layer.fx.xy, + texY, texUV); + rgb = mix(rgb, hidden, layer.cover.x) * (1.0 - layer.cover.z * layer.cover.x); + } + + // Past the camera picture the layer is black, as opaque as the rest of it (spec: a + // corrected crop that reaches past the camera image has a black edge, never the + // wallpaper). The mask and effects have no picture to act on there. + if (in_picture < 0.5) + { + rgb = float3(0.0, 0.0, 0.0); + alpha_mask = 1.0; + } } else { @@ -2562,13 +3847,39 @@ fragment float4 ps_main(VSOut i [[stage_in]], // cadre (`sd_screen_under_bar`, `mb.z` = la remontée du contour intérieur, px). float2 halfsz = layer.quad_px * 0.5; float2 p = i.local - layer.quad_px * 0.5; - float d = (layer.mb.w > 0.5) ? sd_screen_under_bar(p, halfsz, layer.radius_px, layer.mb.z) - : sd_round_rect(p, halfsz, layer.radius_px); + // Une branche et non `?:`, pour ne jamais évaluer les DEUX distances : trois SDF par pixel + // de l'écran au lieu d'une, 0,6 ms par frame 1080p sur une Radeon 610M (mesuré en WGSL). + float d; + if (layer.mb.w > 0.5) + d = sd_screen_under_bar(p, halfsz, layer.radius_px, layer.mb.z); + else + d = sd_round_rect(p, halfsz, layer.radius_px); alpha *= 1.0 - smoothstep(0.0, 1.5, d); } return float4(rgb * alpha, alpha); } +fragment float4 ps_main(VSOut i [[stage_in]], + constant Layer &layer [[buffer(0)]], + texture2d texY [[texture(0)]], + texture2d texUV [[texture(1)]], + texture2d texImg [[texture(2)]], + // Masque de segmentation du sujet webcam. Non lie tant qu'aucun + // masque n'existe : Metal rend alors 0, ce qui est sans effet + // puisque la branche n'est prise que si layer.fx.z > 0.5. + texture2d texMask [[texture(3)]], + // Champ de distance du sprite de curseur (mode 15 seulement), R16F, cf. + // `cursor_sdf.rs`. Le sprite lui-même est en texture(2), comme aux + // modes 7 et 13. + texture2d texSdf [[texture(4)]], + // Pyramide de profondeur de champ (`tilted_sample`), lue par le mode 8 et + // par le repli du mode 18, qui garde texture(2) pour son rendu isolé. + texture2d texDof [[texture(5)]]) +{ + // Premultiplied, so the transparency scales all four channels; 0 leaves them as they are. + return ps_layer(i, layer, texY, texUV, texImg, texMask, texSdf, texDof) * (1.0 - layer.layer_fx.x); +} + // ================================================================================= // Fullscreen pass : RGB -> NV12. Mêmes shaders que la passe équivalente HLSL. // ================================================================================= diff --git a/crates/compositor/src/shared_frames.rs b/crates/compositor/src/shared_frames.rs new file mode 100644 index 0000000000..a6b4f8415a --- /dev/null +++ b/crates/compositor/src/shared_frames.rs @@ -0,0 +1,338 @@ +//! Livraison de la preview par textures partagées : le thread de rendu copie chaque frame +//! composée dans l'une des textures d'un petit anneau, et Electron l'importe côté GPU +//! (`sharedTexture.importSharedTexture`) au lieu d'en recevoir les pixels. Plus de `Map` +//! qui attend le GPU puis recopie l'image en RAM, plus de `Vec` de plusieurs Mo par frame +//! à travers l'IPC : mesuré, ce transport coûtait à lui seul 37 à 55 % du thread principal +//! du renderer à 30 images/s. +//! +//! Deux moitiés : +//! - `SlotBook`, portable et sans GPU : quelle case porte la frame prête, lesquelles +//! Chromium tient encore. Partagé entre le thread de rendu (`claim`, `publish`) et le +//! thread Node (`take`, `release`). +//! - `SharedRing` (Windows, backend matériel) : les textures D3D11 et leurs handles NT. +//! macOS (IOSurface) et Linux (dmabuf) restent sur le readback. + +use crate::frame_geometry::FootageQuad; +use crate::live::FramePosition; +use std::time::{Duration, Instant}; + +/// Cases de l'anneau : une prête, une en transit vers le renderer (tenue jusqu'à ce que +/// Chromium la relâche), une en écriture, et une de marge pour la latence de la libération. +pub const RING_SLOTS: u32 = 4; + +/// Une case tenue plus longtemps que ça a perdu sa libération (renderer rechargé, import +/// échoué en route) : Chromium relâche les autres dans la milliseconde. Même ordre que le +/// délai d'Electron sur `sendSharedTexture` (1 s). +pub const LOST_SLOT_AFTER: Duration = Duration::from_secs(1); + +/// Une frame composée posée dans une case de l'anneau, telle que JS la reçoit. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct SharedFrame { + /// Même compteur que les frames lues en RAM : le consommateur ne voit qu'une suite de + /// générations, quel que soit le transport. + pub gen: u64, + pub slot: u32, + /// Valeur du handle NT de la texture de la case, valable dans CE processus. Electron le + /// duplique à l'import : la case garde le sien. + pub handle: u64, + pub width: u32, + pub height: u32, + pub footage: Option, + pub position: FramePosition, +} + +/// Une case prise par JS : la génération qu'elle porte et quand elle est partie. +#[derive(Debug, Clone, Copy)] +struct Held { + slot: u32, + gen: u64, + since: Instant, +} + +/// Qui tient quelle case de l'anneau. +#[derive(Debug, Default)] +pub struct SlotBook { + /// La dernière frame publiée, que JS n'a pas encore prise. + ready: Option, + /// Les cases prises par JS, que Chromium n'a pas encore relâchées, plus anciennes d'abord. + held: Vec, +} + +impl SlotBook { + /// La case où écrire la prochaine frame. Une case libre d'abord ; à défaut, celle de la + /// frame prête que personne n'a prise, retirée puisqu'elle est périmée. `None` quand + /// Chromium tient toutes les cases : une case tenue n'est jamais réécrite, même perdue + /// (`lost`), puisqu'il peut encore la lire. + pub fn claim(&mut self, slots: u32) -> Option { + let ready = self.ready.map(|frame| frame.slot); + let is_held = |slot: u32| self.held.iter().any(|held| held.slot == slot); + if let Some(free) = (0..slots).find(|slot| !is_held(*slot) && Some(*slot) != ready) { + return Some(free); + } + if let Some(stale) = ready { + self.ready = None; + return Some(stale); + } + None + } + + /// Une case est tenue depuis `LOST_SLOT_AFTER` : sa libération ne viendra plus. + pub fn lost(&self, now: Instant) -> bool { + self.held.first().is_some_and(|held| now.duration_since(held.since) >= LOST_SLOT_AFTER) + } + + pub fn publish(&mut self, frame: SharedFrame) { + self.ready = Some(frame); + } + + /// La frame prête, si elle est plus récente que `since_gen`. Sa case reste tenue jusqu'à + /// `release` : le thread de rendu n'y écrira plus d'ici là. + pub fn take(&mut self, since_gen: u64, now: Instant) -> Option { + let frame = self.ready.filter(|frame| frame.gen > since_gen)?; + self.ready = None; + self.held.push(Held { slot: frame.slot, gen: frame.gen, since: now }); + Some(frame) + } + + /// Chromium a relâché la frame `gen` de la case `slot`. La génération évite qu'une + /// libération en double ne libère la frame suivante posée dans la même case. + pub fn release(&mut self, slot: u32, gen: u64) { + self.held.retain(|held| held.slot != slot || held.gen != gen); + } + + /// Oublie la frame prête : la vue repasse au readback, personne ne viendra la prendre. + pub fn clear_ready(&mut self) { + self.ready = None; + } +} + +#[cfg(windows)] +pub use ring::SharedRing; + +#[cfg(windows)] +mod ring { + use super::RING_SLOTS; + use crate::d3d::Gpu; + use anyhow::{bail, Result}; + use std::time::{Duration, Instant}; + use windows::core::{Interface, PCWSTR}; + use windows::Win32::Foundation::{CloseHandle, BOOL, HANDLE}; + use windows::Win32::Graphics::Direct3D11::{ + ID3D11Device, ID3D11DeviceContext, ID3D11Query, ID3D11Texture2D, + D3D11_ASYNC_GETDATA_DONOTFLUSH, D3D11_BIND_RENDER_TARGET, D3D11_BIND_SHADER_RESOURCE, + D3D11_QUERY_DESC, D3D11_QUERY_EVENT, D3D11_RESOURCE_MISC_SHARED, + D3D11_RESOURCE_MISC_SHARED_NTHANDLE, D3D11_TEXTURE2D_DESC, D3D11_USAGE_DEFAULT, + }; + use windows::Win32::Graphics::Dxgi::Common::{DXGI_FORMAT_R8G8B8A8_UNORM, DXGI_SAMPLE_DESC}; + use windows::Win32::Graphics::Dxgi::{ + IDXGIResource1, DXGI_SHARED_RESOURCE_READ, DXGI_SHARED_RESOURCE_WRITE, + }; + + /// Au-delà, le GPU est bloqué ou perdu : on rend une erreur plutôt que de geler la vue. + const COPY_TIMEOUT: Duration = Duration::from_millis(500); + + struct SlotTexture { + tex: ID3D11Texture2D, + handle: HANDLE, + width: u32, + height: u32, + } + + impl Drop for SlotTexture { + fn drop(&mut self) { + // Electron importe un duplicata du handle : fermer le nôtre ne retire rien à une + // frame que Chromium affiche encore. + unsafe { + let _ = CloseHandle(self.handle); + } + } + } + + /// Les textures partagées d'une vue live, créées à la demande à la taille du rendu. + pub struct SharedRing { + device: ID3D11Device, + context: ID3D11DeviceContext, + copied: ID3D11Query, + slots: Vec>, + } + + impl SharedRing { + pub fn new(gpu: &Gpu) -> Result { + let desc = D3D11_QUERY_DESC { Query: D3D11_QUERY_EVENT, MiscFlags: 0 }; + let mut copied = None; + unsafe { gpu.device.CreateQuery(&desc, Some(&mut copied))? }; + Ok(SharedRing { + device: gpu.device.clone(), + context: gpu.context.clone(), + copied: copied.expect("CreateQuery a réussi sans requête"), + slots: (0..RING_SLOTS).map(|_| None).collect(), + }) + } + + /// Copie `src` (`width`×`height`, RGBA8) dans la case `slot` et rend le handle de sa + /// texture une fois le GPU arrivé au bout de la copie. Chromium la lit depuis son propre + /// device, sans keyed mutex : quand le handle part, plus rien ne doit y écrire. L'attente + /// est celle que faisait déjà le `Map` du readback, la recopie en RAM en moins. + /// + /// La case ne doit être ni prête ni tenue (`SlotBook::claim`) : sa texture peut être + /// recréée ici quand la taille du rendu a changé. + pub unsafe fn write( + &mut self, + slot: u32, + src: &ID3D11Texture2D, + width: u32, + height: u32, + ) -> Result { + let entry = &mut self.slots[slot as usize]; + if !matches!(entry, Some(texture) if texture.width == width && texture.height == height) { + *entry = Some(create_slot(&self.device, width, height)?); + } + let target = entry.as_ref().expect("case créée juste au-dessus"); + self.context.CopyResource(&target.tex, src); + self.context.End(&self.copied); + self.context.Flush(); + let deadline = Instant::now() + COPY_TIMEOUT; + loop { + let mut done = BOOL(0); + self.context.GetData( + &self.copied, + Some(&mut done as *mut BOOL as *mut core::ffi::c_void), + std::mem::size_of::() as u32, + D3D11_ASYNC_GETDATA_DONOTFLUSH.0 as u32, + )?; + if done.as_bool() { + break; + } + if Instant::now() > deadline { + bail!("le GPU n'a pas fini de copier la frame en {COPY_TIMEOUT:?}"); + } + std::thread::yield_now(); + } + Ok(target.handle.0 as u64) + } + } + + unsafe fn create_slot(device: &ID3D11Device, width: u32, height: u32) -> Result { + let desc = D3D11_TEXTURE2D_DESC { + Width: width, + Height: height, + MipLevels: 1, + ArraySize: 1, + // Le format du RT composé : la copie est un `CopyResource`, sans conversion. + Format: DXGI_FORMAT_R8G8B8A8_UNORM, + SampleDesc: DXGI_SAMPLE_DESC { Count: 1, Quality: 0 }, + Usage: D3D11_USAGE_DEFAULT, + BindFlags: (D3D11_BIND_RENDER_TARGET.0 | D3D11_BIND_SHADER_RESOURCE.0) as u32, + CPUAccessFlags: 0, + // Handle NT, le seul qu'Electron importe, sans keyed mutex : `write` attend la fin + // de la copie avant de le livrer. + MiscFlags: (D3D11_RESOURCE_MISC_SHARED.0 | D3D11_RESOURCE_MISC_SHARED_NTHANDLE.0) as u32, + }; + let mut tex = None; + device.CreateTexture2D(&desc, None, Some(&mut tex))?; + let tex = tex.expect("CreateTexture2D a réussi sans texture"); + let resource: IDXGIResource1 = tex.cast()?; + let handle = resource.CreateSharedHandle( + None, + DXGI_SHARED_RESOURCE_READ.0 | DXGI_SHARED_RESOURCE_WRITE.0, + PCWSTR::null(), + )?; + Ok(SlotTexture { tex, handle, width, height }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn frame(gen: u64, slot: u32) -> SharedFrame { + SharedFrame { + gen, + slot, + handle: 0, + width: 2, + height: 2, + footage: None, + position: FramePosition { clip_index: 0, source_time_sec: 0.0 }, + } + } + + /// Toutes les cases prises par JS, la case `n` portant la génération `n + 1`. + fn all_held(now: Instant) -> SlotBook { + let mut book = SlotBook::default(); + for slot in 0..RING_SLOTS { + book.publish(frame(u64::from(slot) + 1, slot)); + assert!(book.take(u64::from(slot), now).is_some()); + } + book + } + + #[test] + fn claims_a_free_slot_before_a_ready_one() { + let now = Instant::now(); + let mut book = SlotBook::default(); + book.publish(frame(1, 0)); + assert_eq!(book.claim(RING_SLOTS), Some(1)); + // La frame prête n'a pas bougé : JS peut toujours la prendre. + assert_eq!(book.take(0, now), Some(frame(1, 0))); + } + + #[test] + fn a_taken_slot_is_never_written_until_released() { + let now = Instant::now(); + let mut book = all_held(now); + assert_eq!(book.claim(RING_SLOTS), None, "Chromium tient toutes les cases"); + book.release(2, 3); + assert_eq!(book.claim(RING_SLOTS), Some(2)); + } + + #[test] + fn a_frame_nobody_took_gives_its_slot_back_when_nothing_else_is_free() { + let now = Instant::now(); + let mut book = SlotBook::default(); + for slot in 0..RING_SLOTS - 1 { + book.publish(frame(u64::from(slot) + 1, slot)); + book.take(u64::from(slot), now); + } + book.publish(frame(9, 3)); + assert_eq!(book.claim(RING_SLOTS), Some(3)); + // Retirée : la livrer maintenant, pendant qu'on la réécrit, donnerait une image déchirée. + assert_eq!(book.take(0, now), None); + } + + #[test] + fn take_hands_out_only_newer_generations() { + let now = Instant::now(); + let mut book = SlotBook::default(); + book.publish(frame(5, 1)); + assert_eq!(book.take(5, now), None); + assert_eq!(book.take(4, now), Some(frame(5, 1))); + assert_eq!(book.take(0, now), None, "déjà prise"); + } + + #[test] + fn a_release_only_frees_the_generation_it_names() { + let now = Instant::now(); + let mut book = SlotBook::default(); + book.publish(frame(1, 0)); + book.take(0, now); + book.release(3, 1); + book.release(0, 7); + assert_eq!(book.claim(1), None, "ni la case 3 ni la génération 7 ne sont tenues"); + book.release(0, 1); + assert_eq!(book.claim(1), Some(0)); + } + + #[test] + fn a_slot_whose_release_never_came_is_reported_lost_but_never_rewritten() { + let start = Instant::now(); + let mut book = all_held(start); + assert!(!book.lost(start + LOST_SLOT_AFTER / 2)); + assert!(book.lost(start + LOST_SLOT_AFTER)); + // Chromium peut encore lire une case perdue : la réécrire déchirerait son image. + assert_eq!(book.claim(RING_SLOTS), None); + // Si sa libération finit par venir, la case sert de nouveau. + book.release(0, 1); + assert_eq!(book.claim(RING_SLOTS), Some(0)); + } +} diff --git a/crates/compositor/src/text_anim.rs b/crates/compositor/src/text_anim.rs index d6d3bade13..ae3887f98c 100644 --- a/crates/compositor/src/text_anim.rs +++ b/crates/compositor/src/text_anim.rs @@ -4,7 +4,8 @@ //! amplitudes. Les sept animations étaient déjà nommées dans le schéma, traduites dans les treize //! langues et transportées jusqu'ici par la scène — mais rien ne les jouait. Reprendre les //! constantes du TS plutôt que d'en réinventer garantit qu'un projet fait à l'époque de l'aperçu -//! DOM s'anime toujours pareil. +//! DOM s'anime toujours pareil. Exception : l'étiquette de la vue bureau (`deskCover`), que seul +//! le compositeur joue et qui n'a pas de courbe propre (cf. `annotation_text_state`). /// Les décalages ci-dessous sont exprimés en px À CETTE HAUTEUR : l'appelant les met à l'échelle /// de la sortie, exactement comme la taille de police (cf. `annotationScale.ts`). En pixels @@ -17,6 +18,10 @@ pub const TEXT_ANIMATION_DURATION_MS: f32 = 700.0; /// fin de l'annotation est une coupe sèche alors que son arrivée est animée. pub const TEXT_EXIT_DURATION_MS: f32 = 300.0; +/// The desk-view label's animation. The app generates it for a turned Full Camera section and +/// never stores it in a document, so it exists only here, not in `annotationTextAnimation.ts`. +pub const DESK_COVER_ANIMATION: &str = "deskCover"; + #[derive(Debug, Clone, Copy, PartialEq)] pub struct TextAnimationState { pub opacity: f32, @@ -37,6 +42,11 @@ fn clamp01(v: f32) -> f32 { v.clamp(0.0, 1.0) } +fn smoothstep01(v: f32) -> f32 { + let t = clamp01(v); + t * t * (3.0 - 2.0 * t) +} + fn ease_out_cubic(v: f32) -> f32 { let t = clamp01(v); 1.0 - (1.0 - t).powi(3) @@ -106,6 +116,30 @@ pub fn text_animation_state( } } +/// `true` for the desk-view label, whose opacity is the camera cover (`annotation_text_state`). +pub fn is_desk_cover(animation: Option<&str>) -> bool { + animation == Some(DESK_COVER_ANIMATION) +} + +/// What the compositors draw a text annotation with. Ordinary animations run on +/// `text_animation_state` (source time, unchanged). The desk-view label has no timing of its +/// own: its opacity IS `cover`, the webcam's cover strength this frame +/// (`camera_fullscreen_cover_at`, already on the screen clock). A second fade derived from the +/// label's own window drifted from the cover on short sections and inside speed regions; one +/// value read twice cannot. The app spans the label over its whole section, so the cover alone +/// decides when it shows. +pub fn annotation_text_state( + animation: Option<&str>, + elapsed_ms: f32, + region_ms: f32, + cover: f32, +) -> TextAnimationState { + if is_desk_cover(animation) { + return TextAnimationState { opacity: clamp01(cover), ..TextAnimationState::IDLE }; + } + text_animation_state(animation, elapsed_ms, region_ms) +} + #[cfg(test)] mod tests { use super::*; @@ -245,4 +279,28 @@ mod tests { fn an_empty_region_does_not_divide_by_zero() { assert!(text_animation_state(Some("fade"), 0.0, 0.0).opacity.is_finite()); } + + #[test] + fn the_desk_label_takes_the_cover_and_nothing_else() { + // Whatever the label's own window says, only the cover counts; the motion fields stay idle. + for cover in [0.0, 0.21, 0.45, 1.0] { + for (elapsed, region) in [(0.0, 2600.0), (900.0, 1150.0), (5000.0, LONG)] { + let s = annotation_text_state(Some(DESK_COVER_ANIMATION), elapsed, region, cover); + assert_eq!(s, TextAnimationState { opacity: cover, ..TextAnimationState::IDLE }); + } + } + } + + #[test] + fn ordinary_animations_ignore_the_cover() { + for name in [None, Some("fade"), Some("rise"), Some("pop"), Some("typewriter"), Some("pulse")] { + for at in [0.0, 200.0, 650.0, 9_900.0] { + assert_eq!( + annotation_text_state(name, at, LONG, 0.37), + text_animation_state(name, at, LONG), + "{name:?} at {at}" + ); + } + } + } } diff --git a/crates/compositor/src/timeline_walk.rs b/crates/compositor/src/timeline_walk.rs index a16271c8b0..5a1205ae01 100644 --- a/crates/compositor/src/timeline_walk.rs +++ b/crates/compositor/src/timeline_walk.rs @@ -17,13 +17,17 @@ use crate::compositor::Compositor; use crate::config::Cfg; use crate::cursor::CursorTrack; use crate::d3d::Gpu; +use crate::extra_cameras::{ + camera_source_time, cameras_in_regions, extra_camera_active, extra_camera_keys, open_once, + step_extra_camera, +}; use crate::ffi::AVFrame; use crate::frame_geometry::webcam_is_real; use crate::pipeline::{ClipSource, Decoder}; use crate::regions::{speed_segments_for_window, SpeedSegment}; -use crate::scene::Scene; +use crate::scene::{Scene, SceneCameraLayoutRegion}; use anyhow::Result; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; /// Ce que le décodeur sait de la PROCHAINE frame, sans l'adopter. /// @@ -142,9 +146,24 @@ pub(crate) unsafe fn advance_decoder_to( /// N" defined exactly once: a GIF driven by its own loop is how the slow-motion /// truncation bug happened. /// +/// The extra cameras (k >= 1) a clip's layout regions show are opened once per path into +/// `extra_decs`, decoded only near those regions, and handed to the compositor with +/// `set_extra_camera_frames`. Unlike camera 0 they never shorten a clip, and a file that will +/// not open (or stops decoding) is skipped with a warning for the rest of the export. +/// /// `on_frame` runs after `compose_frame` with the running output index; /// `on_clip_end` runs once per clip with its clamped source window, the frames /// it produced, and the speed segments used (MP4 needs those for audio). +/// Runs its closure when dropped, so on every exit path of a scope, early `?` returns +/// included. +struct OnExit(F); + +impl Drop for OnExit { + fn drop(&mut self) { + (self.0)(); + } +} + #[allow(clippy::too_many_arguments)] pub(crate) unsafe fn walk_composited_timeline( clips: &[ClipSource], @@ -155,6 +174,7 @@ pub(crate) unsafe fn walk_composited_timeline( scene: &Option, screen_decs: &mut HashMap, webcam_decs: &mut HashMap, + extra_decs: &mut HashMap, on_frame: &mut dyn FnMut(u64) -> Result<()>, on_clip_end: &mut dyn FnMut(usize, f64, u64, &[SpeedSegment]) -> Result<()>, ) -> Result { @@ -168,6 +188,14 @@ pub(crate) unsafe fn walk_composited_timeline( let mut cursor_active_path: Option = None; let mut frames: u64 = 0; + let mut unreadable_extras: HashSet = HashSet::new(); + // The extra decoders outlive this call in the caller's map, but the compositor must not + // keep pointers into their frames once the walk is over — also when it fails half-way + // (a decode or encode error returns early through `?`). + let _forget_extra_frames = OnExit(|| { + // SAFETY: an empty list only resets the compositor's slots; no frame is read. + unsafe { comp.set_extra_camera_frames(&[]) } + }); // L'export doit être reproductible : deux rendus du même projet, les mêmes pixels. Cette // boucle avance aussi vite que la machine décode, sans rapport avec le temps réel, alors que @@ -230,17 +258,12 @@ pub(crate) unsafe fn walk_composited_timeline( clip.source_end_sec, ); } - // Les bornes de clip sont en temps écran. La disponibilité webcam est donc translatée - // par le même offset que le seek (`webcam_time = screen_time - offset`). - let webcam_available_screen_end = - webcam_available_duration.map(|duration| duration + clip.webcam_offset_sec); - let mut source_end_sec = clip.source_end_sec; - if let Some(duration) = screen_available_duration { - source_end_sec = source_end_sec.min(duration); - } - if let Some(duration) = webcam_available_screen_end { - source_end_sec = source_end_sec.min(duration); - } + let source_end_sec = available_clip_end( + clip.source_end_sec, + screen_available_duration, + webcam_available_duration, + clip.webcam_offset_sec, + ); if source_end_sec + 1e-6 < clip.source_end_sec { eprintln!( "[pipeline] warning: clip #{} raccourci de {:.3}s (fin demandée {:.3}s, fin disponible {:.3}s; screen=\"{}\", webcam=\"{}\")", @@ -268,6 +291,27 @@ pub(crate) unsafe fn walk_composited_timeline( source_end_sec, out_fps as f64, ); + // Only the extra cameras this clip's own regions show, and only those it has a file + // for. `None` slots stay null in `set_extra_camera_frames`. + let camera_regions = clip_scene + .as_ref() + .map(|s| s.camera_layout_regions.clone()) + .unwrap_or_default(); + let mut extras: Vec> = + extra_camera_keys(&cameras_in_regions(&camera_regions), &clip.additional_cameras) + .into_iter() + .map(|key| { + let key = key?; + open_once(extra_decs, &mut unreadable_extras, &key.path, |path| { + Decoder::open_for_export(path, gpu) + }) + .then_some(ExportExtraCamera { + path: key.path, + offset_sec: key.offset_sec, + ended_at: None, + }) + }) + .collect(); if clip_scene.is_some() { comp.set_scene(clip_scene); } @@ -328,6 +372,24 @@ pub(crate) unsafe fn walk_composited_timeline( break 'clip_frames; } + let (extra_frames, closed) = { + let _p = crate::export_probe::scope(crate::export_probe::Stage::DecodeWebcam); + step_extra_cameras( + &mut extras, + extra_decs, + &mut unreadable_extras, + &camera_regions, + target_source_time, + ) + }; + if closed { + // A dropped decoder's textures may still sit in the SRV cache; a later + // decoder could reuse their addresses (same contract as the preview's + // `hand_extra_frames`). + comp.clear_srv_cache(); + } + comp.set_extra_camera_frames(&extra_frames); + comp.set_timeline_time(Some(target_source_time as f32)); // Le temps de SORTIE, lui, ne saute ni aux coupes ni aux clips. Même // arithmétique (f64 puis f32) que `ProgrammeClock::at` côté preview. @@ -361,9 +423,104 @@ pub(crate) unsafe fn walk_composited_timeline( Ok(frames) } +/// The screen-time end of a clip: the requested end, clamped to what the screen and camera 0 +/// can deliver. Clip bounds are in screen time, so camera 0's duration is moved by the same +/// offset as its seek (`webcam_time = screen_time - offset`). The extra cameras do not enter +/// here: one that ends early only loses its layer, it never shortens the clip. +fn available_clip_end( + requested_end_sec: f64, + screen_duration: Option, + webcam_duration: Option, + webcam_offset_sec: f64, +) -> f64 { + let mut end = requested_end_sec; + if let Some(duration) = screen_duration { + end = end.min(duration); + } + if let Some(duration) = webcam_duration { + end = end.min(duration + webcam_offset_sec); + } + end +} + +/// An extra camera of the clip being exported; its decoder lives in `extra_decs` under `path`. +struct ExportExtraCamera { + path: String, + offset_sec: f64, + /// See `step_extra_camera`. + ended_at: Option, +} + +/// The frames for `set_extra_camera_frames` at screen source time `t`: each camera that is +/// near one of its regions steps toward its own source time, the others give null. A camera +/// that stops decoding is dropped with a warning and not reopened for the rest of the export; +/// the `bool` says one was dropped, so the caller clears the compositor's texture cache. +unsafe fn step_extra_cameras( + extras: &mut [Option], + extra_decs: &mut HashMap, + unreadable: &mut HashSet, + regions: &[SceneCameraLayoutRegion], + t: f64, +) -> (Vec<*const AVFrame>, bool) { + let mut closed = false; + let mut frames = Vec::with_capacity(extras.len()); + for (k, slot) in extras.iter_mut().enumerate() { + let camera = k + 1; + let mut frame: *const AVFrame = std::ptr::null(); + if let Some(cam) = slot.as_mut().filter(|_| extra_camera_active(regions, camera, t)) { + if let Some(dec) = extra_decs.get_mut(&cam.path) { + let target = camera_source_time(t, cam.offset_sec); + match step_extra_camera(dec, &mut cam.ended_at, target) { + Ok(f) => frame = f, + Err(e) => { + eprintln!( + "WARNING: extra camera {camera} stopped decoding ({}): {e:#}. Its layer will not be drawn.", + cam.path + ); + extra_decs.remove(&cam.path); + unreadable.insert(cam.path.clone()); + *slot = None; + closed = true; + } + } + } + } + frames.push(frame); + } + (frames, closed) +} + #[cfg(test)] mod tests { - use super::{frame_step, FrameStep, NextFrameTime}; + use super::{available_clip_end, frame_step, FrameStep, NextFrameTime, OnExit}; + + /// The guard that forgets the extra camera frames runs on an early `?` return as well as + /// at the end of the walk. + #[test] + fn the_exit_guard_runs_on_every_exit_path() { + fn walk(fail: bool, ran: &std::cell::Cell) -> anyhow::Result<()> { + let _guard = OnExit(|| ran.set(ran.get() + 1)); + if fail { + Err(anyhow::anyhow!("decode error"))?; + } + Ok(()) + } + let ran = std::cell::Cell::new(0); + assert!(walk(true, &ran).is_err()); + assert_eq!(ran.get(), 1, "early return"); + assert!(walk(false, &ran).is_ok()); + assert_eq!(ran.get(), 2, "normal end"); + } + + #[test] + fn a_short_extra_camera_does_not_shorten_the_clip() { + // Screen 60 s, camera 0 50 s at offset 2 s: the clip ends where camera 0 does. + assert_eq!(available_clip_end(60.0, Some(60.0), Some(50.0), 2.0), 52.0); + // A 10 s extra camera in the same clip changes nothing: only the screen and camera 0 + // bound the clip, the extra camera's layer just disappears past its end. + assert_eq!(available_clip_end(30.0, Some(60.0), Some(50.0), 2.0), 30.0); + assert_eq!(available_clip_end(30.0, None, None, 0.0), 30.0); + } /// La cadence de lecture, en une phrase : à 24 fps, une seconde réelle doit adopter 24 /// frames et pas une de plus. Le bug d'origine (un pas fixe de 1/60 s, une frame par diff --git a/crates/compositor/src/vk_shaders/layer.wgsl b/crates/compositor/src/vk_shaders/layer.wgsl index 2eb46a9b30..fdb9463269 100644 --- a/crates/compositor/src/vk_shaders/layer.wgsl +++ b/crates/compositor/src/vk_shaders/layer.wgsl @@ -16,6 +16,15 @@ // // Le rendu est en alpha PRÉMULTIPLIÉ (cf. commentaire HLSL), convention qu'on // retrouve dans tous les autres modes du compositeur (texte, curseur, ombre). +// +// `LAYER_MODELS` n'est PAS declare ici : `compositor_linux.rs` (`layer_source`) le prefixe et +// compile ce source deux fois. Un shader unique alloue a CHAQUE calque les registres de sa +// branche la plus lourde, et ce sont les modeles 3D ray-traces (modes 15 a 17) : 168 VGPR sous +// ACO (Radeon 610M), donc 6 vagues par SIMD au lieu de 32 pour le fond, l'ombre et la video, qui +// couvrent toute la sortie : l'export Linux du banc etait passe de 31 a 57 s. Sans ces trois modes, +// 48 VGPR et 20 vagues. Le pipeline plat (`false`) dessine donc tout le reste, et seuls le +// curseur modelise, l'impact de son clic, l'appareil et son ombre passent par l'autre : c'est +// `LayerCB::needs_models` qui en decide, au draw (`draw_layer`), d'apres le mode du calque. struct Layer { dst: vec4, // x,y,w,h sortie 0..1 (origine haut-gauche) @@ -31,19 +40,27 @@ struct Layer { trail_a: vec4, // mode 8 : coins TL, TR du plan a la frame precedente (px locaux, comme fx) ; mode 18 incline : en fractions de sortie trail_b: vec4, // mode 8 : coins BR, BL du plan a la frame precedente (comme src_prev) ; mode 18 incline : en fractions de sortie trail_mb: vec4, // mode 8 : x = taps, y = force du flou de mouvement (ceux du mode 0) ; mode 18 incline : le `mb` du mode 8 (profondeur de champ), et `color.xy` sa lampe ; 0 ailleurs + cover: vec4, // x = desk-view cover 0..1, y = blur radius (quad px), z = dim, w unused + persp: array, 3>, // mode 0: rows of the homography quad point (0..1 in dst) -> camera uv (0..1 of the valid frame), xyz; read when layer_fx.y = 1 + layer_fx: vec4, // x = transparency 0..1 (0 = opaque), every mode; y = 1 when persp applies; z, w unused } @group(0) @binding(0) var layer: Layer; -@group(0) @binding(1) var texY: texture_2d; // R8Unorm, sample .r ; modes 7, 13 et 15 : le sprite RGBA -@group(0) @binding(2) var texU: texture_2d; // R8Unorm, sample .r ; mode 15 : le champ R16F du sprite +@group(0) @binding(1) var texY: texture_2d; // R8Unorm, sample .r ; modes 7 et 13 : le sprite RGBA ; mode 15 : la copie de l'image composee +@group(0) @binding(2) var texU: texture_2d; // R8Unorm, sample .r @group(0) @binding(3) var samp: sampler; // Masque de segmentation du sujet webcam, R8. Une vue 1x1 est liee quand aucun masque // n'existe : la branche n'est de toute facon prise que si layer.fx.z > 0.5. -// Mode 18 : ce binding porte a la place le rendu isole de l'ecran cadre. +// Mode 18 : ce binding porte a la place le rendu isole de l'ecran cadre ; mode 15, le champ R16F +// du sprite. @group(0) @binding(4) var texMask: texture_2d; // Pyramide RGBA de profondeur de champ (`tilted_sample`), lue par le mode 8 et par le repli du -// mode 18, qui garde le binding 4 pour son rendu isole : d'ou un binding a elle. Vue 1x1 ailleurs. +// mode 18, qui garde le binding 4 pour son rendu isole : d'ou un binding a elle. Mode 15 : le +// sprite RGBA. Vue 1x1 ailleurs. @group(0) @binding(6) var texDof: texture_2d; +// Le maillage de Prism Glow (mode 15), rempli de `prism_mesh.rs` : un uniform et non des tables +// dans le shader, que lavapipe recopierait a chaque pixel de chaque calque (cf. `PrismMesh`). +@group(0) @binding(7) var prism: PrismMesh; // Plafond de la profondeur de champ du mode 8, en niveau de la pyramide demi-resolution. // Meme valeur que `DOF_MAX_LOD` du HLSL. @@ -170,8 +187,12 @@ fn screen_trail(pout: vec2) -> vec4 { // garde sa largeur tout autour du coin. r <= 0 : le rectangle vif d'avant, à l'identique. fn sd_round_rect(p: vec2, halfsz: vec2, r: f32) -> f32 { let hmin = min(halfsz.x, halfsz.y); - if (r <= 0.0 || hmin <= 0.0) { - let q0 = abs(p) - halfsz; + let q0 = abs(p) - halfsz; + // Hors des coins, la distance est celle du rectangle vif quel que soit l'exposant : un seul + // axe deborde de l'etendue E, qui s'y ajoute puis s'en retranche. E ne depasse pas 1,42 r + // (n = 3) : sous 1,5 r du bord sur un axe, on rend la boite sans calculer `n` ni E, deux + // transcendantes de moins sur presque tous les pixels d'un calque arrondi. + if (r <= 0.0 || hmin <= 0.0 || min(q0.x, q0.y) <= -min(1.5 * r, hmin)) { return length(max(q0, vec2(0.0))) + min(max(q0.x, q0.y), 0.0); } let n = 3.0 - smoothstep(0.5, 1.0, r / hmin); @@ -542,9 +563,13 @@ const VOGEL_TAPS = array, 21>( vec3(-0.633036, -0.758588, 0.087119) ); -fn blur_webcam_bg(uv: vec2, intensity: f32, qpx: vec2, local_px: vec2) -> vec3 { - let max_r_px = max(intensity, 0.0) * 22.0 + 1.5; +// Vogel-disc blur of the camera texture at a radius in quad pixels. `valid` is the part of the +// texture the picture fills (fx.xy): decoders allocate aligned textures (a 1080-line camera in a +// 1088-line texture), so each tap is clamped half a chroma texel inside it, never into padding. +fn blur_webcam_radius(uv: vec2, max_r_px: f32, qpx: vec2, local_px: vec2, valid: vec2) -> vec3 { let step = max_r_px / max(qpx, vec2(1.0)); + let chroma = max(vec2(textureDimensions(texU)), vec2(1.0)); + let hi = max(valid - 0.5 / chroma, vec2(0.0)); let noise = fract(52.9829189 * fract(0.06711056 * local_px.x + 0.00583715 * local_px.y)); let angle = noise * 6.2831853; let s = sin(angle); @@ -555,12 +580,28 @@ fn blur_webcam_bg(uv: vec2, intensity: f32, qpx: vec2, local_px: vec2< let p = VOGEL_TAPS[k].xy; let w = VOGEL_TAPS[k].z; let rot_p = vec2(p.x * c - p.y * s, p.x * s + p.y * c); - sum = sum + sample_yuv(clamp(uv + rot_p * step, vec2(0.0), vec2(1.0))) * w; + sum = sum + sample_yuv(clamp(uv + rot_p * step, vec2(0.0), hi)) * w; total = total + w; } return sum / max(total, 1e-4); } +fn blur_webcam_bg(uv: vec2, intensity: f32, qpx: vec2, local_px: vec2, valid: vec2) -> vec3 { + return blur_webcam_radius(uv, max(intensity, 0.0) * 22.0 + 1.5, qpx, local_px, valid); +} + +// Camera homography (`persp`, read when `layer_fx.y` = 1): the camera point (0..1 of its valid +// frame) seen at `local` (0..1 in the layer's dst quad), and in z whether there is one -- 0 when +// the point is behind the projection (q.z <= 0) or outside the camera frame, where the layer is +// black. Mirror of the HLSL and MSL `persp_camera`. +fn persp_camera(local: vec2) -> vec3 { + let p = vec3(local, 1.0); + let q = vec3(dot(layer.persp[0].xyz, p), dot(layer.persp[1].xyz, p), dot(layer.persp[2].xyz, p)); + let cam = q.xy / max(q.z, 1e-6); + let inside = q.z > 0.0 && all(cam >= vec2(0.0)) && all(cam <= vec2(1.0)); + return vec3(cam, select(0.0, 1.0, inside)); +} + // ---- Curseur MODELISE (mode 15) ---- // Port ligne pour ligne de `cursor_model` (HLSL), dont les commentaires font foi : un curseur // SCULPTE (`trail_a.x` > 0, la fleche ou la main d'un des cinq themes d'origine modelee en @@ -568,9 +609,10 @@ fn blur_webcam_bg(uv: vec2, intensity: f32, qpx: vec2, local_px: vec2< // propres et occlusion, et qui porte une ombre douce et une ombre de contact sur le plan de // l'ecran. Constantes : miroir exact de `frame_geometry.rs` (MODEL_*) et de `sculpt.rs` // (SCULPT_*), emplacements du cbuffer : `cursor_model_cb`. -// Textures : le sprite RGBA (alpha droit) au binding 1 (`texY`, comme aux modes 7 et 13), son -// champ R16F au binding 2 (`texU`), sur le meme rect ; `color.rg` = coin du sprite dans le repere -// du modele, `sprite_size()` = sa taille (w/h dans `radius_px`), `color.b` = l'ecrasement au clic. +// Textures : la copie de l'image composee au binding 1 (`texY`), que refracte le cristal de Prism +// Glow ; le sprite RGBA (alpha droit) est au binding 6 (`texDof`), son champ R16F au +// binding 4 (`texMask`), sur le meme rect ; `color.rg` = coin du sprite dans le repere du modele, +// `sprite_size()` = sa taille (w/h dans `radius_px`), `color.b` = l'ecrasement au clic. const MODEL_BEVEL: f32 = 0.045; const MODEL_LIGHT = vec3(-0.4194, -0.5792, 0.6990); const MODEL_FILL = vec3(0.7557, 0.2519, 0.6046); @@ -592,7 +634,7 @@ fn sprite_size() -> vec2 { // Un texel du sprite, en unites du modele : le champ est le sprite surechantillonne x4. const CURSOR_SDF_UPSAMPLE: f32 = 4.0; fn sprite_texel() -> f32 { - let d = vec2(textureDimensions(texU)); + let d = vec2(textureDimensions(texMask)); return CURSOR_SDF_UPSAMPLE / max(d.x, d.y); } @@ -610,8 +652,6 @@ fn sculpt_id() -> i32 { const SCULPT_SCALE: f32 = 0.85; const SCULPT_HOVER: f32 = 0.05; const SCULPT_VOX: f32 = 0.0625; -const SCULPT_AR_ROUND: f32 = 0.03; -const SCULPT_HAND_ZC: f32 = 0.185; const SCULPT_ZREF_ARROW: f32 = 0.2; const SCULPT_ZREF_HAND: f32 = 0.185; const SCULPT_LAMP_DIST: f32 = 1.9; @@ -630,13 +670,6 @@ fn s_opu(a: vec2, b: vec2) -> vec2 { return select(b, a, a.x < b.x); } -fn s_capsule(p: vec3, a: vec3, b: vec3, r: f32) -> f32 { - let pa = p - a; - let ba = b - a; - let h = saturate(dot(pa, ba) / dot(ba, ba)); - return length(pa - ba * h) - r; -} - fn s_round_box(p: vec3, b: vec3, r: f32) -> f32 { let q = abs(p) - b + vec3(r); return length(max(q, vec3(0.0))) + min(max(q.x, max(q.y, q.z)), 0.0) - r; @@ -660,31 +693,6 @@ fn s_rot(v: vec2, a: f32) -> vec2 { return vec2(c * v.x - s * v.y, s * v.x + c * v.y); } -const SCULPT_ARROW = array, 7>( - vec2(0.0, 0.0), vec2(0.0, -0.86), vec2(0.215, -0.665), vec2(0.37, -1.0), - vec2(0.53, -0.93), vec2(0.38, -0.60), vec2(0.64, -0.60) -); - -fn s_arrow2(p: vec2) -> f32 { - var d = dot(p - SCULPT_ARROW[0], p - SCULPT_ARROW[0]); - var s = 1.0; - var j = 6; - for (var i = 0; i < 7; i++) { - let e = SCULPT_ARROW[j] - SCULPT_ARROW[i]; - let w = p - SCULPT_ARROW[i]; - let b = w - e * saturate(dot(w, e) / dot(e, e)); - d = min(d, dot(b, b)); - let c0 = p.y >= SCULPT_ARROW[i].y; - let c1 = p.y < SCULPT_ARROW[j].y; - let c2 = e.x * w.y > e.y * w.x; - if (c0 && c1 && c2) || (!c0 && !c1 && !c2) { - s = -s; - } - j = i; - } - return s * sqrt(d); -} - fn s_star5(p0: vec2, r: f32, rf: f32) -> f32 { let k1 = vec2(0.809016994375, -0.587785252292); let k2 = vec2(-0.809016994375, -0.587785252292); @@ -698,233 +706,426 @@ fn s_star5(p0: vec2, r: f32, rf: f32) -> f32 { return length(p - ba * h) * sign(p.y * ba.x - p.x * ba.y); } -fn s_arrow_solid(p: vec3, h: f32, re: f32, dome: f32) -> f32 { - let d2 = s_arrow2(p.xy) - SCULPT_AR_ROUND; - let hh = h + dome * smoothstep(0.0, 0.07, -d2); - return s_extrude(d2, p.z - (SCULPT_HOVER + h + dome), hh, re); +fn s_vesica(p0: vec2, r: f32, d: f32) -> f32 { + let p = abs(p0); + let b = sqrt(r * r - d * d); + return select(length(p + vec2(d, 0.0)) - r, length(p - vec2(0.0, b)), (p.y - b) * d > p.x * b); } -fn s_piping(p: vec3, ztop: f32) -> f32 { - let d2 = s_arrow2(p.xy) - SCULPT_AR_ROUND; - return length(vec2(d2 + 0.075, p.z - ztop)) - 0.017; -} +// Curseurs cercles (cf. HLSL) : Studio Ink, Pop Coral, Star Sprout. +const RIM_POLY = array, 28>( + vec2(0.0, -0.06), vec2(0.0, -0.7712), vec2(0.165, -0.6325), vec2(0.313, -0.9318), + vec2(0.4234, -0.8557), vec2(0.2672, -0.5686), vec2(0.456, -0.5393), + vec2(-0.173, -0.5237), vec2(-0.016, -0.75), vec2(0.329, -0.75), vec2(0.411, -0.592), + vec2(0.411, -0.43), vec2(0.19, -0.435), vec2(-0.03, -0.44), + vec2(-0.1825, -0.6215), vec2(0.0037, -0.9195), vec2(0.3462, -0.9195), vec2(0.467, -0.6906), + vec2(0.467, -0.49), vec2(0.2, -0.49), vec2(-0.03, -0.49), + vec2(-0.1807, -0.5534), vec2(0.0194, -0.872), vec2(0.3485, -0.872), vec2(0.47, -0.6795), + vec2(0.47, -0.49), vec2(0.2, -0.49), vec2(-0.03, -0.49) +); -fn s_glove(p: vec3, zc: f32) -> f32 { - let index = s_capsule(p, vec3(0.0, -0.10, zc), vec3(0.0, -0.52, zc), 0.098); - let palm = s_round_box(p - vec3(0.185, -0.70, zc), vec3(0.255, 0.19, 0.105), 0.1); - let f1 = s_capsule(p, vec3(0.17, -0.57, zc + 0.012), vec3(0.17, -0.41, zc + 0.045), 0.086); - let f2 = s_capsule(p, vec3(0.31, -0.59, zc + 0.01), vec3(0.31, -0.45, zc + 0.04), 0.08); - let f3 = s_capsule(p, vec3(0.435, -0.625, zc + 0.005), vec3(0.435, -0.52, zc + 0.03), 0.07); - let thumb = s_capsule(p, vec3(0.03, -0.77, zc + 0.03), vec3(-0.165, -0.60, zc + 0.065), 0.082); - var d = s_smin(palm, min(f1, min(f2, f3)), 0.035); - d = s_smin(d, index, 0.05); - return s_smin(d, thumb, 0.05); -} +const RIM_GLOVE = array, 27>( + vec4(0.0, -0.1077, 0.0, -0.6), vec4(0.1493, -0.3279, 0.1493, -0.6), + vec4(0.2863, -0.3654, 0.2863, -0.6), vec4(0.4193, -0.4043, 0.4193, -0.6), + vec4(-0.2, -0.478, -0.075, -0.625), vec4(0.0788, -0.25, 0.0788, -0.3992), + vec4(0.2188, -0.3, 0.2188, -0.4234), vec4(0.3552, -0.33, 0.3552, -0.4534), + vec4(-0.0845, -0.5386, 0.762, 0.648), + vec4(0.0, -0.1186, 0.0, -0.65), vec4(0.1614, -0.3838, 0.1614, -0.65), + vec4(0.3177, -0.427, 0.3177, -0.65), vec4(0.4645, -0.4848, 0.4645, -0.65), + vec4(-0.215, -0.535, -0.1208, -0.6864), vec4(0.0832, -0.3, 0.0832, -0.48), + vec4(0.2414, -0.35, 0.2414, -0.482), vec4(0.3959, -0.41, 0.3959, -0.533), + vec4(-0.122, -0.5718, 0.867, 0.498), + vec4(0.0, -0.122, 0.0, -0.65), vec4(0.172, -0.387, 0.172, -0.65), + vec4(0.3267, -0.427, 0.3267, -0.65), vec4(0.4727, -0.4867, 0.4727, -0.65), + vec4(-0.199, -0.483, -0.102, -0.645), vec4(0.0927, -0.3, 0.0927, -0.476), + vec4(0.2493, -0.35, 0.2493, -0.478), vec4(0.402, -0.41, 0.402, -0.52), + vec4(-0.0993, -0.5267, 0.858, 0.514) +); -fn s_cuff(p: vec3, zc: f32) -> f32 { - let q = p - vec3(0.185, -0.925, zc); - let ab = vec2(0.272, 0.12); - let e = (length(q.xz / ab) - 1.0) * min(ab.x, ab.y); - return s_extrude(e, q.y, 0.07, 0.06); -} +const RIM_GLOVE_R = array, 9>( + vec4(0.0552, 0.047, 0.045, 0.0375), vec4(0.052, 0.0235, 0.0225, 0.0225), vec4(0.0552, -0.548, 0.0525, 0.0), + vec4(0.064, 0.0585, 0.0585, 0.0515), vec4(0.06, 0.0197, 0.0178, 0.0172), vec4(0.0648, -0.6, 0.058, 0.0), + vec4(0.066, 0.0553, 0.0553, 0.0507), vec4(0.063, 0.024, 0.022, 0.02), vec4(0.0647, -0.555, 0.056, 0.0) +); -fn s_sprout(p: vec3, c: vec3) -> vec2 { - let q = p - c; - let st2 = s_star5(q.xy, 0.125, 0.52) - 0.022; - var r = vec2(s_extrude(st2, q.z, 0.038, 0.034), 3.0); - let e = vec3(abs(q.x) - 0.032, q.y + 0.005, q.z - 0.036); - r = s_opu(r, vec2(length(e) - 0.0135, 5.0)); - let l1 = vec3(s_rot(q.xy - vec2(-0.04, 0.15), -0.6), q.z); - let l2 = vec3(s_rot(q.xy - vec2(0.045, 0.155), 0.7), q.z); - let leaves = min(s_ellipsoid(l1, vec3(0.03, 0.058, 0.02)), s_ellipsoid(l2, vec3(0.03, 0.058, 0.02))); - return s_opu(r, vec2(leaves, 4.0)); +fn s_rim_poly(p: vec2, base: i32) -> f32 { + var d = dot(p - RIM_POLY[base], p - RIM_POLY[base]); + var s = 1.0; + var j = base + 6; + for (var i = base; i < base + 7; i++) { + let e = RIM_POLY[j] - RIM_POLY[i]; + let w = p - RIM_POLY[i]; + let b = w - e * saturate(dot(w, e) / dot(e, e)); + d = min(d, dot(b, b)); + let c0 = p.y >= RIM_POLY[i].y; + let c1 = p.y < RIM_POLY[j].y; + let c2 = e.x * w.y > e.y * w.x; + if (c0 && c1 && c2) || (!c0 && !c1 && !c2) { + s = -s; + } + j = i; + } + return s * sqrt(d); } -// Pixel Candy : une ligne par entier, bit c = colonne c (cf. HLSL et `sculpt.rs`). -const SCULPT_ARROWPIX = array(0, 1, 3, 7, 31, 63, 127, 255, 511, 63, 55, 115, 113, 224, 224, 64); -const SCULPT_ARROWBACK = array( - 3, 7, 15, 31, 63, 127, 255, 511, 1023, 2047, 4095, 255, 511, 511, 999, 995, 960, 192 +fn s_rim_glove(p: vec2, g: i32, palm: f32) -> vec2 { + let r0 = RIM_GLOVE_R[3 * g]; + let r1 = RIM_GLOVE_R[3 * g + 1]; + let r2 = RIM_GLOVE_R[3 * g + 2]; + let i = 9 * g; + var f = min(sd_segment(p, RIM_GLOVE[i].xy, RIM_GLOVE[i].zw) - r0.x, + sd_segment(p, RIM_GLOVE[i + 1].xy, RIM_GLOVE[i + 1].zw) - r0.y); + f = min(f, sd_segment(p, RIM_GLOVE[i + 2].xy, RIM_GLOVE[i + 2].zw) - r0.z); + f = min(f, sd_segment(p, RIM_GLOVE[i + 3].xy, RIM_GLOVE[i + 3].zw) - r0.w); + f = min(f, sd_segment(p, RIM_GLOVE[i + 4].xy, RIM_GLOVE[i + 4].zw) - r1.x); + var grooves = min(sd_segment(p, RIM_GLOVE[i + 5].xy, RIM_GLOVE[i + 5].zw) - r1.y, + sd_segment(p, RIM_GLOVE[i + 6].xy, RIM_GLOVE[i + 6].zw) - r1.z); + grooves = min(grooves, sd_segment(p, RIM_GLOVE[i + 7].xy, RIM_GLOVE[i + 7].zw) - r1.w); + let sil = s_smin(palm, f, 0.03); + let v = RIM_GLOVE[i + 8]; + let wedge = max(max(-dot(p - v.xy, v.zw), p.x + r2.x), r2.y - p.y); + return vec2(sil, max(sil, -min(grooves, wedge))); +} + +fn s_piece(d: f32, dc: f32, sil: f32, z: f32, w: f32, zt: f32, h: f32, bump: f32, mat: f32) -> vec2 { + let tray = s_extrude(sil - w, z - 0.5 * (SCULPT_HOVER + zt), 0.5 * (zt - SCULPT_HOVER), 0.012); + let bead = length(vec2(d - 0.5 * w, z - zt)) - 0.5 * w; + let u = saturate(-dc / 0.045); + let cushion = 0.8 * s_extrude(dc, z - zt, h + 0.022 * u * (2.0 - u) + bump, 0.75 * h); + return s_opu(vec2(min(tray, bead), 5.0), vec2(cushion, mat)); +} + +fn s_paper(d: f32, sil: f32, z: f32, w: f32) -> vec2 { + let navy = s_extrude(sil - w, z - 0.5 * (SCULPT_HOVER + 0.15), 0.5 * (0.15 - SCULPT_HOVER), 0.006); + let sheet = s_extrude(d, z - 0.16, 0.01, 0.004); + return s_opu(vec2(navy, 5.0), vec2(sheet, 1.0)); +} + +fn s_bump(p: vec2, c: vec2, r: f32) -> f32 { + let k = saturate(1.0 - dot(p - c, p - c) / (r * r)); + return k * k; +} + +fn s_dash(p: vec2, a: vec2, b0: vec2, ra: f32, rb: f32) -> f32 { + let q0 = p - a; + let b = b0 - a; + let hb = dot(b, b); + let q = vec2(abs(dot(q0, vec2(b.y, -b.x))), dot(q0, b)) / hb; + let c = vec2(sqrt(hb - (ra - rb) * (ra - rb)), ra - rb); + let k = c.x * q.y - c.y * q.x; + if k < 0.0 { + return sqrt(hb * dot(q, q)) - ra; + } + if k > c.x { + return sqrt(hb * (dot(q, q) + 1.0 - 2.0 * q.y)) - rb; + } + return dot(c, q) - ra; +} + +fn s_rimmed(p: vec3, theme: i32, shape: i32) -> vec2 { + let arrow = shape == 0; + let g = select(select(2, 1, theme == 0), 0, theme == 4); + let poly = s_rim_poly(p.xy, select(7 * g + 7, 0, arrow)); + var body = vec2(poly, poly); + var w = 0.06; + var zt = 0.17; + var h = 0.03; + var bump = 0.022 * s_bump(p.xy, vec2(0.15, -0.45), 0.3); + var back = 1e9; + if !arrow { + w = RIM_GLOVE_R[3 * g + 2].z; + for (var i = 0; i < 2; i++) { + let q = p.xy + select(vec2(0.0), vec2(0.03, 0.04), i == 1); + let v = s_rim_glove(q, g, select(poly, s_rim_poly(q, 7 * g + 7), i == 1) - 0.05); + if i == 1 { + back = v.x - w; + } else { + body = v; + if theme != 2 || v.x - w <= 0.0 { + break; + } + } + } + zt = 0.185; + h = 0.028; + bump = 0.03 * s_bump(p.xy, vec2(0.19, -0.62), 0.28); + } + var dc = body.y; + if arrow && theme == 0 { + body = body - vec2(0.015); + w = 0.045; + dc = 1.0; + } + var r: vec2; + if theme == 2 { + r = s_paper(body.y, body.x, p.z, w); + } else { + r = s_piece(body.y, dc, body.x, p.z, w, zt, h, bump, 1.0); + } + if theme == 0 { + if arrow { + let band = s_extrude(abs(poly + 0.0075) - 0.0225, p.z - zt, 0.028, 0.012); + r = s_opu(r, vec2(band, 2.0)); + } + return r; + } + if theme == 2 { + var dash: f32; + if arrow { + let back = s_rim_poly(p.xy + vec2(0.025, 0.05), 0) - 0.06; + r = s_opu(r, vec2(s_extrude(back, p.z - 0.08, 0.03, 0.006), 2.0)); + dash = min(s_dash(p.xy, vec2(-0.1179, -0.1374), vec2(-0.1799, -0.0443), 0.028, 0.045), + s_dash(p.xy, vec2(-0.1347, -0.2493), vec2(-0.248, -0.2056), 0.026, 0.042)); + } else { + r = s_opu(r, vec2(s_extrude(back, p.z - 0.08, 0.03, 0.006), 3.0)); + dash = min(s_dash(p.xy, vec2(0.216, -0.1695), vec2(0.2593, -0.0685), 0.028, 0.042), + s_dash(p.xy, vec2(0.3054, -0.2339), vec2(0.3949, -0.1652), 0.0275, 0.041)); + } + return s_opu(r, vec2(s_extrude(dash, p.z - 0.11, 0.06, 0.006), 3.0)); + } + if !arrow { + let cq = p.xy - vec2(0.1541, -0.9047); + let bq = abs(vec2(cq.x, cq.y - 0.2066 * cq.x * cq.x)) - vec2(0.214, 0.0361); + let cuff = length(max(bq, vec2(0.0))) + min(max(bq.x, bq.y), 0.0) - 0.03; + r = s_opu(r, s_piece(cuff, cuff, cuff, p.z, 0.0477, 0.225, 0.025, 0.0, 2.0)); + } + let c = select(vec2(0.15, -0.885), vec2(0.6118, -0.8079), arrow); + let rs = select(0.128, 0.1678, arrow); + let zs = select(0.285, 0.23, arrow); + let q = s_rot(p.xy - c, select(0.0, 0.2443, arrow)) / rs; + let star = (s_star5(q, 0.82, 0.55) - 0.18) * rs; + let l1 = s_rot(q - vec2(-0.33, 1.38), -0.925); + let l2 = s_rot(q - vec2(0.53, 1.37), 0.873); + let leaves = min(s_vesica(l1, 0.4296, 0.2626), s_vesica(l2, 0.4296, 0.2626)) * rs; + r = s_opu(r, s_piece(leaves, leaves, leaves, p.z, 0.038, zs - 0.02, 0.018, 0.0, 4.0)); + r = s_opu(r, s_piece(star, star, star, p.z, 0.038, zs, 0.022, 0.018 * s_bump(q, vec2(0.0), 1.0), 3.0)); + let e = vec3(abs(q.x) - 0.25, q.y - 0.08, (p.z - zs - 0.056) / rs); + return s_opu(r, vec2(s_ellipsoid(e, vec3(0.075, 0.13, 0.1)) * rs, 5.0)); +} + +// Pixel Candy : tables générées par scripts/generate-pixel-candy-voxels.mjs (cf. HLSL). +// +const PIX_BODY = array(1, 3, 7, 15, 31, 63, 127, 255, 511, 1023, 2047, 127, 247, 243, 480, 192, 48, 120, 120, 120, 504, 4088, 32760, 65534, 65535, 65535, 65534, 32766, 32764, 16380, 16376, 16376); +const PIX_LINE = array(1, 3, 5, 9, 17, 33, 65, 129, 257, 513, 1985, 73, 149, 147, 288, 192, 48, 72, 72, 72, 456, 3656, 29256, 37454, 32777, 32769, 32770, 16386, 16388, 8196, 8200, 16376); +const PIX_HI = array(0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 18, 34, 32, 64, 0, 0, 16, 16, 16, 16, 16, 16, 16, 22, 18, 4, 4, 8, 8, 16, 0); +const PIX_SHADE = array(0, 0, 0, 4, 8, 16, 32, 64, 128, 448, 40, 36, 64, 64, 128, 0, 0, 32, 32, 32, 32, 288, 2336, 18720, 16384, 16384, 16384, 8192, 8192, 4096, 8160, 0); +const PIX_RECT_N = array(0, 17, 29); +const PIX_GRID = array, 2>( + vec4(11, 16, 0, 0), + vec4(16, 16, 16, 0) ); -const SCULPT_HANDPIX = array(4, 14, 14, 14, 110, 878, 7022, 7022, 8190, 8191, 8191, 8191, 8190, 4092, 4092); -const SCULPT_HANDBACK = array( - 28, 62, 62, 62, 510, 4094, 32766, 32766, 32766, 32767, 32767, 32767, 32767, 32767, 32766, 16380, 16380 +const PIX_ORIGIN = array, 2>( + vec2(0.0, 0.0), + vec2(-0.3125, 0.0) ); -const SCULPT_HANDSPAN = array, 17>( - vec2(2.0, 4.0), vec2(1.0, 5.0), vec2(1.0, 5.0), vec2(1.0, 5.0), vec2(1.0, 8.0), - vec2(1.0, 11.0), vec2(1.0, 14.0), vec2(1.0, 14.0), vec2(1.0, 14.0), vec2(0.0, 14.0), - vec2(0.0, 14.0), vec2(0.0, 14.0), vec2(0.0, 14.0), vec2(0.0, 14.0), vec2(1.0, 14.0), - vec2(2.0, 13.0), vec2(2.0, 13.0) +const PIX_BOX = array, 2>( + vec4(0.3438, -0.5, 0.3438, 0.5), + vec4(0.1875, -0.5, 0.5, 0.5) ); +const PIX_RECT = array, 29>( + vec4(0.0313, -0.0313, 0.0313, 0.0313), + vec4(0.0625, -0.0938, 0.0625, 0.0313), + vec4(0.0938, -0.1563, 0.0938, 0.0313), + vec4(0.125, -0.2188, 0.125, 0.0313), + vec4(0.1563, -0.2813, 0.1563, 0.0313), + vec4(0.1875, -0.3438, 0.1875, 0.0313), + vec4(0.2188, -0.4063, 0.2188, 0.0313), + vec4(0.25, -0.4688, 0.25, 0.0313), + vec4(0.2813, -0.5313, 0.2813, 0.0313), + vec4(0.3125, -0.5938, 0.3125, 0.0313), + vec4(0.3438, -0.6563, 0.3438, 0.0313), + vec4(0.2188, -0.7188, 0.2188, 0.0313), + vec4(0.0938, -0.7813, 0.0938, 0.0313), + vec4(0.375, -0.8125, 0.125, 0.0625), + vec4(0.0625, -0.8438, 0.0625, 0.0313), + vec4(0.4375, -0.9063, 0.125, 0.0313), + vec4(0.4375, -0.9688, 0.0625, 0.0313), + vec4(0.0, -0.0313, 0.0625, 0.0313), + vec4(0.0, -0.1563, 0.125, 0.0938), + vec4(0.0625, -0.2813, 0.1875, 0.0313), + vec4(0.1563, -0.3438, 0.2813, 0.0313), + vec4(0.25, -0.4063, 0.375, 0.0313), + vec4(0.2188, -0.4688, 0.4688, 0.0313), + vec4(0.1875, -0.5625, 0.5, 0.0625), + vec4(0.2188, -0.6563, 0.4688, 0.0313), + vec4(0.1875, -0.7188, 0.4375, 0.0313), + vec4(0.2188, -0.7813, 0.4063, 0.0313), + vec4(0.1875, -0.8438, 0.375, 0.0313), + vec4(0.2188, -0.9375, 0.3438, 0.0625) +); +// -fn s_grid_origin(shape: i32) -> vec2 { - return select(vec2(-2.5 * SCULPT_VOX, 0.0), vec2(0.0), shape == 0); -} - -fn s_bit(row: i32, r: i32, c: i32, rows: i32, cols: i32) -> bool { - return r >= 0 && r < rows && c >= 0 && c < cols && ((row >> u32(clamp(c, 0, 31))) & 1) == 1; +fn s_pix_bit(m: i32, c: i32) -> bool { + return ((m >> u32(clamp(c, 0, 31))) & 1) == 1; } -fn s_occ(id: vec2, shape: i32) -> bool { - let c = i32(id.x); - let r = -i32(id.y) - 1; - if shape == 0 { - return s_bit(SCULPT_ARROWPIX[clamp(r, 0, 15)], r, c, 16, 16); - } - return s_bit(SCULPT_HANDPIX[clamp(r, 0, 14)], r, c, 15, 13); +fn s_pix_body(shape: i32, c: i32, r: i32) -> bool { + let g = PIX_GRID[shape]; + return c >= 0 && c < g.x && r >= 0 && r < g.y && s_pix_bit(PIX_BODY[g.z + clamp(r, 0, g.y - 1)], c); } -fn s_occ_back(id: vec2, shape: i32) -> bool { - let c = i32(id.x) + 1; - let r = -i32(id.y); - if shape == 0 { - return s_bit(SCULPT_ARROWBACK[clamp(r, 0, 17)], r, c, 18, 17); - } - return s_bit(SCULPT_HANDBACK[clamp(r, 0, 16)], r, c, 17, 15); +fn s_pix_cell(p: vec2, shape: i32) -> vec2 { + let o = PIX_ORIGIN[shape]; + return vec2(i32(floor((p.x - o.x) / SCULPT_VOX)), i32(floor((o.y - p.y) / SCULPT_VOX))); } fn s_voxels(p: vec3, shape: i32) -> vec2 { - let o = s_grid_origin(shape); - let cell = floor((p.xy - o) / SCULPT_VOX); - let bc = select(vec2(0.25, -0.47), vec2(0.33, -0.5), shape == 0); - let bh = select(vec2(0.48, 0.55), vec2(0.44, 0.61), shape == 0); - let bq = max(abs(p.xy - bc) - bh, vec2(0.0)); + let b = PIX_BOX[shape]; + let bq = max(abs(p.xy - b.xy) - b.zw, vec2(0.0)); let bz = max(abs(p.z - (SCULPT_HOVER + 0.07)) - 0.07, 0.0); - let far = max(SCULPT_VOX, sqrt(dot(bq, bq) + bz * bz)); - var dF = far; - var dB = far; - let half_cell = vec3(0.5 * SCULPT_VOX, 0.5 * SCULPT_VOX, 0.04); + var d = max(SCULPT_VOX, sqrt(dot(bq, bq) + bz * bz)); + let o = PIX_ORIGIN[shape]; + let cell = s_pix_cell(p.xy, shape); + let cube = vec3(0.5 * SCULPT_VOX, 0.5 * SCULPT_VOX, 0.07); for (var j = -1; j <= 1; j++) { for (var i = -1; i <= 1; i++) { - let id = cell + vec2(f32(i), f32(j)); - let q = vec3(p.xy - (o + (id + vec2(0.5)) * SCULPT_VOX), p.z); - if s_occ(id, shape) { - dF = min(dF, s_round_box(q - vec3(0.0, 0.0, SCULPT_HOVER + 0.1), half_cell, 0.009)); - } - if s_occ_back(id, shape) { - dB = min(dB, s_round_box(q - vec3(0.0, 0.0, SCULPT_HOVER + 0.04), half_cell, 0.009)); + let c = cell.x + i; + let r = cell.y + j; + if s_pix_body(shape, c, r) { + let q = vec3(p.x - o.x - (f32(c) + 0.5) * SCULPT_VOX, p.y - o.y + (f32(r) + 0.5) * SCULPT_VOX, + p.z - SCULPT_HOVER - 0.07); + d = min(d, s_round_box(q, cube, 0.006)); } } } - return select(vec2(dB, 7.0), vec2(dF, 6.0), dF < dB); + return vec2(d, 6.0); } -fn s_pixel_hand_dist(p: vec2) -> f32 { +fn s_pixel_outline(p: vec2, shape: i32) -> f32 { var d = 1e9; - for (var r = 0; r < 17; r++) { - let x = (SCULPT_HANDSPAN[r] + vec2(-3.5, -2.5)) * SCULPT_VOX; - let y = vec2(-f32(r), 1.0 - f32(r)) * SCULPT_VOX; - let q = max(max(vec2(x.x, y.x) - p, p - vec2(x.y, y.y)), vec2(0.0)); - d = min(d, length(q)); + for (var i = PIX_RECT_N[shape]; i < PIX_RECT_N[shape + 1]; i++) { + let q = abs(p - PIX_RECT[i].xy) - PIX_RECT[i].zw; + d = min(d, length(max(q, vec2(0.0))) + min(max(q.x, q.y), 0.0)); } return d; } -fn s_gem_edge(p: vec3, a: vec2, b: vec2, z0: f32) -> f32 { - let e = b - a; - let len = length(e); - let d = e / len; - let q = p.xy - a; - let dist = dot(q, vec2(-d.y, d.x)); - let along = abs(dot(q, d) - 0.5 * len); - let z = p.z - z0; - let girdle = (z - 0.035 - 2.2 * dist) * 0.4138; - let crown = (z - 0.07 - 0.6 * dist + 0.18 * along) * 0.8438; - return max(-dist, max(girdle, crown)); -} - -fn s_gem_arrow(p: vec3) -> f32 { - let z0 = SCULPT_HOVER + 0.03; - let slab = max(p.z - z0 - 0.2, z0 - 0.03 - p.z); - let head = max(slab, max(s_gem_edge(p, vec2(-0.02, 0.03), vec2(-0.02, -0.88), z0), - max(s_gem_edge(p, vec2(-0.02, -0.88), vec2(0.66, -0.61), z0), - s_gem_edge(p, vec2(0.66, -0.61), vec2(-0.02, 0.03), z0)))); - let tail = max(slab, max(max(s_gem_edge(p, vec2(0.215, -0.665), vec2(0.37, -1.0), z0), - s_gem_edge(p, vec2(0.37, -1.0), vec2(0.53, -0.93), z0)), - max(s_gem_edge(p, vec2(0.53, -0.93), vec2(0.38, -0.60), z0), - s_gem_edge(p, vec2(0.38, -0.60), vec2(0.215, -0.665), z0)))); - return min(head, tail); -} - -fn s_facet_capsule(p: vec3, a: vec3, b: vec3, r: f32, spin: f32) -> f32 { - let u = normalize(b - a); - let v = normalize(cross(u, vec3(0.0, 0.0, 1.0))); - let w = cross(u, v); - let q = p - b; - let h = dot(q, u); - let rad = vec2(dot(q, v), dot(q, w)); - var d = max(h - r, -dot(p - a, u) - r); - for (var k = 0; k < 6; k++) { - let an = spin + f32(k) * 1.0471976; - let s = dot(rad, vec2(cos(an), sin(an))); - d = max(d, s - r); - d = max(d, dot(rad, vec2(cos(an + 0.5236), sin(an + 0.5236))) * 0.8660 + h * 0.5 - r); - d = max(d, s * 0.5 + h * 0.8660 - r); +// ---- Prism Glow : un cristal en MAILLAGE ---- (port du HLSL, dont les commentaires font foi) +// Les facettes tracees sur son dessin, en triangles dans le repere du modele ; le serti est la +// silhouette extrudee (champ de distance), le cristal est lance de rayons boite par boite. +// prism mesh: generated by design/cursors/prism-glow/model/export_compositor.py +var PRISM_TRI_START = array(0, 31); +var PRISM_TRI_COUNT = array(31, 130); +var PRISM_SIL_START = array(0, 25); +var PRISM_SIL_COUNT = array(25, 42); +var PRISM_OUT_START = array(67, 76); +var PRISM_OUT_COUNT = array(9, 34); +var PRISM_BOX_START = array(0, 4); +var PRISM_BOX_COUNT = array(4, 18); +struct PrismMesh { + tris: array, 644>, + poly: array, 110>, + boxes: array, 44>, +} +const PRISM_BEVEL: f32 = 0.012883; +// end of the prism mesh + +const PRISM_IOR: f32 = 1.61; +const PRISM_DISPERSION: f32 = 0.035; +const PRISM_GLOW: f32 = 0.45; +const PRISM_FOLD: f32 = 0.35; +const PRISM_BOUNCES: i32 = 6; +const PRISM_MAT: f32 = 9.0; + +fn prism_shape() -> i32 { + let id = sculpt_id(); + if id == 3 || id == 4 { + return id - 3; } - return d; + return -1; } -const SCULPT_FACETS = array, 10>( - vec3(1.0, 0.0, 0.0), vec3(0.0, 1.0, 0.0), vec3(0.0, 0.0, 1.0), - vec3(0.7071, 0.7071, 0.0), vec3(0.7071, 0.0, 0.7071), vec3(0.0, 0.7071, 0.7071), - vec3(0.5774, 0.5774, 0.5774), vec3(0.4472, 0.0, 0.8944), vec3(0.0, 0.4472, 0.8944), - vec3(0.3015, 0.3015, 0.9045) -); +// Drapeaux d'un triangle : paroi (bit 0), aretes reelles (bits 1 a 3). +fn prism_flags(k: i32) -> i32 { + return i32(prism.tris[4 * k].w + 0.5); +} -fn s_facet_ellipsoid(p: vec3, r: vec3) -> f32 { - let q = abs(p); - var d = -1e9; - for (var i = 0; i < 10; i++) { - d = max(d, dot(q, SCULPT_FACETS[i]) - length(r * SCULPT_FACETS[i])); +fn prism_poly_dist(p: vec2, i0: i32, n: i32) -> f32 { + var vj = prism.poly[i0 + n - 1].xy; + var d = 1e9; + var sgn = 1.0; + for (var i = 0; i < n; i++) { + let vi = prism.poly[i0 + i].xy; + let e = vj - vi; + let w = p - vi; + let b = w - e * saturate(dot(w, e) / dot(e, e)); + d = min(d, dot(b, b)); + let c0 = p.y >= vi.y; + let c1 = p.y < vj.y; + let c2 = e.x * w.y > e.y * w.x; + if (c0 && c1 && c2) || (!c0 && !c1 && !c2) { + sgn = -sgn; + } + vj = vi; } - return d; + return sgn * sqrt(d); } -fn s_crystal_hand(p: vec3) -> f32 { - let zc = SCULPT_HAND_ZC; - var d = s_facet_ellipsoid(p - vec3(0.185, -0.70, zc), vec3(0.3, 0.235, 0.125)); - d = min(d, s_facet_capsule(p, vec3(0.0, -0.52, zc), vec3(0.0, -0.10, zc), 0.098, 0.3)); - d = min(d, s_facet_capsule(p, vec3(0.17, -0.57, zc + 0.012), vec3(0.17, -0.41, zc + 0.045), 0.086, 0.1)); - d = min(d, s_facet_capsule(p, vec3(0.31, -0.59, zc + 0.01), vec3(0.31, -0.45, zc + 0.04), 0.08, 0.5)); - d = min(d, s_facet_capsule(p, vec3(0.435, -0.625, zc + 0.005), vec3(0.435, -0.52, zc + 0.03), 0.07, 0.2)); - return min(d, s_facet_capsule(p, vec3(0.03, -0.77, zc + 0.03), vec3(-0.165, -0.60, zc + 0.065), 0.082, 0.7)); +fn prism_rim(p: vec3, shape: i32) -> f32 { + let half_t = model_thick() * 0.5; + let sil = prism_poly_dist(p.xy, PRISM_SIL_START[shape], PRISM_SIL_COUNT[shape]); + let w = vec2(sil + PRISM_BEVEL, abs(p.z + half_t) - (half_t - PRISM_BEVEL)); + return min(max(w.x, w.y), 0.0) + length(max(w, vec2(0.0))) - PRISM_BEVEL; +} + +fn prism_tri(ro: vec3, rd: vec3, k: i32) -> f32 { + let v0 = prism.tris[4 * k].xyz; + let e1 = prism.tris[4 * k + 1].xyz; + let e2 = prism.tris[4 * k + 2].xyz; + let pv = cross(rd, e2); + let det = dot(e1, pv); + if abs(det) < 1e-10 { + return -1.0; + } + let inv = 1.0 / det; + let sv = ro - v0; + let u = dot(sv, pv) * inv; + let qv = cross(sv, e1); + let v = dot(rd, qv) * inv; + if u < -2e-4 || v < -2e-4 || u + v > 1.0004 { + return -1.0; + } + return dot(e2, qv) * inv; +} + +// (distance, indice) du triangle du cristal `shape` le plus proche devant (ro, rd) ; (1e9, -1) +// s'il n'y en a pas. Les triangles sont parcourus boîte par boîte (cf. HLSL). +fn prism_trace(ro: vec3, rd: vec3, shape: i32) -> vec2 { + let inv = vec3(1.0) / select(vec3(1e-6), rd, abs(rd) > vec3(1e-6)); + let b0 = PRISM_BOX_START[shape]; + let nb = PRISM_BOX_COUNT[shape]; + var best = 1e9; + var hit = -1; + for (var j = b0; j < b0 + nb; j++) { + let lo = prism.boxes[2 * j]; + let hi = prism.boxes[2 * j + 1]; + let t0 = (lo.xyz - ro) * inv; + let t1 = (hi.xyz - ro) * inv; + let tn = min(t0, t1); + let tf = max(t0, t1); + let tmin = max(max(tn.x, tn.y), tn.z); + let tmax = min(min(tf.x, tf.y), tf.z); + if tmax < max(tmin, 1e-5) || tmin > best { + continue; + } + let k0 = i32(lo.w + 0.5); + let n = i32(hi.w + 0.5); + for (var i = k0; i < k0 + n; i++) { + let t = prism_tri(ro, rd, i); + if t > 1e-5 && t < best { + best = t; + hit = i; + } + } + } + return vec2(best, f32(hit)); } fn sculpt_proto(p: vec3, theme: i32, shape: i32) -> vec2 { if theme == 3 { return s_voxels(p, shape); } - if theme == 1 { - return vec2(select(s_crystal_hand(p), s_gem_arrow(p), shape == 0), 8.0); - } - var body: vec2; - var star: vec3; - if shape == 0 { - var h = 0.075; - var re = 0.03; - var dome = 0.0; - if theme == 2 { - h = 0.07; - re = 0.06; - dome = 0.035; - } - if theme == 4 { - h = 0.07; - re = 0.055; - dome = 0.025; - } - body = vec2(s_arrow_solid(p, h, re, dome), 1.0); - if theme == 0 { - body = s_opu(body, vec2(s_piping(p, SCULPT_HOVER + 2.0 * h - 0.004), 2.0)); - } - star = vec3(0.57, -0.86, SCULPT_HOVER + 2.0 * h + dome); - } else { - body = s_opu(vec2(s_glove(p, SCULPT_HAND_ZC), 1.0), vec2(s_cuff(p, SCULPT_HAND_ZC), 2.0)); - star = vec3(0.185, -0.93, SCULPT_HAND_ZC + 0.15); - } - if theme == 4 { - return s_opu(body, s_sprout(p, star)); - } - return body; + return s_rimmed(p, theme, shape); } fn sculpt_point(q: vec3) -> vec3 { @@ -940,14 +1141,18 @@ fn sculpt_eval(q: vec3, occ: bool) -> vec2 { let id = sculpt_id() - 1; let theme = id / 2; let shape = id % 2; + if theme == 1 { + // Prism Glow : son serti seul, dans le repere du modele (le cristal est trace a part). + return vec2(prism_rim(q, shape), 8.0); + } let p = sculpt_point(q); var r: vec2; if occ && theme == 3 { - let d2 = select(s_pixel_hand_dist(p.xy), s_arrow2(p.xy) - 1.2 * SCULPT_VOX, shape == 0); + let d2 = s_pixel_outline(p.xy, shape); let dz = abs(p.z - (SCULPT_HOVER + 0.07)) - 0.07; r = vec2(length(max(vec2(d2, dz), vec2(0.0))) + min(max(d2, dz), 0.0), 7.0); } else { - r = sculpt_proto(p, select(theme, 2, occ && theme == 1), shape); + r = sculpt_proto(p, theme, shape); } return vec2(r.x * sculpt_units(), r.y); } @@ -956,7 +1161,7 @@ fn sd_sprite2(p: vec2) -> f32 { let lo = layer.color.rg; let c = clamp(p, lo, lo + sprite_size()); // Niveau 0 explicite : la marche est une boucle a sortie anticipee (pas de derivees). - let d = textureSampleLevel(texU, samp, (c - lo) / sprite_size(), 0.0).r; + let d = textureSampleLevel(texMask, samp, (c - lo) / sprite_size(), 0.0).r; let o = p - c; let out2 = dot(o, o); let e = max(d, 0.0); @@ -1016,7 +1221,7 @@ fn model_albedo(p: vec2) -> vec3 { let g = vec2(sd_sprite2(p + vec2(e, 0.0)) - sd_sprite2(p - vec2(e, 0.0)), sd_sprite2(p + vec2(0.0, e)) - sd_sprite2(p - vec2(0.0, e))); let q = p - g / max(length(g), 1e-6) * max(sd_sprite2(p) + MODEL_RIM_INSET * texel, 0.0); - return textureSampleLevel(texY, samp, (q - layer.color.rg) / sprite_size(), 0.0).rgb; + return textureSampleLevel(texDof, samp, (q - layer.color.rg) / sprite_size(), 0.0).rgb; } struct SculptMat { @@ -1028,61 +1233,60 @@ struct SculptMat { } fn s_pixel_colour(p: vec3, shape: i32) -> vec3 { - let id = floor((p.xy - s_grid_origin(shape)) / SCULPT_VOX); - if !s_occ(id + vec2(-1.0, 0.0), shape) || !s_occ(id + vec2(0.0, -1.0), shape) { - return s_lin(0.52, 0.91, 0.77); + let cell = s_pix_cell(p.xy, shape); + let g = PIX_GRID[shape]; + let row = g.z + clamp(cell.y, 0, g.y - 1); + if s_pix_bit(PIX_LINE[row], cell.x) { + return s_lin(0.29, 0.12, 0.36); } - if !s_occ(id + vec2(1.0, 0.0), shape) || !s_occ(id + vec2(0.0, 1.0), shape) { + if s_pix_bit(PIX_HI[row], cell.x) { return s_lin(1.0, 0.78, 0.87); } - return s_lin(1.0, 0.50, 0.71); + if s_pix_bit(PIX_SHADE[row], cell.x) { + return s_lin(0.87, 0.27, 0.51); + } + return s_lin(1.0, 0.435, 0.66); } fn sculpt_material(mat: f32, p: vec3, theme: i32, shape: i32) -> SculptMat { let primary = mat < 1.5; if theme == 0 { - if shape == 0 && primary { - return SculptMat(s_lin(0.10, 0.10, 0.115), 0.3, 0.2, 0.0, 0.9); - } - if shape == 0 { - return SculptMat(s_lin(0.94, 0.91, 0.84), 0.45, 0.4, 0.2, 0.3); - } - if primary { - return SculptMat(s_lin(0.95, 0.92, 0.85), 0.55, 0.35, 0.35, 0.25); + if mat < 2.5 { + return SculptMat(s_lin(0.95, 0.92, 0.85), 0.3, 0.6, 0.35, 0.25); } - return SculptMat(s_lin(0.17, 0.18, 0.22), 0.35, 0.6, 0.0, 0.6); + return SculptMat(s_lin(0.1, 0.1, 0.11), 0.55, 0.05, 0.0, 0.1); + } + if theme == 1 { + return SculptMat(s_lin(0.02, 0.05, 0.33), 0.35, 0.5, 0.05, 0.3); } if theme == 2 { - if shape == 0 { - return SculptMat(s_lin(1.0, 0.40, 0.30), 0.5, 0.45, 0.4, 0.25); + if (primary && shape == 0) || (mat > 2.5 && mat < 3.5 && shape == 1) { + return SculptMat(s_lin(1.0, 0.40, 0.30), 0.85, 0.08, 0.15, 0.03); } - if primary { - return SculptMat(s_lin(1.0, 0.79, 0.16), 0.5, 0.45, 0.4, 0.25); + if mat < 3.5 { + return SculptMat(s_lin(1.0, 0.80, 0.10), 0.85, 0.08, 0.15, 0.03); } - return SculptMat(s_lin(0.18, 0.20, 0.29), 0.4, 0.5, 0.0, 0.4); + return SculptMat(s_lin(0.09, 0.13, 0.45), 0.9, 0.05, 0.05, 0.02); } if theme == 4 { if primary && shape == 0 { - return SculptMat(s_lin(0.62, 0.91, 0.78), 0.22, 0.8, 0.25, 0.6); + return SculptMat(s_lin(0.68, 0.93, 0.80), 0.25, 0.7, 0.3, 0.3); } if primary { - return SculptMat(s_lin(0.96, 0.94, 0.88), 0.5, 0.35, 0.35, 0.25); + return SculptMat(s_lin(0.97, 0.95, 0.90), 0.3, 0.6, 0.35, 0.25); } if mat < 2.5 { - return SculptMat(s_lin(0.62, 0.91, 0.78), 0.25, 0.7, 0.25, 0.5); + return SculptMat(s_lin(0.52, 0.87, 0.78), 0.25, 0.7, 0.3, 0.3); } if mat < 3.5 { - return SculptMat(s_lin(1.0, 0.80, 0.20), 0.3, 0.6, 0.3, 0.4); + return SculptMat(s_lin(1.0, 0.75, 0.25), 0.25, 0.7, 0.3, 0.3); } if mat < 4.5 { - return SculptMat(s_lin(0.38, 0.80, 0.55), 0.35, 0.5, 0.35, 0.3); + return SculptMat(s_lin(0.62, 0.92, 0.72), 0.3, 0.6, 0.3, 0.35); } - return SculptMat(s_lin(0.16, 0.12, 0.10), 0.2, 0.8, 0.0, 0.5); - } - if mat < 6.5 { - return SculptMat(s_pixel_colour(p, shape), 0.45, 0.35, 0.15, 0.2); + return SculptMat(s_lin(0.07, 0.15, 0.33), 0.7, 0.15, 0.05, 0.08); } - return SculptMat(s_lin(0.36, 0.18, 0.54), 0.45, 0.35, 0.1, 0.2); + return SculptMat(s_pixel_colour(p, shape), 0.65, 0.1, 0.12, 0.04); } fn model_env(d: vec3, rough: f32, l: vec3, fill: vec3) -> vec3 { @@ -1094,36 +1298,6 @@ fn model_env(d: vec3, rough: f32, l: vec3, fill: vec3) -> vec3 vec3 { - let k = saturate(k0) * 3.0; - let a = s_lin(0.20, 0.95, 1.0); - let b = s_lin(0.15, 0.42, 1.0); - let c = s_lin(0.45, 0.25, 0.95); - let d = s_lin(0.88, 0.50, 1.0); - if k < 1.0 { - return mix(a, b, k); - } - if k < 2.0 { - return mix(b, c, k - 1.0); - } - return mix(c, d, k - 2.0); -} - -fn model_shade_crystal(n: vec3, rd: vec3, L: vec3, fall: f32, l: vec3, fill: vec3) -> vec3 { - let cosi = saturate(dot(-rd, n)); - let F = 0.04 + 0.96 * pow(1.0 - cosi, 5.0); - let t = refract(rd, n, 1.0 / 1.6); - let k = 0.42 + 0.9 * dot(vec2(n.x, -n.y), vec2(0.7557, -0.6549)) - + 0.6 * dot(vec2(t.x, -t.y), vec2(0.6, -0.8)); - let body = vec3(s_gem(k - 0.08).r, s_gem(k).g, s_gem(k + 0.08).b); - var col = body * (0.1 + 1.8 * fall * pow(max(dot(n, L), 0.0), 2.5)); - col = col + body * model_env(reflect(t, vec3(0.0, 0.0, 1.0)) * vec3(1.0, 1.0, -1.0), 0.15, l, fill) * 0.5; - col = col + vec3(pow(max(dot(reflect(rd, n), L), 0.0), 30.0) * 2.0); - col = col + model_env(reflect(rd, n), 0.05, l, fill) * F; - col = col + s_lin(0.5, 0.9, 1.0) * pow(1.0 - cosi, 4.0) * 0.6; - return col; -} - fn model_tonemap(c0: vec3) -> vec3 { let start = 0.76; let x = min(c0.r, min(c0.g, c0.b)); @@ -1142,14 +1316,15 @@ fn model_tonemap(c0: vec3) -> vec3 { fn model_shade(q: vec3, n: vec3, rd: vec3, L: vec3, fall: f32, sh: f32, ao: f32, mat: f32, l: vec3, fill: vec3) -> vec3 { let id = sculpt_id(); - if id > 0 && (id - 1) / 2 == 1 { - return model_tonemap(model_shade_crystal(n, rd, L, fall, l, fill)); - } var m: SculptMat; var gloss = 1.0; if id > 0 { let p = sculpt_point(q) - vec3(n.x, -n.y, n.z) * 0.01; m = sculpt_material(mat, p, (id - 1) / 2, (id - 1) % 2); + if id == 3 || id == 4 { + // Le serti de Prism Glow, comme un sprite : brillant sur son arrondi seulement. + gloss = 1.0 - smoothstep(0.97, 0.995, abs(n.z)); + } } else { m = SculptMat(pow(model_albedo(q.xy), vec3(2.2)), 0.45, 0.35, 0.2, 0.3); gloss = 1.0 - smoothstep(0.97, 0.995, abs(n.z)); @@ -1171,13 +1346,160 @@ fn model_shade(q: vec3, n: vec3, rd: vec3, L: vec3, fall: f3 return model_tonemap(col); } +// ---- Le cristal de Prism Glow ---- + +fn prism_fresnel(c: f32, ior: f32) -> f32 { + var f0 = (ior - 1.0) / (ior + 1.0); + f0 = f0 * f0; + return f0 + (1.0 - f0) * pow(1.0 - c, 5.0); +} + +// (distance, facette) du cristal sur le rayon ; (1e9, -1) si rate, hors de Prism Glow, ou sur une +// paroi sous le dessus du serti (le rayon a traverse le serti avant). +fn prism_primary(ro: vec3, rd: vec3) -> vec2 { + let shape = prism_shape(); + if shape < 0 { + return vec2(1e9, -1.0); + } + let r = prism_trace(ro, rd, shape); + let h = i32(r.y); + if h >= 0 && (prism_flags(h) & 1) == 1 && ro.z + rd.z * r.x < 0.0 { + return vec2(1e9, -1.0); + } + return r; +} + +fn prism_normal(h: i32, rd: vec3) -> vec3 { + let n = prism.tris[4 * h + 3].xyz; + return select(n, -n, dot(n, rd) > 0.0); +} + +fn prism_seg(p: vec3, a: vec3, b: vec3) -> f32 { + let pa = p - a; + let ba = b - a; + return length(pa - ba * saturate(dot(pa, ba) / dot(ba, ba))); +} + +// Distance, en pixels, de `p` aux aretes reelles de la facette `h` (plis et bord du serti). +fn prism_edge_px(p: vec3, h: i32, px: f32) -> f32 { + let a = prism.tris[4 * h].xyz; + let b = a + prism.tris[4 * h + 1].xyz; + let c = a + prism.tris[4 * h + 2].xyz; + let bits = prism_flags(h) >> 1u; + var d = 1e9; + if (bits & 1) != 0 { + d = min(d, prism_seg(p, b, c)); + } + if (bits & 2) != 0 { + d = min(d, prism_seg(p, c, a)); + } + if (bits & 4) != 0 { + d = min(d, prism_seg(p, a, b)); + } + return d / px; +} + +// La rotation inverse de `world_to_plane` : du repere du plan a celui de la camera. +fn plane_to_world(v: vec3, f: ModelFrame) -> vec3 { + let x = v.x * f.c.z - v.y * f.s.z; + let y = v.x * f.s.z + v.y * f.c.z; + let z = -x * f.s.y + v.z * f.c.y; + return vec3(x * f.c.y + v.z * f.s.y, y * f.c.x - z * f.s.x, y * f.s.x + z * f.c.x); +} + +// L'image composee la ou le rayon retombe sur le plan (cf. HLSL) : sa copie au binding 1. +fn prism_screen(q: vec3, d: vec3, nz: vec3, hz: f32, tip: vec3, unit: f32, f: ModelFrame) -> vec3 { + let denom = dot(d, nz); + if denom > -1e-4 { + return SCULPT_SCREEN * 0.2; + } + let g = q + d * ((hz - dot(q, nz)) / denom); + let w = plane_to_world(tip + unit * model_to_plane(g, f), f); + let k = 1.0 - w.z / layer.src.z; + if k < 1e-3 { + return SCULPT_SCREEN * 0.2; + } + let uv = layer.dst.xy + ((w.xy + layer.mb.zw) / k - layer.src.xy) / layer.quad_px * layer.dst.zw; + if any(uv < vec2(0.0)) || any(uv > vec2(1.0)) { + return SCULPT_SCREEN * 0.2; + } + return pow(textureSampleLevel(texY, samp, uv, 0.0).rgb, vec3(2.2)); +} + +fn prism_shade(p: vec3, n: vec3, rd: vec3, h: i32, px: f32, l: vec3, fill: vec3, + nz: vec3, hz: f32, tip: vec3, unit: f32, f: ModelFrame) -> vec3 { + let shape = prism_shape(); + let facet = vec3(prism.tris[4 * h + 1].w, prism.tris[4 * h + 2].w, prism.tris[4 * h + 3].w); + let F = prism_fresnel(saturate(-dot(rd, n)), PRISM_IOR); + let zb = -model_thick(); + var trans = vec3(0.0); + for (var ch = 0; ch < 3; ch++) { + // Le rouge plie le moins, le bleu le plus ; le canal passe par un masque (comme au HLSL). + let mask = vec3(f32(ch == 0), f32(ch == 1), f32(ch == 2)); + let ior = PRISM_IOR + PRISM_DISPERSION * f32(ch - 1); + var d = refract(rd, n, 1.0 / ior); + var pos = p; + var thr = 1.0; + var acc = 0.0; + for (var b = 0; b < PRISM_BOUNCES; b++) { + let hit = prism_trace(pos, d, shape); + let hh = i32(hit.y); + let th = hit.x; + var tb = 1e9; + if d.z < -1e-6 { + tb = (zb - pos.z) / d.z; + } + if tb <= th { + let qb = pos + d * tb; + let d2 = refract(d, vec3(0.0, 0.0, 1.0), ior); + if dot(d2, d2) < 1e-8 { + pos = qb; + d = reflect(d, vec3(0.0, 0.0, 1.0)); + continue; + } + acc = acc + thr * dot(prism_screen(qb, d2, nz, hz, tip, unit, f), mask); + thr = 0.0; + break; + } + if hh < 0 { + break; + } + pos = pos + d * th; + let rn = prism.tris[4 * hh + 3].xyz; + if (prism_flags(hh) & 1) == 1 && pos.z < 0.0 { + acc = acc + thr * dot(s_lin(0.03, 0.06, 0.37) * 0.35, mask); + thr = 0.0; + break; + } + let dout = refract(d, -rn, ior); + if dot(dout, dout) < 1e-8 { + d = reflect(d, -rn); + continue; + } + let fi = prism_fresnel(saturate(dot(dout, rn)), ior); + acc = acc + thr * (1.0 - fi) * dot(model_env(dout, 0.05, l, fill), mask); + thr = thr * fi; + d = reflect(d, -rn); + } + trans = trans + acc * mask; + } + var col = F * model_env(reflect(rd, n), 0.05, l, fill) + (1.0 - F) * trans + facet * PRISM_GLOW; + let fold = 1.0 - smoothstep(0.2, 1.0, prism_edge_px(p, h, px)); + col = mix(col, s_lin(0.96, 0.99, 1.0) * 1.3, PRISM_FOLD * fold * (0.35 + 0.65 * saturate(dot(n, l)))); + return model_tonemap(col); +} + // Les passes de la boucle unique de `cursor_model` (cf. HLSL : un seul appel de `model_eval`). const STAGE_MARCH: i32 = 0; const STAGE_NORMAL: i32 = 1; const STAGE_AO: i32 = 2; const STAGE_SELF: i32 = 3; const STAGE_PLANE: i32 = 4; -const STAGE_DONE: i32 = 5; +const STAGE_EDGE: i32 = 5; +const STAGE_SHADE: i32 = 6; +const STAGE_DONE: i32 = 7; + +const MODEL_SUBPIXEL = array, 3>(vec2(0.35, 0.2), vec2(-0.35, 0.2), vec2(0.0, -0.4)); fn model_tetra(k: i32) -> vec3 { let b = vec3(f32(((k + 3) >> 1u) & 1), f32((k >> 1u) & 1), f32(k & 1)); @@ -1199,7 +1521,7 @@ fn cursor_model(local: vec2) -> vec4 { let hi = vec3(layer.color.rg + sprite_size(), layer.trail_a.z); let dw = vec3(local + layer.src.xy, -persp); - let dlen = length(dw); + var dlen = length(dw); // Plan translate de mb.zw dans le repere camera (camera reelle ; 0 sous un angle fixe). let ro = plane_to_model((world_to_plane(vec3(-layer.mb.z, -layer.mb.w, persp), f) - tip) / unit, f); let rd = plane_to_model(world_to_plane(dw / dlen, f), f); @@ -1210,15 +1532,22 @@ fn cursor_model(local: vec2) -> vec4 { let stride = select(1.0, 0.85, sculpt_id() > 0); let lamp = vec3(layer.color.rg + sprite_size() * 0.5, 0.0) + l * SCULPT_LAMP_DIST; - let tb = ray_box(ro, rd, lo - vec3(0.02), hi + vec3(0.02)); + var ray = rd; + var tb = ray_box(ro, ray, lo - vec3(0.02), hi + vec3(0.02)); var stage = select(STAGE_DONE, STAGE_MARCH, tb.x < tb.y && tb.y > 0.0); var plane_next = stage == STAGE_DONE; + // Prism Glow : le cristal, trace au depart de chaque rayon (`tc` son point, `hc` sa facette). + var need_tc = stage == STAGE_MARCH; + var tc = 1e9; + var hc = -1; var k = 0; var t = max(tb.x, 0.0); var best = 1e9; var t_best = t; + var d_best = 0.0; var mat = 0.0; var cov = 0.0; + var cov_s = 0.0; var q = vec3(0.0); var n = vec3(0.0); var L = vec3(0.0); @@ -1230,10 +1559,19 @@ fn cursor_model(local: vec2) -> vec4 { var inside = 0.0; var contact = 0.0; var dropped = 0.0; - for (var it = 0; it < 180; it++) { + var sub = 0; + var shaded = 0; + var acc = vec4(0.0); + for (var it = 0; it < 500; it++) { + if need_tc { + need_tc = false; + let pr = prism_primary(ro, ray); + tc = pr.x; + hc = i32(pr.y); + } if plane_next { plane_next = false; - stage = STAGE_DONE; + stage = select(STAGE_DONE, STAGE_SHADE, cov > 0.0); let denom = dot(rd, nz); if cov < 1.0 && denom < -1e-4 { g = ro + rd * ((hz - dot(ro, nz)) / denom); @@ -1246,11 +1584,48 @@ fn cursor_model(local: vec2) -> vec4 { } } } + if stage == STAGE_SHADE { + if cov_s > 0.0 { + var c: vec3; + if mat > PRISM_MAT - 0.5 { + c = prism_shade(q, n, ray, hc, t_best / dlen, l, fill, nz, hz, tip, unit, f); + } else { + let tl = lamp - q; + let fall = SCULPT_LAMP_DIST * SCULPT_LAMP_DIST / dot(tl, tl); + c = model_shade(q, n, ray, normalize(tl), fall, sh, ao, mat, l, fill); + } + acc = acc + vec4(c * cov_s, cov_s); + } + shaded++; + stage = STAGE_DONE; + if sub > 0 { + let dws = vec3(local + MODEL_SUBPIXEL[3 - sub] + layer.src.xy, -persp); + sub--; + dlen = length(dws); + ray = plane_to_model(world_to_plane(dws / dlen, f), f); + tb = ray_box(ro, ray, lo - vec3(0.02), hi + vec3(0.02)); + cov_s = 0.0; + stage = STAGE_SHADE; + if tb.x < tb.y && tb.y > 0.0 { + stage = STAGE_MARCH; + k = 0; + t = max(tb.x, 0.0); + best = 1e9; + t_best = t; + need_tc = true; + } + } + continue; + } var pos: vec3; if stage == STAGE_MARCH { - pos = ro + rd * t; + pos = ro + ray * t; } else if stage == STAGE_NORMAL { pos = q + 0.002 * model_tetra(k); + } else if stage == STAGE_EDGE { + let side = normalize(cross(n, ray)); + let e = select(side, cross(n, side), k >= 2) * select(1.0, -1.0, k % 2 == 1); + pos = q + e * 0.5 * t_best / dlen; } else if stage == STAGE_AO { pos = q + (0.01 + 0.0175 * f32(k)) * SCULPT_SCALE * n; } else if stage == STAGE_SELF { @@ -1264,22 +1639,57 @@ fn cursor_model(local: vec2) -> vec4 { let d = m.x; if stage == STAGE_MARCH { let fp = t / dlen; - let hit = d < 0.1 * fp; + var hit = d < 0.1 * fp; if hit || d / fp < best { best = select(d / fp, 0.0, hit); t_best = t; + d_best = d; mat = m.y; } t = t + d * stride; k++; + if !hit && t >= tc { + // Le pas depasse le cristal : c'est lui que le rayon touche, avant le serti. + hit = true; + best = 0.0; + t_best = tc; + d_best = 0.0; + mat = PRISM_MAT; + } if hit || t > tb.y || k == 96 { - cov = saturate(1.0 - best); - if cov > 0.0 { - q = ro + rd * t_best; - stage = STAGE_NORMAL; - k = 0; + cov_s = saturate(1.0 - best); + if shaded > 0 { + stage = STAGE_SHADE; + if cov_s > 0.0 { + q = ro + ray * t_best; + n = vec3(0.0); + stage = STAGE_NORMAL; + k = 0; + } } else { - plane_next = true; + cov = cov_s; + if cov > 0.0 { + q = ro + ray * t_best; + stage = STAGE_NORMAL; + k = 0; + } else { + plane_next = true; + } + } + if stage == STAGE_NORMAL && mat > PRISM_MAT - 0.5 { + // Le cristal : la normale de sa facette, ni occlusion ni ombre propre ; un bord + // reel a moins d'un pixel appelle les trois rayons de plus. + n = prism_normal(hc, ray); + k = 0; + if shaded > 0 { + stage = STAGE_SHADE; + } else { + if prism_edge_px(q, hc, t_best / dlen) < 0.75 { + sub = 3; + } + stage = STAGE_DONE; + plane_next = true; + } } } } else if stage == STAGE_NORMAL { @@ -1287,6 +1697,25 @@ fn cursor_model(local: vec2) -> vec4 { k++; if k == 4 { n = normalize(n); + k = 0; + stage = select(STAGE_EDGE, STAGE_SHADE, shaded > 0); + } + } else if stage == STAGE_EDGE { + if m.y != mat || abs(d - d_best) > 0.02 * t_best / dlen { + sub = 3; + } + if k == 0 && mat > 7.5 { + // Le serti de Prism Glow au bord du cristal : son contour a moins d'un pixel. + let ps = prism_shape(); + if ps >= 0 { + let dout = prism_poly_dist(q.xy, PRISM_OUT_START[ps], PRISM_OUT_COUNT[ps]); + if abs(dout) < 0.75 * t_best / dlen { + sub = 3; + } + } + } + k++; + if k == 4 { stage = STAGE_AO; k = 0; ao = 0.0; @@ -1323,7 +1752,7 @@ fn cursor_model(local: vec2) -> vec4 { res = 1.0; k = 1; if !(tbp.x < tbp.y && tbp.y > 0.0) { - stage = STAGE_DONE; + stage = select(STAGE_DONE, STAGE_SHADE, cov > 0.0); } } else { res = min(res, MODEL_SOFTNESS * d / ts.x); @@ -1332,20 +1761,15 @@ fn cursor_model(local: vec2) -> vec4 { if res < 0.002 || ts.x > ts.y || k == 33 { res = saturate(res); dropped = 1.0 - res * res * (3.0 - 2.0 * res); - stage = STAGE_DONE; + stage = select(STAGE_DONE, STAGE_SHADE, cov > 0.0); } } } - var rgb = vec3(0.0); - if cov > 0.0 { - let tl = lamp - q; - let fall = SCULPT_LAMP_DIST * SCULPT_LAMP_DIST / dot(tl, tl); - rgb = model_shade(q, n, rd, normalize(tl), fall, sh, ao, mat, l, fill); - } + let w = 1.0 / f32(max(shaded, 1)); let shadow = inside * max(dropped * MODEL_SHADOW_ALPHA, contact * MODEL_CONTACT_ALPHA); - let a = cov * layer.color.a; - return vec4(rgb * a, a + (1.0 - a) * shadow * layer.color.a); // premultiplie, ombre noire + let a = acc.a * w * layer.color.a; + return vec4(acc.rgb * w * layer.color.a, a + (1.0 - a) * shadow * layer.color.a); // premultiplie, ombre noire } // ---- Impact du clic (mode 16) ---- @@ -1855,12 +2279,14 @@ fn device_frame(local: vec2) -> vec4 { return vec4(rgb * a, a); // premultiplie } -@fragment -fn fs_main(i: VsOut) -> @location(0) vec4 { +// The body of `fs_main`, which only adds the layer's transparency on top of it. +fn fs_layer(i: VsOut) -> vec4 { var rgb: vec3; var alpha: f32; // 1 sauf en detourage, ou il porte le masque du sujet. Cf. la branche fx.z plus bas. var alpha_mask = 1.0; + // 0 where a camera homography finds no camera point (`persp_camera`), 1 everywhere else. + var in_picture = 1.0; if layer.mode < 0.5 { // Mode 0 — vidéo NV12 + flou de mouvement par vélocité (§8), port 1:1 du @@ -1870,18 +2296,31 @@ fn fs_main(i: VsOut) -> @location(0) vec4 { // du calque sans avoir à transporter un champ de vitesse. let taps = i32(layer.mb.x); let mb_scale = clamp(layer.mb.y, 0.0, 1.0); + var uv_now = i.uv; + if layer.layer_fx.y > 0.5 { + // Camera homography: the texture uv comes from the fragment's place in the quad + // (`src` is not used), scaled to the valid part of the decoder texture. + let cam = persp_camera((i.pout - layer.dst.xy) / layer.dst.zw); + in_picture = cam.z; + uv_now = cam.xy * layer.fx.xy; + } // `taps` d'abord : un draw qui a oublié `dst_prev` le laisse à zéro, et // la division par `dst_prev.zw` produirait des UV infinis. Dégrader vers // le chemin net est le seul échec acceptable pour un effet cosmétique. if taps <= 1 || mb_scale <= 0.001 || layer.dst_prev.z <= 0.0 || layer.dst_prev.w <= 0.0 { - rgb = sample_yuv(i.uv); + rgb = sample_yuv(uv_now); } else { let localp = (i.pout - layer.dst_prev.xy) / layer.dst_prev.zw; - let uv_prev = layer.src_prev.xy + localp * (layer.src_prev.zw - layer.src_prev.xy); - let duv = i.uv - uv_prev; + var uv_prev = layer.src_prev.xy + localp * (layer.src_prev.zw - layer.src_prev.xy); + if layer.layer_fx.y > 0.5 { + // The previous frame's uv is the same map at the quad's previous place. + let cam_prev = persp_camera(localp); + uv_prev = select(uv_now, cam_prev.xy * layer.fx.xy, cam_prev.z > 0.5); + } + let duv = uv_now - uv_prev; let duv_blur = duv * mb_scale; if dot(duv_blur, duv_blur) < 1e-9 { - rgb = sample_yuv(i.uv); + rgb = sample_yuv(uv_now); } else { // Borne 16 en dur, identique au HLSL et au MSL : `taps` vient d'un // uniform et une boucle sans borne statique ne se déroule pas. @@ -1891,7 +2330,7 @@ fn fs_main(i: VsOut) -> @location(0) vec4 { let step = 1.0 / f32(taps - 1); for (var k: i32 = 0; k < 16; k = k + 1) { if k >= taps { break; } - acc = acc + sample_yuv(i.uv - duv_blur * (1.0 - f32(k) * step)); + acc = acc + sample_yuv(uv_now - duv_blur * (1.0 - f32(k) * step)); } rgb = acc / f32(taps); } @@ -1902,16 +2341,31 @@ fn fs_main(i: VsOut) -> @location(0) vec4 { // l'etendue valide de la texture webcam pour ramener uv dans l'espace du masque. let effect = layer.fx.z; if effect > 0.5 { - let mask_uv = i.uv / max(layer.fx.xy, vec2(1e-6)); + let mask_uv = uv_now / max(layer.fx.xy, vec2(1e-6)); let person = clamp(textureSample(texMask, samp, mask_uv).r, 0.0, 1.0); if effect > 2.5 { rgb = mix(layer.color.rgb, rgb, person); } else if effect > 1.5 { - rgb = mix(blur_webcam_bg(i.uv, layer.fx.w, layer.quad_px, i.local), rgb, person); + rgb = mix(blur_webcam_bg(uv_now, layer.fx.w, layer.quad_px, i.local, layer.fx.xy), rgb, person); } else { alpha_mask = person; } } + + // Desk-view cover: the camera is being tilted, so the whole picture is blurred and + // dimmed (cover.x = strength, cover.y = radius in quad px, cover.z = dim at full cover). + if layer.cover.x > 0.001 { + let hidden = blur_webcam_radius(uv_now, layer.cover.y, layer.quad_px, i.local, layer.fx.xy); + rgb = mix(rgb, hidden, layer.cover.x) * (1.0 - layer.cover.z * layer.cover.x); + } + + // Past the camera picture the layer is black, as opaque as the rest of it (spec: a + // corrected crop that reaches past the camera image has a black edge, never the + // wallpaper). The mask and effects have no picture to act on there. + if in_picture < 0.5 { + rgb = vec3(0.0); + alpha_mask = 1.0; + } } else if layer.mode < 1.5 { // Mode 1 — couleur pleine. rgb = layer.color.rgb; @@ -2106,9 +2560,13 @@ fn fs_main(i: VsOut) -> @location(0) vec4 { let plane_px = layer.dst_prev.xy; let p = vec2(r.x, r.y) * plane_px - plane_px * 0.5; let rad = max(layer.radius_px, 0.0); - let d = select(sd_round_rect(p, plane_px * 0.5, rad), - sd_screen_under_bar(p, plane_px * 0.5, rad, layer.color.z), - layer.dst_prev.z > 0.5); + // Une branche et non un `select`, qui evaluerait les DEUX distances (trois SDF par pixel). + var d: f32; + if layer.dst_prev.z > 0.5 { + d = sd_screen_under_bar(p, plane_px * 0.5, rad, layer.color.z); + } else { + d = sd_round_rect(p, plane_px * 0.5, rad); + } // Slot d'un layout en bloc (`color.w` = rayon de ses coins, px ; 0 ailleurs, sans effet) : // `dst` EST le slot, dont le rect arrondi rogne le plan (`ScreenMask`, cf. HLSL). let tilt_a = (1.0 - smoothstep(0.0, 1.5, d)) @@ -2238,18 +2696,26 @@ fn fs_main(i: VsOut) -> @location(0) vec4 { frame_rgb = mix(frame_rgb, vec3(0.157, 0.784, 0.251), disc_cov(q, vec2(x0 + 2.0 * dx, bar * 0.5), dr)); let fa = cov * layer.color.a; return vec4(frame_rgb * fa, fa); // premultiplie - } else if layer.mode > 14.5 && layer.mode < 15.5 { - // Mode 15 -- curseur modelise (`cursor_model`). Clip « Clip to canvas » dans - // `dst_prev`, comme au mode 13. - if i.pout.x < layer.dst_prev.x || i.pout.x > layer.dst_prev.x + layer.dst_prev.z - || i.pout.y < layer.dst_prev.y || i.pout.y > layer.dst_prev.y + layer.dst_prev.w { + } else if layer.mode > 14.5 && layer.mode < 17.5 { + // Modes 15 a 17 -- les modeles 3D, que seul le pipeline `LAYER_MODELS` compile (cf. le + // haut du fichier). Le pipeline plat ne les recoit jamais ; s'il en recevait un, il ne + // dessinerait rien plutot qu'une ombre faite de leurs emplacements. + if !LAYER_MODELS { return vec4(0.0, 0.0, 0.0, 0.0); } - return cursor_model(i.local); - } else if layer.mode > 15.5 && layer.mode < 16.5 { - // Mode 16 -- impact du clic sous le curseur modelise (`cursor_impact`). - return cursor_impact(i.local); - } else if layer.mode > 16.5 && layer.mode < 17.5 { + if layer.mode < 15.5 { + // Mode 15 -- curseur modelise (`cursor_model`). Clip « Clip to canvas » dans + // `dst_prev`, comme au mode 13. + if i.pout.x < layer.dst_prev.x || i.pout.x > layer.dst_prev.x + layer.dst_prev.z + || i.pout.y < layer.dst_prev.y || i.pout.y > layer.dst_prev.y + layer.dst_prev.w { + return vec4(0.0, 0.0, 0.0, 0.0); + } + return cursor_model(i.local); + } + if layer.mode < 16.5 { + // Mode 16 -- impact du clic sous le curseur modelise (`cursor_impact`). + return cursor_impact(i.local); + } // Mode 17 -- cadre d'appareil modelise (`device_frame`). return device_frame(i.local); } else if layer.mode > 17.5 { @@ -2282,11 +2748,22 @@ fn fs_main(i: VsOut) -> @location(0) vec4 { // cadre (`sd_screen_under_bar`, `mb.z` = la remontee du contour interieur, px). let halfsz = layer.quad_px * 0.5; let p = i.local - layer.quad_px * 0.5; - let d = select(sd_round_rect(p, halfsz, layer.radius_px), - sd_screen_under_bar(p, halfsz, layer.radius_px, layer.mb.z), - layer.mb.w > 0.5); + // Une branche et non un `select` : celui-ci evaluait les DEUX distances, soit trois SDF + // par pixel de l'ecran au lieu d'une -- 0,6 ms par frame 1080p sur une Radeon 610M. + var d: f32; + if layer.mb.w > 0.5 { + d = sd_screen_under_bar(p, halfsz, layer.radius_px, layer.mb.z); + } else { + d = sd_round_rect(p, halfsz, layer.radius_px); + } alpha *= 1.0 - smoothstep(0.0, 1.5, d); } return vec4(rgb * alpha, alpha); // alpha prémultiplié } + +@fragment +fn fs_main(i: VsOut) -> @location(0) vec4 { + // Premultiplied, so the transparency scales all four channels; 0 leaves them as they are. + return fs_layer(i) * (1.0 - layer.layer_fx.x); +} diff --git a/crates/compositor/tests/compose_linux.rs b/crates/compositor/tests/compose_linux.rs index c8bc459367..1c74e011ee 100644 --- a/crates/compositor/tests/compose_linux.rs +++ b/crates/compositor/tests/compose_linux.rs @@ -799,6 +799,7 @@ fn export_linux_mp4() { source_end_sec: 1.0, webcam_offset_sec: 0.0, has_audio: true, + additional_cameras: Vec::new(), }]; let params = ExportParams { width: 640, diff --git a/crates/compositor/tests/cursor_model_render.rs b/crates/compositor/tests/cursor_model_render.rs index 86d28bf700..70151158fa 100644 --- a/crates/compositor/tests/cursor_model_render.rs +++ b/crates/compositor/tests/cursor_model_render.rs @@ -161,13 +161,13 @@ fn cursors_dir() -> String { .replace('\\', "/") } -/// Les cinq thèmes d'origine et les hotspots de leur flèche et de leur main plates -/// (`CURSOR_THEMES`, `src/lib/cursor/cursorThemes.ts`, sur 32). +/// Les thèmes d'origine et les hotspots de leur flèche et de leur main plates (`CURSOR_THEMES`, +/// `src/lib/cursor/cursorThemes.ts`, sur 32). const SCULPTED: [(&str, [f32; 2], [f32; 2]); 5] = [ ("studio-ink", [6.2304, 2.0992], [12.848, 2.0704]), ("prism-glow", [6.3456, 2.0672], [11.968, 2.0352]), ("pop-coral", [10.4768, 2.1792], [12.3456, 2.0]), - ("pixel-candy", [7.3664, 2.0], [13.376, 1.9264]), + ("pixel-candy", [6.5, 2.0], [10.75, 2.0]), ("star-sprout", [4.7232, 2.1152], [13.1712, 2.0384]), ]; @@ -447,6 +447,20 @@ fn opaque_mask(on_blue: &[u8], on_orange: &[u8], bare_blue: &[u8]) -> Vec .collect() } +/// Les pixels que couvre un curseur de VERRE (le cristal de Prism Glow, qui laisse voir le contenu +/// et le réfracte) : opaques, ou changés sur les deux teintes sans y être plus sombres (une ombre +/// ne fait qu'assombrir ; le verre réfracte, reflète et luit). +fn covered_mask(on_blue: &[u8], on_orange: &[u8], bare_blue: &[u8], bare_orange: &[u8]) -> Vec { + let opaque = opaque_mask(on_blue, on_orange, bare_blue); + let px = |img: &[u8], i: usize| [img[i * 4], img[i * 4 + 1], img[i * 4 + 2], 255]; + (0..1280 * 720) + .map(|i| { + let lit = |on: &[u8], bare: &[u8]| px(on, i) != px(bare, i) && luma(px(on, i)) >= luma(px(bare, i)) - 2.0; + opaque[i] || (lit(on_blue, bare_blue) && lit(on_orange, bare_orange)) + }) + .collect() +} + fn iou(a: &[bool], b: &[bool]) -> f32 { let inter = a.iter().zip(b).filter(|(x, y)| **x && **y).count(); let union = a.iter().zip(b).filter(|(x, y)| **x || **y).count(); @@ -765,16 +779,18 @@ fn the_motion_blur_trail_draws_modelled_copies() { assert!(rgba != flat, "la traînée 3D est celle du sprite plat"); } -/// Les dix curseurs sculptés (cinq thèmes, flèche et main), dessinés par le shader et non -/// extrudés d'un PNG : chacun est là, sa pointe sur le hotspot, son corps en bas à droite de -/// celle-ci (y vers le bas : un modèle retourné finirait au-dessus), et il porte son ombre en -/// l'air. +/// Les curseurs sculptés (flèche et main de chaque thème), dessinés par le shader et non extrudés +/// d'un PNG : chacun est là, sa pointe sur le hotspot, son corps en bas à droite de celle-ci (y +/// vers le bas : un modèle retourné finirait au-dessus), et il porte son ombre en l'air. Le +/// cristal de Prism Glow est de verre : son corps compte ce qu'il couvre, et il doit laisser voir +/// le contenu (une bonne part de lui change avec la teinte de l'écran). #[test] fn the_sculpted_cursors_stand_at_the_hotspot() { let Some(gpu) = gpu() else { return }; let comp = Compositor::new_sized(&gpu, 1280, 720).expect("compositor"); let (blue, orange) = (FakeFrame::new(&gpu, Tint::Blue), FakeFrame::new(&gpu, Tint::Orange)); let bare = render_any(&comp, &blue, &hidden_json("null"), &resting("sculpt-bare", false)).0; + let bare_orange = render_any(&comp, &orange, &hidden_json("null"), &resting("sculpt-bare", false)).0; let mut failures = Vec::new(); for (theme, ..) in SCULPTED { for state in ["arrow", "pointer"] { @@ -783,7 +799,8 @@ fn the_sculpted_cursors_stand_at_the_hotspot() { let (hover, p) = render(&comp, &blue, &json, &still); let hover_b = render(&comp, &orange, &json, &still).0; save(&format!("sculpt-{theme}-{state}"), &hover); - let mask = opaque_mask(&hover, &hover_b, &bare); + let glass = theme == "prism-glow"; + let mask = if glass { covered_mask(&hover, &hover_b, &bare, &bare_orange) } else { opaque_mask(&hover, &hover_b, &bare) }; let (mut body, mut c, mut near) = (0usize, [0.0f32; 2], f32::MAX); for y in 0..720 { for x in 0..1280 { @@ -819,6 +836,13 @@ fn the_sculpted_cursors_stand_at_the_hotspot() { if shadow * 10 < body * 3 { failures.push(format!("{theme}/{state} : pas d'ombre en l'air ({shadow} px)")); } + if glass { + let through = (0..1280 * 720).filter(|&i| mask[i] && hover[i * 4..i * 4 + 3] != hover_b[i * 4..i * 4 + 3]).count(); + println!("{theme}/{state} : {through} px du corps laissent voir l'écran"); + if through * 10 < body * 3 { + failures.push(format!("{theme}/{state} : le cristal ne laisse voir l'écran que sur {through} px")); + } + } } } assert!(failures.is_empty(), "{failures:#?}"); @@ -865,7 +889,7 @@ fn contact_sheets() { } tilted.save(format!("{dir}/states-touch-flat-and-iso.png")).expect("planche"); - // Les dix curseurs sculptés : un thème par ligne, flèche et main en l'air sur l'écran à plat, + // Les curseurs sculptés : un thème par ligne, flèche et main en l'air sur l'écran à plat, // puis posées sur l'écran incliné. let cases = [("arrow", "null", false), ("pointer", "null", false), ("arrow", r#""iso""#, true), ("pointer", r#""iso""#, true)]; let mut sculpted = image::RgbaImage::new(4 * CELL, SCULPTED.len() as u32 * CELL); diff --git a/crates/compositor/tests/export_timing.rs b/crates/compositor/tests/export_timing.rs index 34d5e8de4d..adb90bb43d 100644 --- a/crates/compositor/tests/export_timing.rs +++ b/crates/compositor/tests/export_timing.rs @@ -117,6 +117,7 @@ fn whole_clip(dir: &PathBuf) -> ClipSource { source_end_sec: SOURCE_SEC, webcam_offset_sec: 0.0, has_audio: false, + additional_cameras: Vec::new(), } } diff --git a/crates/compositor/tests/no_camera_stand_in.rs b/crates/compositor/tests/no_camera_stand_in.rs new file mode 100644 index 0000000000..d741f12e9c --- /dev/null +++ b/crates/compositor/tests/no_camera_stand_in.rs @@ -0,0 +1,109 @@ +//! Sans caméra, la preview compose partout avec la frame écran à la place de la webcam : le +//! décodeur remplaçant n'est plus ni avancé ni recherché (`live::open_webcam_or_stand_in`). +//! +//! Couvre les chemins qui composent : le seek en pause (`present_frame`), y compris au-delà de +//! la dernière image, la lecture libre (`step`), et la recomposition à l'arrêt (`recompose`). +//! Une frame webcam nulle les ferait tous rendre `false`, preview figée. Vérifie aussi que le +//! remplaçant ne bouge pas, `seek_active` compris : composer suffit à ne rien prouver, puisqu'il +//! ouvre le même fichier valide que l'écran. +//! +//! La CI ne le joue pas : il faut un GPU et une source vidéo, sinon il se saute. N'importe quel +//! MP4 d'au moins 4 s convient : +//! +//! ```sh +//! OPENSCREEN_LIVE_SOURCE=/chemin/source.mp4 cargo test -p openscreen-compositor --test no_camera_stand_in -- --nocapture +//! ``` + +#![cfg(any(windows, target_os = "macos"))] + +use openscreen_compositor::compositor::Compositor; +use openscreen_compositor::config; +use openscreen_compositor::d3d::Gpu; +use openscreen_compositor::frame_geometry::live_params_from_scene; +use openscreen_compositor::live::Player; +use openscreen_compositor::scene::Scene; + +fn scene_json(source: &str) -> String { + let s = source.replace('\\', "/"); + format!( + r##"{{ + "clips": [{{"screenPath":"{s}","webcamPath":"","sourceStartSec":0,"sourceEndSec":100000,"webcamOffsetSec":0,"hasAudio":false}}], + "layout": {{"preset":"no-webcam","webcamSize":1.0,"webcamShape":"rounded","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false}}, + "effects": {{"padding":0.1,"blur":false,"shadow":0.0,"roundnessFrac":0,"motionBlur":0.0}}, + "background": {{"kind":"color","color":"#303030"}}, + "zoomRegions": [], + "speedRegions": [], + "cursor": {{"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0,"clipToBounds":false,"theme":"default"}}, + "cropByClip": [null], + "output": {{"width":480,"height":270,"fps":30}} + }}"## + ) +} + +#[test] +fn every_compose_path_works_without_a_camera() { + let Ok(source) = std::env::var("OPENSCREEN_LIVE_SOURCE") else { + println!("SKIP: definir OPENSCREEN_LIVE_SOURCE (voir l'en-tete du fichier)."); + return; + }; + let gpu = Gpu::create(false).expect("device"); + let mut cfg = config::all().pop().expect("au moins une config"); + cfg.zoom = false; + cfg.layout_anim = false; + let comp = Compositor::new_sized(&gpu, 480, 270).expect("compositor"); + let scene = Scene::from_json(&scene_json(&source)).expect("scene valide"); + comp.set_live_params(live_params_from_scene(&scene)); + comp.set_scene(Some(scene.clone())); + comp.clear_cursor(); + + unsafe { + let mut player = Player::open(&source, "", &gpu).expect("ouvrir la source"); + assert!(!player.webcam_decoder_is_real(), "pas de caméra déclarée : décodeur remplaçant"); + player.set_programme_clock(Some(&scene), 0); + + let stand_in_time = player.webcam_time_sec(); + + assert!(player.present_frame(&comp, &cfg, 1.0).expect("seek"), "seek en pause"); + let start = player.screen_time_sec(); + + // Lecture libre à 2× sur une seconde d'horloge, au pas de 1/60 s. À l'EOF, `step` + // reboucle à 0 : la cible ne serait alors plus jamais atteinte, d'où l'arrêt sur un + // temps qui recule. + let mut target = start; + let mut composed = 0; + for _ in 0..60 { + target += 2.0 / 60.0; + loop { + let before = player.screen_time_sec(); + if !player.step(&comp, &cfg, target).expect("step") { + break; + } + assert!( + player.screen_time_sec() >= before, + "la source a rebouclé avant {target:.2} s : il faut une source d'au moins 4 s" + ); + composed += 1; + } + } + assert!(composed > 0, "la lecture libre n'a composé aucune frame"); + assert!( + player.screen_time_sec() > start + 1.5, + "la lecture libre a avancé de {:.3} s au lieu de ~2 s", + player.screen_time_sec() - start + ); + + assert!(player.recompose(&comp, &cfg).expect("recompose"), "recomposition à l'arrêt"); + + // Au-delà de la dernière image : la dernière image, composée. + assert!(player.present_frame(&comp, &cfg, 100_000.0).expect("seek fin"), "seek au-delà de la fin"); + + assert!(player.seek_active(0.5).expect("seek_active"), "seek_active"); + assert!(player.step(&comp, &cfg, 0.5).expect("step après seek_active"), "step après seek_active"); + + assert_eq!( + player.webcam_time_sec().to_bits(), + stand_in_time.to_bits(), + "le décodeur remplaçant a bougé : il ne doit être ni avancé ni recherché" + ); + } +} diff --git a/crates/compositor/tests/shared_frame_handoff.rs b/crates/compositor/tests/shared_frame_handoff.rs new file mode 100644 index 0000000000..2465f2b983 --- /dev/null +++ b/crates/compositor/tests/shared_frame_handoff.rs @@ -0,0 +1,135 @@ +//! La texture qu'une vue live livre par handle NT, rouverte par un AUTRE device D3D11 — la +//! place qu'occupe le processus GPU de Chromium quand Electron l'importe. Ce que le +//! consommateur lit doit être exactement ce que le compositeur a composé : pas une image +//! à moitié copiée, pas une case périmée, et une case redimensionnée doit livrer un handle +//! neuf à la nouvelle taille. +//! +//! Demande un device matériel : saute sans lui (runner sans adaptateur). + +#![cfg(windows)] + +use openscreen_compositor::d3d::Gpu; +use openscreen_compositor::shared_frames::SharedRing; +use windows::core::Interface; +use windows::Win32::Foundation::{HANDLE, HMODULE}; +use windows::Win32::Graphics::Direct3D::{D3D_DRIVER_TYPE_HARDWARE, D3D_FEATURE_LEVEL_11_1}; +use windows::Win32::Graphics::Direct3D11::{ + D3D11CreateDevice, ID3D11Device, ID3D11Device1, ID3D11DeviceContext, ID3D11Texture2D, + D3D11_BIND_SHADER_RESOURCE, D3D11_CPU_ACCESS_READ, D3D11_CREATE_DEVICE_FLAG, + D3D11_MAPPED_SUBRESOURCE, D3D11_MAP_READ, D3D11_SDK_VERSION, D3D11_SUBRESOURCE_DATA, + D3D11_TEXTURE2D_DESC, D3D11_USAGE_DEFAULT, D3D11_USAGE_STAGING, +}; +use windows::Win32::Graphics::Dxgi::Common::{DXGI_FORMAT_R8G8B8A8_UNORM, DXGI_SAMPLE_DESC}; + +fn pattern(width: u32, height: u32, seed: u8) -> Vec { + (0..width * height) + .flat_map(|i| { + let (x, y) = (i % width, i / width); + [x as u8 ^ seed, y as u8, seed, 255] + }) + .collect() +} + +fn texture_with(gpu: &Gpu, width: u32, height: u32, pixels: &[u8]) -> ID3D11Texture2D { + let desc = D3D11_TEXTURE2D_DESC { + Width: width, + Height: height, + MipLevels: 1, + ArraySize: 1, + Format: DXGI_FORMAT_R8G8B8A8_UNORM, + SampleDesc: DXGI_SAMPLE_DESC { Count: 1, Quality: 0 }, + Usage: D3D11_USAGE_DEFAULT, + BindFlags: D3D11_BIND_SHADER_RESOURCE.0 as u32, + CPUAccessFlags: 0, + MiscFlags: 0, + }; + let init = D3D11_SUBRESOURCE_DATA { + pSysMem: pixels.as_ptr().cast(), + SysMemPitch: width * 4, + SysMemSlicePitch: 0, + }; + let mut tex = None; + unsafe { gpu.device.CreateTexture2D(&desc, Some(&init), Some(&mut tex)) }.expect("texture"); + tex.expect("texture") +} + +/// Le second device, sans rien de commun avec celui du compositeur que l'adaptateur. +fn consumer() -> (ID3D11Device, ID3D11DeviceContext) { + let mut device = None; + let mut context = None; + unsafe { + D3D11CreateDevice( + None, + D3D_DRIVER_TYPE_HARDWARE, + HMODULE::default(), + D3D11_CREATE_DEVICE_FLAG(0), + Some(&[D3D_FEATURE_LEVEL_11_1]), + D3D11_SDK_VERSION, + Some(&mut device), + None, + Some(&mut context), + ) + } + .expect("second device"); + (device.unwrap(), context.unwrap()) +} + +/// Ce que le consommateur lit à travers le handle, en RGBA serré. +fn read_through(handle: u64, width: u32, height: u32) -> Vec { + let (device, context) = consumer(); + let device1: ID3D11Device1 = device.cast().expect("ID3D11Device1"); + let shared: ID3D11Texture2D = + unsafe { device1.OpenSharedResource1(HANDLE(handle as *mut core::ffi::c_void)) } + .expect("le handle doit s'ouvrir depuis un autre device"); + let mut desc = D3D11_TEXTURE2D_DESC::default(); + unsafe { shared.GetDesc(&mut desc) }; + assert_eq!((desc.Width, desc.Height), (width, height)); + desc.Usage = D3D11_USAGE_STAGING; + desc.BindFlags = 0; + desc.CPUAccessFlags = D3D11_CPU_ACCESS_READ.0 as u32; + desc.MiscFlags = 0; + let mut staging = None; + unsafe { device.CreateTexture2D(&desc, None, Some(&mut staging)) }.expect("staging"); + let staging = staging.unwrap(); + let mut mapped = D3D11_MAPPED_SUBRESOURCE::default(); + let mut out = vec![0u8; (width * height * 4) as usize]; + unsafe { + context.CopyResource(&staging, &shared); + context.Map(&staging, 0, D3D11_MAP_READ, 0, Some(&mut mapped)).expect("map"); + for y in 0..height as usize { + std::ptr::copy_nonoverlapping( + (mapped.pData as *const u8).add(y * mapped.RowPitch as usize), + out.as_mut_ptr().add(y * width as usize * 4), + width as usize * 4, + ); + } + context.Unmap(&staging, 0); + } + out +} + +#[test] +fn another_device_reads_exactly_the_composed_frame() { + let Ok(gpu) = Gpu::create(false) else { + eprintln!("pas de device D3D11 matériel — saute"); + return; + }; + let mut ring = SharedRing::new(&gpu).expect("anneau"); + + let (w, h) = (64, 36); + let first = pattern(w, h, 0x11); + let handle = unsafe { ring.write(0, &texture_with(&gpu, w, h, &first), w, h) }.expect("écriture"); + assert_eq!(read_through(handle, w, h), first); + + // La même case réécrite : même texture, même handle, nouveau contenu. + let second = pattern(w, h, 0x22); + let again = unsafe { ring.write(0, &texture_with(&gpu, w, h, &second), w, h) }.expect("réécriture"); + assert_eq!(again, handle); + assert_eq!(read_through(again, w, h), second); + + // Le rendu change de taille : la case est recréée, avec un handle à la nouvelle taille. + let (w2, h2) = (32, 18); + let resized = pattern(w2, h2, 0x33); + let fresh = unsafe { ring.write(0, &texture_with(&gpu, w2, h2, &resized), w2, h2) }.expect("taille"); + assert_eq!(read_through(fresh, w2, h2), resized); +} diff --git a/crates/poc-d3d/src/bench.rs b/crates/poc-d3d/src/bench.rs index 3da91883ba..2b657b91a5 100644 --- a/crates/poc-d3d/src/bench.rs +++ b/crates/poc-d3d/src/bench.rs @@ -154,6 +154,7 @@ fn run_bench(args: &[String]) -> Result<()> { source_end_sec: 6.0, // la fixture entière (§ fixture.json : 6 s, 360 frames) webcam_offset_sec: 0.0, has_audio: false, + additional_cameras: Vec::new(), }; let path = format!("{out}/{}_{:?}.mp4", cfg.name, backend).to_lowercase(); let s = pipeline::run_composited_multi( @@ -296,6 +297,7 @@ fn run_gif_bench( source_end_sec: f64::MAX, webcam_offset_sec: 0.0, has_audio: false, + additional_cameras: Vec::new(), }]; for r in 0..repeat { // Each run writes to the same path — the last frame wins. The diff --git a/design/cursors/3d-concept.png b/design/cursors/3d-concept.png index fc828101b7..4b7a01c98a 100644 Binary files a/design/cursors/3d-concept.png and b/design/cursors/3d-concept.png differ diff --git a/design/cursors/3d-direction.md b/design/cursors/3d-direction.md index 6333c23d0a..1205d3174b 100644 --- a/design/cursors/3d-direction.md +++ b/design/cursors/3d-direction.md @@ -2,21 +2,47 @@ `3d-concept.png` is a visual reference, not a render from OpenScreen. -With the 3D option on, mode 15 ray-marches each original theme's arrow and hand -as a sculpted model: signed distance functions written in the compositor's three -shaders (HLSL, Metal, WGSL), not a PNG face or a height map. -`crates/compositor/src/sculpt.rs` names the ten models and holds their bounding -boxes and hotspots. The scene names the model (`"/"`); text, -resize, move and the other OS states keep their built-in art and the -compositor's beveled extrusion. +With the 3D option on, mode 15 ray-marches the arrow and hand of Studio Ink, Pop +Coral, Pixel Candy and Star Sprout as sculpted models: signed distance functions +written in the compositor's three shaders (HLSL, Metal, WGSL), not a PNG face or +a height map. Prism Glow's arrow and hand are glass crystals: a triangle mesh +traced on its drawing, ray-traced in a navy rim. `crates/compositor/src/sculpt.rs` +names the ten models and holds their bounding boxes and hotspots. The scene names +the model (`"/"`); text, resize, move and the other OS states keep +their built-in art and the compositor's beveled extrusion. | Theme | Arrow model | Hand model | | --- | --- | --- | -| Studio Ink | Matte black body with an ivory piping set in from the edge. | Ivory glove with a dark cuff. | -| Prism Glow | Cut crystal: flat facets, a polygonal silhouette, cyan to violet dispersion. | The same glove cut into faceted fingers and palm. | -| Pop Coral | Soft coral rubber with rounded edges and a gentle dome. | Plump yellow rubber glove with a navy cuff. | -| Pixel Candy | Beveled voxels: a pink face with mint and pale pink edges, purple blocks one step behind. | Voxel glove with the same colours and purple back layer. | -| Star Sprout | Glossy mint ceramic carrying a star with a face and two leaves. | Ivory glove with a mint cuff and the same star. | +| Studio Ink | Black rim, a raised ivory band, a recessed satin black field. | Ivory glove cushion in a black rim, black ridges between the fingers. | +| Prism Glow | A glass crystal cut along the facets of its drawing, set in a thin navy rim. The picture under it shows through, refracted with a slight dispersion; each facet glows faintly in its drawn colour, and a light line marks its folds. | The same crystal: a facet for each plane of the drawn hand, the fingers and thumb included. | +| Pop Coral | Cut paper: flat-topped sheets, no rounded bead or gloss. A coral sheet on a navy sheet, a yellow sheet behind them offset down-left, two yellow click dashes. | A yellow sheet on a navy sheet, the navy showing between the fingers, a coral sheet behind offset down-left, two coral click dashes. | +| Pixel Candy | Simplified pixel art, one cube per pixel: a plum outline, a pink body with a pale highlight down the lit edge and a darker shade along the outline. The 3D model and the 2D sprite come from the same grid. | The same, a pixel hand: the index up, three knuckles apart, the thumb. | +| Star Sprout | Glossy mint cushion framed by the navy outline of its drawing, carrying a star with a face and two leaves. | Ivory glove cushion in the same navy frame, navy ridges between the fingers, a mint cuff and the same star. | + +Studio Ink, Pop Coral and Star Sprout keep their drawn outline, as in +`3d-concept.png` (and `star-sprout/3d-reference.png` for the Star Sprout hand). +Each piece is a tray in the outline colour under its silhouette, a rounded bead +along the stroke and a puffy colour cushion inside it; the grooves between the +fingers become ridges of the outline colour. The three arrows share one polygon +and the three gloves one construction, each glove measured on its own drawing +(`RIM_POLY`, `RIM_GLOVE`). Pop Coral turns the same shapes into flat paper +sheets instead (`s_paper`). What stands in front is theme-specific: Studio +Ink's ivory band, Pop Coral's yellow and coral layers and click dashes, Star +Sprout's cuff, star, leaves and eyes. + +Prism Glow is built from its 2D drawing alone (`prism-glow/source.png`), not from +`3d-concept.png`. `prism-glow/model/` holds the mesh: `prism_geo.py` lists its +vertices, heights and facets and writes `prism.json`; `build_blend.py` builds +`prism-glow.blend` and `prism-glow.glb` from it in Blender; `export_compositor.py` +writes the mesh into `crates/compositor/src/prism_mesh.rs` and the three shaders. +The compositor ray-traces the crystal, its triangles grouped in bounding boxes that +a ray skips when it misses them: refraction per colour +channel, total internal reflection, and an exit through its flat base onto the +picture under the cursor. That picture is a copy of the composed frame taken just +before the cursor is drawn: the recording, then its privacy blurs, then the cursor, +sharp on top, so only blurred pixels show through the glass. The rim is its +silhouette extruded, a distance field +like the other models: it gives the coverage and the shadows. All models share one close lamp: a gradient and a highlight even on flat faces, soft self shadows, ambient occlusion and a studio environment in the @@ -27,4 +53,8 @@ click and size changes. To change a model, edit it in all three shaders (`sculpt_proto`, `sculpt_material`) and keep `sculpt.rs` in step: its boxes must contain the shapes, and a test checks that the shaders mirror its constants. Pixel Candy's -arrow voxels are sampled from the arrow polygon; `sculpt.rs` says how. +voxel grids are tables written by `scripts/generate-pixel-candy-voxels.mjs`: +change a grid there and run the script; it rewrites all three shaders, and a +test checks the tables agree. To change Prism Glow, edit `prism_geo.py`, run it, +run `build_blend.py` in Blender, then `export_compositor.py`; a test checks that +the three shaders carry the mesh of `prism_mesh.rs`. diff --git a/design/cursors/README.md b/design/cursors/README.md index b3a21cd3c1..c6cd471c4f 100644 --- a/design/cursors/README.md +++ b/design/cursors/README.md @@ -18,14 +18,17 @@ fringe pixels, and resize them into transparent 128 × 128 PNGs under `public/cursors//`. It prints the normalized tip hotspots; copy them into `src/lib/cursor/cursorThemes.ts` after changing a master. -With the 3D option on, the compositor draws each theme's arrow and hand as a -model sculpted in its shaders (`crates/compositor/src/sculpt.rs`), not from these -PNGs. The theme picker and the flat cursor keep using the PNGs. Text, resize, -move and other states remain state-accurate through the built-in art and -receive the shared 3D extrusion. +With the 3D option on, the compositor draws the arrow and hand of Studio Ink, +Pop Coral, Pixel Candy and Star Sprout as models sculpted in its shaders +(`crates/compositor/src/sculpt.rs`), not from these PNGs. Prism Glow's arrow and +hand are glass crystals meshed on its drawing (`prism-glow/model/`). The theme +picker and the flat cursor keep using the PNGs. Text, resize, move and other +states remain state-accurate through the built-in art and receive the shared 3D +extrusion. `contact-sheet.png` shows the sprites enlarged on a light background; `dark-32px.png` shows them at their 32-pixel reference size on a dark background. -The 3D modeling direction and concept image are in `3d-direction.md` and -`3d-concept.png`. +The 3D modeling direction and the reference sheet the models follow are in +`3d-direction.md` and `3d-concept.png`; `star-sprout/3d-reference.png` is a closer +view of the Star Sprout hand. The consolidated acceptance list is in `requirements.md`. diff --git a/design/cursors/pixel-candy/source.png b/design/cursors/pixel-candy/source.png index 79b5c859dd..daeed19bea 100644 Binary files a/design/cursors/pixel-candy/source.png and b/design/cursors/pixel-candy/source.png differ diff --git a/design/cursors/prism-glow/model/build_blend.py b/design/cursors/prism-glow/model/build_blend.py new file mode 100644 index 0000000000..4a941e04c5 --- /dev/null +++ b/design/cursors/prism-glow/model/build_blend.py @@ -0,0 +1,265 @@ +"""Build the Prism Glow cursor in Blender from prism.json, then save prism-glow.blend and export +prism-glow.glb next to this script. + + blender --background --python build_blend.py + +Per cursor, two objects: the faceted crystal (a closed mesh, one flat face per facet of the 2D art, +with its colour in the face attribute `concept`) and the navy rim (a filled 2D curve with a hole, +extruded and rounded). Units: 1 = 1000 px of source.png, origin on the hotspot, y up, z toward the +viewer, rim top at z = 0, everything's underside at z = -0.04. +""" + +import json +import math +import os + +import bmesh +import bpy +from mathutils import Matrix, Vector + +HERE = globals().get("HERE") or os.path.dirname(os.path.abspath(__file__)) +S = 0.001 +T_RIM = 40.0 # rim and crystal-body thickness, px +BEVEL = 5.0 # rim edge rounding, px +HAND_AT = ((1220 - 208) * S, -(50 - 42) * S) # the hand stands where it is in source.png + +D = json.load(open(os.path.join(HERE, "prism.json"))) + + +def srgb_to_linear(c): + return c / 12.92 if c <= 0.04045 else ((c + 0.055) / 1.055) ** 2.4 + + +def hex_rgba(h): + return tuple(srgb_to_linear(int(h[i : i + 2], 16) / 255) for i in (1, 3, 5)) + (1.0,) + + +def signed_area(pts): + return 0.5 * sum(a[0] * b[1] - b[0] * a[1] for a, b in zip(pts, pts[1:] + pts[:1])) + + +def model_xy(sh, x, y): + hx, hy = sh["hotspot"] + return ((x - hx) * S, -(y - hy) * S) + + +def fresh_collection(name): + col = bpy.data.collections.get(name) or bpy.data.collections.new(name) + if col.name not in bpy.context.scene.collection.children: + bpy.context.scene.collection.children.link(col) + for ob in list(col.objects): + bpy.data.objects.remove(ob) + return col + + +def build_crystal(name, sh): + V = sh["verts"] + bm = bmesh.new() + concept = bm.faces.layers.float_color.new("concept") + top = {k: bm.verts.new((*model_xy(sh, x, y), z * S)) for k, (x, y, z) in V.items()} + outline = list(sh["outline"]) + if signed_area([model_xy(sh, *V[k][:2]) for k in outline]) < 0: + outline.reverse() + bot = {k: bm.verts.new((*model_xy(sh, *V[k][:2]), -T_RIM * S)) for k in outline} + for f, hexcol in zip(sh["facets"], sh["colors"]): + vs = [top[k] for k in f] + if signed_area([v.co.xy[:] for v in vs]) < 0: + vs.reverse() + bm.faces.new(vs)[concept] = hex_rgba(hexcol) + side = hex_rgba("#1a3cc8") + for a, b in zip(outline, outline[1:] + outline[:1]): + bm.faces.new((top[a], bot[a], bot[b], top[b]))[concept] = side + bm.faces.new([bot[k] for k in reversed(outline)])[concept] = side + bmesh.ops.triangulate(bm, faces=[f for f in bm.faces if len(f.verts) > 3], + quad_method="BEAUTY", ngon_method="BEAUTY") + bmesh.ops.recalc_face_normals(bm, faces=bm.faces[:]) + me = bpy.data.meshes.get(name) or bpy.data.meshes.new(name) + bm.to_mesh(me) + bm.free() + # glTF carries colours per corner: mirror the face colour there + corner = me.attributes.get("Col") or me.attributes.new("Col", "FLOAT_COLOR", "CORNER") + for p in me.polygons: + p.use_smooth = False + for li in p.loop_indices: + corner.data[li].color = me.attributes["concept"].data[p.index].color + me.color_attributes.active_color = corner + return bpy.data.objects.new(name, me) + + +def inset(loop, d): + """Offset a closed counter-clockwise loop inward by d (mitred).""" + out = [] + for i in range(len(loop)): + p0, p1, p2 = Vector(loop[i - 1]), Vector(loop[i]), Vector(loop[(i + 1) % len(loop)]) + e0, e1 = (p1 - p0).normalized(), (p2 - p1).normalized() + n0, n1 = Vector((-e0.y, e0.x)), Vector((-e1.y, e1.x)) + m = (n0 + n1).normalized() if (n0 + n1).length > 1e-6 else n0 + out.append(tuple(p1 + m * (d / max(m.dot(n0), 0.25)))) + return out + + +def build_rim(name, sh): + outer = [model_xy(sh, x, y) for x, y in sh["silhouette"]] + if signed_area(outer) < 0: + outer.reverse() + inner = [model_xy(sh, *sh["verts"][k][:2]) for k in sh["outline"]] + # the bevel rounds outward from the spline: pull the outer loop in so the silhouette stays exact + outer = inset(outer, BEVEL * S) + cu = bpy.data.curves.get(name) or bpy.data.curves.new(name, "CURVE") + cu.splines.clear() + cu.dimensions = "2D" + cu.fill_mode = "BOTH" + cu.extrude = (T_RIM / 2 - BEVEL) * S + cu.bevel_depth = BEVEL * S + cu.bevel_resolution = 3 + for loop in (outer, inner): + sp = cu.splines.new("POLY") + sp.points.add(len(loop) - 1) + for p, (x, y) in zip(sp.points, loop): + p.co = (x, y, 0.0, 1.0) + sp.use_cyclic_u = True + ob = bpy.data.objects.new(name, cu) + ob.location.z = -T_RIM / 2 * S + return ob + + +def nodes_of(name): + m = bpy.data.materials.get(name) or bpy.data.materials.new(name) + m.use_nodes = True + m.node_tree.nodes.clear() + return m, m.node_tree, m.node_tree.nodes.new("ShaderNodeOutputMaterial") + + +def material_crystal(): + """Clear glass whose facets glow with their colour in the art (the look chosen on 30/09/2026).""" + m, nt, out = nodes_of("Prism Crystal") + bsdf = nt.nodes.new("ShaderNodeBsdfPrincipled") + attr = nt.nodes.new("ShaderNodeAttribute") + attr.attribute_name = "concept" + nt.links.new(attr.outputs["Color"], bsdf.inputs["Emission Color"]) + bsdf.inputs["Base Color"].default_value = (1, 1, 1, 1) + bsdf.inputs["Emission Strength"].default_value = 0.45 + bsdf.inputs["Transmission Weight"].default_value = 1.0 + bsdf.inputs["IOR"].default_value = 1.61 + bsdf.inputs["Roughness"].default_value = 0.0 + nt.links.new(bsdf.outputs["BSDF"], out.inputs["Surface"]) + return m + + +def material_rim(): + m, nt, out = nodes_of("Prism Rim") + bsdf = nt.nodes.new("ShaderNodeBsdfPrincipled") + bsdf.inputs["Base Color"].default_value = hex_rgba("#040c5a") + bsdf.inputs["Roughness"].default_value = 0.22 + bsdf.inputs["Coat Weight"].default_value = 1.0 + bsdf.inputs["Coat Roughness"].default_value = 0.05 + nt.links.new(bsdf.outputs["BSDF"], out.inputs["Surface"]) + return m + + +def build(): + col = fresh_collection("Prism Glow") + crystal, rim = material_crystal(), material_rim() + obs = [] + for key, at in (("arrow", (0.0, 0.0)), ("hand", HAND_AT)): + c = build_crystal(f"prism_{key}_crystal", D[key]) + r = build_rim(f"prism_{key}_rim", D[key]) + c.data.materials.clear() + c.data.materials.append(crystal) + r.data.materials.clear() + r.data.materials.append(rim) + for ob in (c, r): + ob.location.x, ob.location.y = at + col.objects.link(ob) + obs += [c, r] + return obs + + +def studio(): + """Studio HDRI for light and reflections, the editor screenshot as the screen, a 3/4 camera.""" + sc = bpy.context.scene + w = sc.world or bpy.data.worlds.new("World") + sc.world = w + w.use_nodes = True + nt = w.node_tree + nt.nodes.clear() + env = nt.nodes.new("ShaderNodeTexEnvironment") + env.image = bpy.data.images.load(os.path.join(bpy.utils.system_resource("DATAFILES"), "studiolights", "world", "studio.exr"), check_existing=True) + env.image.pack() # the install path differs per machine; 100 KB inside the .blend does not + bg, flat = nt.nodes.new("ShaderNodeBackground"), nt.nodes.new("ShaderNodeBackground") + flat.inputs["Color"].default_value = (0.06, 0.06, 0.07, 1) + lp, mix, out = nt.nodes.new("ShaderNodeLightPath"), nt.nodes.new("ShaderNodeMixShader"), nt.nodes.new("ShaderNodeOutputWorld") + nt.links.new(env.outputs["Color"], bg.inputs["Color"]) + nt.links.new(lp.outputs["Is Camera Ray"], mix.inputs["Fac"]) + nt.links.new(bg.outputs["Background"], mix.inputs[1]) + nt.links.new(flat.outputs["Background"], mix.inputs[2]) + nt.links.new(mix.outputs["Shader"], out.inputs["Surface"]) + + shot = os.path.normpath(os.path.join(HERE, "..", "..", "..", "..", "public", "preview4.png")) + screen = bpy.data.objects.get("screen") + if screen is None and os.path.exists(shot): + me = bpy.data.meshes.new("screen") + me.from_pydata([(-3, -1.66, 0), (3, -1.66, 0), (3, 1.66, 0), (-3, 1.66, 0)], [], [(0, 1, 2, 3)]) + uv = me.uv_layers.new() + for li, st in enumerate(((0, 0), (1, 0), (1, 1), (0, 1))): + uv.data[li].uv = st + m, mnt, mout = nodes_of("screen") + em, tex = mnt.nodes.new("ShaderNodeEmission"), mnt.nodes.new("ShaderNodeTexImage") + tex.image = bpy.data.images.load(shot, check_existing=True) + mnt.links.new(tex.outputs["Color"], em.inputs["Color"]) + mnt.links.new(em.outputs["Emission"], mout.inputs["Surface"]) + me.materials.append(m) + screen = bpy.data.objects.new("screen", me) + screen.location = (1.9, -1.0, -T_RIM * S - 0.0005) + sc.collection.objects.link(screen) + + target = Vector((0.679, -0.43, 0.0)) + cam = bpy.data.objects.get("cam_34") or bpy.data.objects.new("cam_34", bpy.data.cameras.new("cam_34")) + if cam.name not in sc.collection.objects: + sc.collection.objects.link(cam) + eye = target + Matrix.Rotation(math.radians(-28), 3, "Y") @ Matrix.Rotation(math.radians(-38), 3, "X") @ Vector((0, 0, 4.4)) + f = (target - eye).normalized() + r = f.cross(Vector((0, 1, 0))).normalized() + cam.location = eye + cam.rotation_euler = Matrix((r, r.cross(f), -f)).transposed().to_euler() + cam.data.lens = 85 + sc.camera = cam + sc.render.engine = "CYCLES" + sc.cycles.device = "GPU" + sc.cycles.samples = 128 + sc.cycles.transmission_bounces = sc.cycles.max_bounces = 24 + sc.render.resolution_x, sc.render.resolution_y = 1600, 1000 + sc.view_settings.view_transform = "AgX" + sc.view_settings.look = "AgX - Medium High Contrast" + + +def export_glb(obs, path): + """The rims are curves; glTF wants meshes: bake them into temporary twins for the export.""" + dg = bpy.context.evaluated_depsgraph_get() + twins = [] + for ob in obs: + if ob.type == "CURVE": + name = ob.name + me = bpy.data.meshes.new_from_object(ob.evaluated_get(dg)) + ob.name = name + "_curve" # the twin takes the name the glTF node should carry + tw = bpy.data.objects.new(name, me) + tw.matrix_world = ob.matrix_world + me.materials.append(ob.data.materials[0]) + bpy.context.scene.collection.objects.link(tw) + twins.append((tw, ob, name)) + bpy.ops.object.select_all(action="DESELECT") + for ob in [o for o in obs if o.type == "MESH"] + [tw for tw, _, _ in twins]: + ob.select_set(True) + bpy.ops.export_scene.gltf(filepath=path, export_format="GLB", use_selection=True, + export_vertex_color="ACTIVE", export_apply=True) + for tw, ob, name in twins: + me = tw.data + bpy.data.objects.remove(tw) + bpy.data.meshes.remove(me) + ob.name = name + + +obs = build() +studio() +export_glb(obs, os.path.join(HERE, "prism-glow.glb")) +bpy.ops.wm.save_as_mainfile(filepath=os.path.join(HERE, "prism-glow.blend"), relative_remap=True) diff --git a/design/cursors/prism-glow/model/export_compositor.py b/design/cursors/prism-glow/model/export_compositor.py new file mode 100644 index 0000000000..15cd261afe --- /dev/null +++ b/design/cursors/prism-glow/model/export_compositor.py @@ -0,0 +1,363 @@ +"""Write the Prism Glow meshes into the compositor: `crates/compositor/src/prism_mesh.rs` and the +matching tables in the three shaders, between their `prism mesh` markers. + + python export_compositor.py # after build_blend.py has written prism-glow.glb + +Everything is expressed in the mode 15 model frame: the unit is the side of the theme's 128 px +sprite (the flat cursor's size), origin on the hotspot, x right, y DOWN, z toward the camera, +rim top at z = 0. The source-to-sprite transform is the one scripts/generate-original-cursor-themes.mjs +applies, so the crystal lands exactly on the flat cursor. +""" + +import json +import math +import struct +from pathlib import Path + +import cv2 +import numpy as np + +HERE = Path(__file__).resolve().parent +REPO = HERE.parents[3] +SRC = REPO / "crates" / "compositor" / "src" +SPRITE, INNER, ALPHA_MIN = 128, 112, 28 # generate-original-cursor-themes.mjs +BEVEL_PX = 12.0 # rounding of the rim's edges, source px +EDGE_FOLD_DEG = 25.0 +LEAF = 8 # triangles per bounding box at most +BOX_PAD = 1e-3 # model units: past prism_tri's edge slack + + +def glb_meshes(path): + """{node name: (positions (n,3), triangle indices (m,3), colours (n,3))}, glTF frame (y up).""" + b = path.read_bytes() + n = struct.unpack_from("= ALPHA_MIN + ys, xs = np.nonzero(a) + w, h = xs.max() - xs.min() + 1, ys.max() - ys.min() + 1 + scale = INNER / max(w, h) + tx, ty = (SPRITE - w * scale) / 2, (SPRITE - h * scale) / 2 + hx = (tx + (hotspot[0] - x0 - xs.min()) * scale) / SPRITE + hy = (ty + (hotspot[1] - ys.min()) * scale) / SPRITE + return scale / SPRITE, (hx, hy), ty / SPRITE + + +def crystal_triangles(pos, tri, col, sheet, s): + """Triangles of the crystal (facets and walls; the flat bottom is left to the shader), each + with its outward normal, its colour in the art and which of its edges are real fold lines.""" + # glTF (x, y up, z) -> Blender (x, y, z up), hotspot at the origin, then the model frame + blender = np.c_[pos[:, 0], -pos[:, 2], pos[:, 1]] - np.array([sheet[0], sheet[1], 0.0]) + model = np.c_[blender[:, 0], -blender[:, 1], blender[:, 2]] * (1000.0 * s) + zmin = model[:, 2].min() + tris = [] + for t in tri: + a, b, c = model[t] + nb = np.cross(blender[t[1]] - blender[t[0]], blender[t[2]] - blender[t[0]]) + nb /= np.linalg.norm(nb) + n = np.array([nb[0], -nb[1], nb[2]]) # the y mirror keeps the normal outward + if n[2] < -0.9 and abs(a[2] - zmin) < 1e-6: + continue + tris.append({"v": (a, b, c), "n": n, "col": col[t[0]], "key": tuple(map(tuple, np.round(model[t], 6)))}) + # real edges: the mesh's boundary, a change of facet colour, or a sharp fold + owners = {} + for i, t in enumerate(tris): + for e in range(3): + k = tuple(sorted((t["key"][(e + 1) % 3], t["key"][(e + 2) % 3]))) + owners.setdefault(k, []).append(i) + for i, t in enumerate(tris): + bits = 0 + for e in range(3): + k = tuple(sorted((t["key"][(e + 1) % 3], t["key"][(e + 2) % 3]))) + other = [j for j in owners[k] if j != i] + real = not other or np.abs(tris[other[0]]["col"] - t["col"]).sum() > 1e-3 or ( + tris[other[0]]["n"] @ t["n"] < math.cos(math.radians(EDGE_FOLD_DEG))) + bits |= int(real) << e + t["flags"] = (1 if abs(t["n"][2]) < 0.2 else 0) + 2 * bits + return tris, float(zmin), float(model[:, 2].max()), model + + +def cluster(tris): + """The triangles regrouped by neighbourhood, and one bounding box per group: a ray that + misses a box skips all its triangles. Groups are halves of the centroids' longest extent, + split until they hold LEAF triangles at most; the triangles come back in group order.""" + groups = [] + + def split(idx): + if len(idx) <= LEAF: + groups.append(idx) + return + cent = np.array([np.mean(tris[i]["v"], axis=0) for i in idx]) + axis = int(np.argmax(cent.max(0) - cent.min(0))) + order = [idx[j] for j in np.argsort(cent[:, axis], kind="stable")] + split(order[: len(order) // 2]) + split(order[len(order) // 2 :]) + + split(list(range(len(tris)))) + ordered, leaves = [], [] + for g in groups: + pts = np.array([v for i in g for v in tris[i]["v"]]) + leaves.append((pts.min(0) - BOX_PAD, pts.max(0) + BOX_PAD, len(ordered), len(g))) + ordered += [tris[i] for i in g] + return ordered, leaves + + +def main(): + img = cv2.imread(str(HERE.parent / "source.png"), cv2.IMREAD_UNCHANGED) + geo = json.loads((HERE / "prism.json").read_text()) + meshes = glb_meshes(HERE / "prism-glow.glb") + shapes = [] + for kind, node, (x0, x1), sheet in ( + ("arrow", "prism_arrow_crystal", (0, 887), (0.0, 0.0)), + ("hand", "prism_hand_crystal", (887, 1774), ((1220 - 208) / 1000, -(50 - 42) / 1000)), + ): + hot = geo[kind]["hotspot"] + s, (hx, hy), top = sprite_transform(img, x0, x1, hot) + tris, zmin, zmax, model = crystal_triangles(*meshes[node], sheet, s) + tris, leaves = cluster(tris) + sil = [((x - hot[0]) * s, (y - hot[1]) * s) for x, y in geo[kind]["silhouette"]] + out = [((geo[kind]["verts"][k][0] - hot[0]) * s, (geo[kind]["verts"][k][1] - hot[1]) * s) + for k in geo[kind]["outline"]] + sx = [p[0] for p in sil] + # the draw box: the sprite's full height (1 unit), the silhouette's width + bx0, bx1 = min(sx) - 0.01, max(sx) + 0.01 + shapes.append({ + "kind": kind, "tris": tris, "leaves": leaves, "sil": sil, "out": out, "thick": -zmin, + "height": zmax, + "size": (bx1 - bx0, 1.0), "hotspot": (-bx0 / (bx1 - bx0), hy), "top": top, + "per_px": s, + }) + print(f"{kind}: {len(tris)} triangles in {len(leaves)} boxes, {len(sil)} silhouette points, " + f"box {bx1 - bx0:.4f} x 1, hotspot ({-bx0 / (bx1 - bx0):.4f}, {hy:.4f}), " + f"height {zmax:.4f}, thick {-zmin:.4f}") + write_rust(shapes) + write_shaders(shapes) + + +def f(x): + return f"{x:.6f}".rstrip("0").rstrip(".") if abs(x) > 5e-7 else "0.0" + + +def fl(x): # a float literal every language reads as a float + s = f(x) + return s if "." in s else s + ".0" + + +def bevel(shapes): + return BEVEL_PX * shapes[0]["per_px"] + + +def polygons(shapes): + """Silhouettes, then outlines, in one table; their starts.""" + poly, sil_start, out_start = [], [], [] + for sh in shapes: + sil_start.append(len(poly)) + poly += sh["sil"] + for sh in shapes: + out_start.append(len(poly)) + poly += sh["out"] + return poly, sil_start, out_start + + +def records(shapes): + """Per triangle, four float4: (v0, flags), (e1, r), (e2, g), (n, b).""" + rows = [] + for sh in shapes: + for t in sh["tris"]: + a, b, c = t["v"] + e1, e2 = b - a, c - a + r, g, bl = t["col"] + rows += [(*a, t["flags"]), (*e1, r), (*e2, g), (*t["n"], bl)] + return rows + + +def boxes(shapes): + """Per box, two float4: (lo, first triangle), (hi, triangle count); each model's first box.""" + rows, box_start, k = [], [], 0 + for sh in shapes: + box_start.append(len(rows) // 2) + for lo, hi, first, count in sh["leaves"]: + rows += [(*lo, k + first), (*hi, count)] + k += len(sh["tris"]) + return rows, box_start + + +def write_rust(shapes): + tri_start, rows = [], records(shapes) + k = 0 + for sh in shapes: + tri_start.append(k) + k += len(sh["tris"]) + poly, sil_start, out_start = polygons(shapes) + box_rows, box_start = boxes(shapes) + lines = [ + "//! Les deux curseurs de Prism Glow en MAILLAGE : les facettes tracées sur l'art 2D", + "//! (`design/cursors/prism-glow/model`), taillées en cristal. Fichier GÉNÉRÉ par", + "//! `design/cursors/prism-glow/model/export_compositor.py` : ne pas l'éditer à la main.", + "//!", + "//! Repère du modèle du mode 15 : unité = côté du sprite 128 px du thème, origine au hotspot,", + "//! x à droite, y vers le bas, z vers la caméra, dessus du serti en z = 0. Chaque triangle du", + "//! cristal tient en quatre float4 : (v0, drapeaux), (e1, r), (e2, g), (normale, b) ; drapeaux =", + "//! paroi (1) + 2 × arêtes réelles (bit k : l'arête opposée au sommet k). Le fond plat, en", + "//! z = -épaisseur, est laissé au shader. `POLYS` tient les silhouettes (bord extérieur du", + "//! serti) puis les contours du cristal (son bord intérieur). `BOXES` range les triangles", + "//! par voisinage : deux float4 par boîte, (coin bas, premier triangle), (coin haut, nombre).", + "", + "/// Un des deux modèles : ses triangles et sa silhouette dans les tables, sa boîte de dessin.", + "pub struct PrismModel {", + " pub tri_start: usize,", + " pub tri_count: usize,", + " pub sil_start: usize,", + " pub sil_count: usize,", + " pub out_start: usize,", + " pub out_count: usize,", + " pub box_start: usize,", + " pub box_count: usize,", + " /// Largeur de la boîte, hauteur 1 (le sprite), en unités du modèle.", + " pub size: [f32; 2],", + " /// Hotspot, fraction de la boîte.", + " pub hotspot: [f32; 2],", + " /// Haut de la silhouette, fraction de la hauteur.", + " pub top: f32,", + " /// Sommet du cristal au-dessus de z = 0, et épaisseur sous z = 0.", + " pub height: f32,", + " pub thick: f32,", + "}", + "", + "/// Arrondi des arêtes du serti, en unités du modèle (`PRISM_BEVEL` des shaders).", + f"pub const BEVEL: f32 = {fl(bevel(shapes))};", + "", + "/// Flèche puis main, dans l'ordre des identifiants du shader.", + "pub const MODELS: [PrismModel; 2] = [", + ] + for i, sh in enumerate(shapes): + lines += [ + " PrismModel {", + f" tri_start: {tri_start[i]},", + f" tri_count: {len(sh['tris'])},", + f" sil_start: {sil_start[i]},", + f" sil_count: {len(sh['sil'])},", + f" out_start: {out_start[i]},", + f" out_count: {len(sh['out'])},", + f" box_start: {box_start[i]},", + f" box_count: {len(sh['leaves'])},", + f" size: [{fl(sh['size'][0])}, {fl(sh['size'][1])}],", + f" hotspot: [{fl(sh['hotspot'][0])}, {fl(sh['hotspot'][1])}],", + f" top: {fl(sh['top'])},", + f" height: {fl(sh['height'])},", + f" thick: {fl(sh['thick'])},", + " },", + ] + lines += ["];", "", f"pub const TRIS: [[f32; 4]; {len(rows)}] = ["] + lines += [f" [{', '.join(fl(v) for v in r)}]," for r in rows] + lines += ["];", "", f"pub const POLYS: [[f32; 2]; {len(poly)}] = ["] + lines += [f" [{fl(x)}, {fl(y)}]," for x, y in poly] + lines += ["];", "", f"pub const BOXES: [[f32; 4]; {len(box_rows)}] = ["] + lines += [f" [{', '.join(fl(v) for v in r)}]," for r in box_rows] + lines += ["];", ""] + (SRC / "prism_mesh.rs").write_text("\n".join(lines), encoding="utf-8", newline="\n") + + +def table_lines(shapes, lang): + rows = records(shapes) + poly, sil_start, out_start = polygons(shapes) + box_rows, box_start = boxes(shapes) + bcounts = [len(sh["leaves"]) for sh in shapes] + counts = [len(sh["tris"]) for sh in shapes] + scounts = [len(sh["sil"]) for sh in shapes] + ocounts = [len(sh["out"]) for sh in shapes] + if lang == "hlsl": + v4 = lambda r: f"float4({', '.join(fl(v) for v in r)})" + v2 = lambda p: f"float2({fl(p[0])}, {fl(p[1])})" + head4 = f"static const float4 PRISM_TRIS[{len(rows)}] = {{" + headb = f"static const float4 PRISM_BOXES[{len(box_rows)}] = {{" + head2 = f"static const float2 PRISM_POLY[{len(poly)}] = {{" + tail = "};" + ints = lambda name, vals: f"static const int {name}[2] = {{ {vals[0]}, {vals[1]} }};" + scalar = f"static const float PRISM_BEVEL = {fl(bevel(shapes))};" + elif lang == "metal": + v4 = lambda r: f"float4({', '.join(fl(v) for v in r)})" + v2 = lambda p: f"float2({fl(p[0])}, {fl(p[1])})" + head4 = f"constant float4 PRISM_TRIS[{len(rows)}] = {{" + headb = f"constant float4 PRISM_BOXES[{len(box_rows)}] = {{" + head2 = f"constant float2 PRISM_POLY[{len(poly)}] = {{" + tail = "};" + ints = lambda name, vals: f"constant int {name}[2] = {{ {vals[0]}, {vals[1]} }};" + scalar = f"constant float PRISM_BEVEL = {fl(bevel(shapes))};" + else: + ints = lambda name, vals: f"var {name} = array({vals[0]}, {vals[1]});" + scalar = f"const PRISM_BEVEL: f32 = {fl(bevel(shapes))};" + out = [ints("PRISM_TRI_START", [0, counts[0]]), ints("PRISM_TRI_COUNT", counts), + ints("PRISM_SIL_START", sil_start), ints("PRISM_SIL_COUNT", scounts), + ints("PRISM_OUT_START", out_start), ints("PRISM_OUT_COUNT", ocounts), + ints("PRISM_BOX_START", box_start), ints("PRISM_BOX_COUNT", bcounts)] + if lang == "wgsl": + # The triangles, polygons and boxes themselves go in a uniform buffer that compositor_linux.rs + # fills from prism_mesh.rs (a polygon point per vec4, the stride of a uniform array). As + # a `const` or `var` table, lavapipe copies them into every invocation of every + # layer: a frame without any cursor rendered 3.4 times slower. + return out + ["struct PrismMesh {", f" tris: array, {len(rows)}>,", + f" poly: array, {len(poly)}>,", + f" boxes: array, {len(box_rows)}>,", "}", scalar] + out.append(head4) + out += [f" {v4(r)}," for r in rows] + out[-1] = out[-1].rstrip(",") + out += [tail, headb] + out += [f" {v4(r)}," for r in box_rows] + out[-1] = out[-1].rstrip(",") + out += [tail, head2] + out += [f" {v2(p)}," for p in poly] + out[-1] = out[-1].rstrip(",") + out += [tail, scalar] + return out + + +BEGIN = "prism mesh: generated by design/cursors/prism-glow/model/export_compositor.py" +END = "end of the prism mesh" + + +def write_shaders(shapes): + for name, lang in (("shaders.hlsl", "hlsl"), ("shaders.metal", "metal"), ("vk_shaders/layer.wgsl", "wgsl")): + path = SRC / name + text = path.read_text(encoding="utf-8") + a, b = text.find(BEGIN), text.find(END) + if a < 0 or b < 0: + print(f"{name}: no markers yet, tables not written") + continue + start = text.index("\n", a) + 1 + stop = text.rindex("\n", 0, b) + 1 + text = text[:start] + "\n".join(table_lines(shapes, lang)) + "\n" + text[stop:] + path.write_text(text, encoding="utf-8", newline="\n") + print(f"{name}: tables written") + + +if __name__ == "__main__": + main() diff --git a/design/cursors/prism-glow/model/prism-glow.blend b/design/cursors/prism-glow/model/prism-glow.blend new file mode 100644 index 0000000000..8edc4553b1 Binary files /dev/null and b/design/cursors/prism-glow/model/prism-glow.blend differ diff --git a/design/cursors/prism-glow/model/prism-glow.glb b/design/cursors/prism-glow/model/prism-glow.glb new file mode 100644 index 0000000000..2aaa2d145b Binary files /dev/null and b/design/cursors/prism-glow/model/prism-glow.glb differ diff --git a/design/cursors/prism-glow/model/prism.json b/design/cursors/prism-glow/model/prism.json new file mode 100644 index 0000000000..5aae7a713f --- /dev/null +++ b/design/cursors/prism-glow/model/prism.json @@ -0,0 +1,985 @@ +{ + "arrow": { + "hotspot": [ + 208, + 42 + ], + "verts": { + "A": [ + 237, + 100, + 0.0 + ], + "B": [ + 443, + 290, + 0.0 + ], + "F": [ + 712, + 549, + 0.0 + ], + "K": [ + 543, + 563, + 0.0 + ], + "J": [ + 612, + 744, + 0.0 + ], + "I": [ + 492, + 807, + 0.0 + ], + "H": [ + 394, + 597, + 0.0 + ], + "G": [ + 241, + 732, + 0.0 + ], + "D": [ + 241, + 491, + 0.0 + ], + "C": [ + 366, + 374, + 108.0 + ], + "E": [ + 458, + 539, + 76.0 + ], + "X": [ + 515, + 449, + 66.0 + ] + }, + "outline": [ + "A", + "B", + "F", + "K", + "J", + "I", + "H", + "G", + "D" + ], + "facets": [ + [ + "A", + "C", + "D" + ], + [ + "A", + "B", + "C" + ], + [ + "B", + "X", + "C" + ], + [ + "B", + "F", + "X" + ], + [ + "C", + "X", + "E" + ], + [ + "X", + "F", + "E" + ], + [ + "E", + "F", + "K" + ], + [ + "C", + "E", + "H" + ], + [ + "C", + "H", + "G" + ], + [ + "C", + "G", + "D" + ], + [ + "E", + "I", + "H" + ], + [ + "E", + "J", + "I" + ], + [ + "E", + "K", + "J" + ] + ], + "colors": [ + "#01f8ee", + "#cadffc", + "#a643fc", + "#c348fc", + "#033df4", + "#002cc0", + "#124efb", + "#00208a", + "#001b80", + "#00b7fb", + "#0022b7", + "#008afc", + "#00fbfb" + ], + "silhouette": [ + [ + 214, + 40 + ], + [ + 207, + 43 + ], + [ + 204, + 49 + ], + [ + 204, + 793 + ], + [ + 207, + 796 + ], + [ + 212, + 796 + ], + [ + 229, + 784 + ], + [ + 381, + 653 + ], + [ + 470, + 846 + ], + [ + 475, + 853 + ], + [ + 481, + 854 + ], + [ + 642, + 770 + ], + [ + 654, + 762 + ], + [ + 654, + 753 + ], + [ + 588, + 594 + ], + [ + 628, + 588 + ], + [ + 768, + 576 + ], + [ + 773, + 572 + ], + [ + 773, + 568 + ], + [ + 768, + 560 + ], + [ + 675, + 464 + ], + [ + 604, + 393 + ], + [ + 478, + 272 + ], + [ + 284, + 96 + ], + [ + 221, + 42 + ] + ] + }, + "hand": { + "hotspot": [ + 1220, + 50 + ], + "verts": { + "I1": [ + 1180, + 122, + 0.0 + ], + "I2": [ + 1221, + 85, + 0.0 + ], + "I3": [ + 1261, + 90, + 0.0 + ], + "I4": [ + 1290, + 132, + 0.0 + ], + "IT1": [ + 1224, + 118, + 46.0 + ], + "IT2": [ + 1274, + 135, + 35.0 + ], + "IT3": [ + 1191, + 262, + 54.0 + ], + "IL": [ + 1179, + 265, + 0.0 + ], + "P1": [ + 1292, + 427, + 0.0 + ], + "W1": [ + 1321, + 456, + 0.0 + ], + "M1": [ + 1323, + 334, + 0.0 + ], + "M2": [ + 1356, + 302, + 0.0 + ], + "M3": [ + 1395, + 309, + 0.0 + ], + "M4": [ + 1415, + 340, + 0.0 + ], + "MT1": [ + 1354, + 334, + 46.0 + ], + "MT2": [ + 1390, + 344, + 35.0 + ], + "P2": [ + 1339, + 479, + 59.0 + ], + "MB1": [ + 1386, + 481, + 54.0 + ], + "MB2": [ + 1415, + 462, + 0.0 + ], + "RL": [ + 1441, + 467, + 0.0 + ], + "R1": [ + 1443, + 368, + 0.0 + ], + "R2": [ + 1476, + 338, + 0.0 + ], + "R3": [ + 1515, + 345, + 0.0 + ], + "R4": [ + 1536, + 377, + 0.0 + ], + "RT1": [ + 1480, + 367, + 43.0 + ], + "RT2": [ + 1513, + 381, + 34.0 + ], + "RB": [ + 1448, + 475, + 57.0 + ], + "RB1": [ + 1476, + 498, + 54.0 + ], + "RB2": [ + 1535, + 489, + 0.0 + ], + "PK": [ + 1567, + 502, + 0.0 + ], + "K1": [ + 1568, + 403, + 0.0 + ], + "K2": [ + 1600, + 375, + 0.0 + ], + "K3": [ + 1637, + 382, + 0.0 + ], + "K4": [ + 1658, + 412, + 0.0 + ], + "KT1": [ + 1601, + 404, + 40.0 + ], + "KT2": [ + 1640, + 420, + 30.0 + ], + "K5": [ + 1658, + 608, + 0.0 + ], + "PR": [ + 1651, + 622, + 0.0 + ], + "BR": [ + 1522, + 818, + 0.0 + ], + "BL": [ + 1208, + 820, + 0.0 + ], + "HC": [ + 1406, + 640, + 76.0 + ], + "Q": [ + 1178, + 572, + 49.0 + ], + "TB": [ + 1128, + 682, + 0.0 + ], + "TT2": [ + 1015, + 490, + 0.0 + ], + "TL": [ + 1013, + 427, + 0.0 + ], + "TT": [ + 1069, + 402, + 0.0 + ], + "TR": [ + 1133, + 449, + 0.0 + ], + "TT1": [ + 1065, + 435, + 43.0 + ], + "TH": [ + 1144, + 541, + 40.0 + ] + }, + "outline": [ + "I1", + "I2", + "I3", + "I4", + "P1", + "W1", + "M1", + "M2", + "M3", + "M4", + "MB2", + "RL", + "R1", + "R2", + "R3", + "R4", + "RB2", + "PK", + "K1", + "K2", + "K3", + "K4", + "K5", + "PR", + "BR", + "BL", + "TB", + "TT2", + "TL", + "TT", + "TR", + "TH", + "Q", + "IL" + ], + "facets": [ + [ + "IT1", + "IT2", + "IT3" + ], + [ + "IT1", + "I2", + "I3", + "IT2" + ], + [ + "I1", + "I2", + "IT1" + ], + [ + "I1", + "IT1", + "IT3", + "IL" + ], + [ + "I3", + "I4", + "IT2" + ], + [ + "IT2", + "I4", + "P1", + "IT3" + ], + [ + "IL", + "IT3", + "P1", + "Q" + ], + [ + "Q", + "P1", + "W1" + ], + [ + "Q", + "W1", + "P2" + ], + [ + "Q", + "P2", + "HC" + ], + [ + "MT1", + "MT2", + "P2" + ], + [ + "MT1", + "M2", + "M3", + "MT2" + ], + [ + "M1", + "M2", + "MT1" + ], + [ + "M1", + "MT1", + "P2", + "W1" + ], + [ + "M3", + "M4", + "MT2" + ], + [ + "MT2", + "M4", + "MB2", + "MB1", + "P2" + ], + [ + "RT1", + "RT2", + "RB" + ], + [ + "RT1", + "R2", + "R3", + "RT2" + ], + [ + "R1", + "R2", + "RT1" + ], + [ + "R1", + "RT1", + "RB", + "RL" + ], + [ + "R3", + "R4", + "RT2" + ], + [ + "RT2", + "R4", + "RB2", + "RB1", + "RB" + ], + [ + "KT1", + "KT2", + "PK" + ], + [ + "KT1", + "K2", + "K3", + "KT2" + ], + [ + "K1", + "K2", + "KT1" + ], + [ + "K1", + "KT1", + "PK" + ], + [ + "K3", + "K4", + "KT2" + ], + [ + "KT2", + "K4", + "K5", + "PR", + "PK" + ], + [ + "P2", + "MB1", + "MB2", + "RL", + "RB", + "RB1", + "RB2", + "PK", + "HC" + ], + [ + "HC", + "PK", + "BR" + ], + [ + "PK", + "PR", + "BR" + ], + [ + "HC", + "BR", + "BL" + ], + [ + "Q", + "HC", + "BL" + ], + [ + "TT1", + "TH", + "TT2" + ], + [ + "TL", + "TT", + "TT1", + "TT2" + ], + [ + "TT", + "TR", + "TH", + "TT1" + ], + [ + "TT2", + "TH", + "TB" + ], + [ + "TH", + "Q", + "TB" + ], + [ + "Q", + "BL", + "TB" + ] + ], + "colors": [ + "#b2fcfc", + "#01fbfc", + "#00faf3", + "#01faf4", + "#00b7f4", + "#00a5eb", + "#bf54fc", + "#662ffa", + "#7237fa", + "#7651fc", + "#88fbfc", + "#00e8ee", + "#00f5f4", + "#76fafb", + "#00beed", + "#00b1f0", + "#8cfcfc", + "#00f3f3", + "#00e1ed", + "#01edf3", + "#00cef6", + "#00b7f7", + "#82fbfc", + "#01f1f5", + "#00e0ee", + "#01f1f3", + "#00c0f2", + "#6f31fb", + "#0129a6", + "#1542fb", + "#0021af", + "#002eba", + "#01f7fc", + "#bffcfc", + "#01fafc", + "#00e3f0", + "#00ddee", + "#00beef", + "#008cf5" + ], + "silhouette": [ + [ + 1209, + 50 + ], + [ + 1169, + 85 + ], + [ + 1145, + 109 + ], + [ + 1143, + 113 + ], + [ + 1142, + 404 + ], + [ + 1137, + 404 + ], + [ + 1073, + 360 + ], + [ + 1066, + 361 + ], + [ + 988, + 401 + ], + [ + 978, + 409 + ], + [ + 976, + 477 + ], + [ + 977, + 500 + ], + [ + 1060, + 640 + ], + [ + 1179, + 846 + ], + [ + 1186, + 852 + ], + [ + 1197, + 856 + ], + [ + 1530, + 856 + ], + [ + 1543, + 852 + ], + [ + 1556, + 839 + ], + [ + 1695, + 623 + ], + [ + 1695, + 401 + ], + [ + 1651, + 348 + ], + [ + 1647, + 346 + ], + [ + 1589, + 338 + ], + [ + 1584, + 340 + ], + [ + 1566, + 357 + ], + [ + 1538, + 317 + ], + [ + 1531, + 311 + ], + [ + 1469, + 301 + ], + [ + 1464, + 303 + ], + [ + 1445, + 320 + ], + [ + 1416, + 276 + ], + [ + 1412, + 273 + ], + [ + 1358, + 264 + ], + [ + 1348, + 264 + ], + [ + 1330, + 280 + ], + [ + 1328, + 256 + ], + [ + 1329, + 200 + ], + [ + 1327, + 121 + ], + [ + 1283, + 61 + ], + [ + 1278, + 57 + ], + [ + 1225, + 50 + ] + ] + } +} diff --git a/design/cursors/prism-glow/model/prism_geo.py b/design/cursors/prism-glow/model/prism_geo.py new file mode 100644 index 0000000000..28a115496b --- /dev/null +++ b/design/cursors/prism-glow/model/prism_geo.py @@ -0,0 +1,157 @@ +"""Prism Glow cursor as a faceted mesh: the facets traced from ../source.png, and their heights. + +Coordinates are source-PNG pixels (x right, y down); `z` is the height above the navy rim's top in +the same pixels (unlisted vertices sit on the rim). `outline` is the crystal's edge against the rim, +`facets` are the flat faces of the 2D art, each an ordered polygon. + + python prism_geo.py # writes prism.json: vertices, facets, their colours in the art, silhouettes + +Then build the Blender scene and the glTF from it with build_blend.py. +""" + +import json +from pathlib import Path + +import cv2 +import numpy as np + +HERE = Path(__file__).resolve().parent + +ARROW = { + "hotspot": (208, 42), + "verts": { + "A": (237, 100), "B": (443, 290), "F": (712, 549), "K": (543, 563), "J": (612, 744), + "I": (492, 807), "H": (394, 597), "G": (241, 732), "D": (241, 491), + "C": (366, 374), "E": (458, 539), "X": (515, 449), + }, + "outline": ["A", "B", "F", "K", "J", "I", "H", "G", "D"], + "z": {"C": 108, "E": 76, "X": 66}, + "facets": [ + ["A", "C", "D"], ["A", "B", "C"], ["B", "X", "C"], ["B", "F", "X"], + ["C", "X", "E"], ["X", "F", "E"], ["E", "F", "K"], + # the notch H sits on the E-G line: one concept facet, folded along C-H + ["C", "E", "H"], ["C", "H", "G"], ["C", "G", "D"], + ["E", "I", "H"], ["E", "J", "I"], ["E", "K", "J"], + ], +} + +HAND = { + "hotspot": (1220, 50), + "verts": { + # index finger + "I1": (1180, 122), "I2": (1221, 85), "I3": (1261, 90), "I4": (1290, 132), + "IT1": (1224, 118), "IT2": (1274, 135), "IT3": (1191, 262), "IL": (1179, 265), + "P1": (1292, 427), + # middle finger + "W1": (1321, 456), "M1": (1323, 334), "M2": (1356, 302), "M3": (1395, 309), + "M4": (1415, 340), "MT1": (1354, 334), "MT2": (1390, 344), "P2": (1339, 479), + "MB1": (1386, 481), "MB2": (1415, 462), + # ring finger + "RL": (1441, 467), "R1": (1443, 368), "R2": (1476, 338), "R3": (1515, 345), + "R4": (1536, 377), "RT1": (1480, 367), "RT2": (1513, 381), "RB": (1448, 475), + "RB1": (1476, 498), "RB2": (1535, 489), + # pinky + "PK": (1567, 502), "K1": (1568, 403), "K2": (1600, 375), "K3": (1637, 382), + "K4": (1658, 412), "KT1": (1601, 404), "KT2": (1640, 420), "K5": (1658, 608), + "PR": (1651, 622), + # palm + "BR": (1522, 818), "BL": (1208, 820), "HC": (1406, 640), "Q": (1178, 572), + # thumb + "TB": (1128, 682), "TT2": (1015, 490), "TL": (1013, 427), "TT": (1069, 402), + "TR": (1133, 449), "TT1": (1065, 435), "TH": (1144, 541), + }, + # Each fingertip is cut like a gem: a table (the white facet), crown facets down to the rim. + # The thumb web (TH, Q) rises above the rim so the thumb's lower facets are not flat. + "z": { + "IT1": 46, "IT2": 35, "IT3": 54, + "MT1": 46, "MT2": 35, "P2": 59, "MB1": 54, + "RT1": 43, "RT2": 34, "RB": 57, "RB1": 54, + "KT1": 40, "KT2": 30, + "TT1": 43, "TH": 40, "Q": 49, + "HC": 76, + }, + "outline": [ + "I1", "I2", "I3", "I4", "P1", "W1", "M1", "M2", "M3", "M4", "MB2", "RL", "R1", "R2", + "R3", "R4", "RB2", "PK", "K1", "K2", "K3", "K4", "K5", "PR", "BR", "BL", "TB", "TT2", + "TL", "TT", "TR", "TH", "Q", "IL", + ], + "facets": [ + # index: table, crown, sides + ["IT1", "IT2", "IT3"], ["IT1", "I2", "I3", "IT2"], ["I1", "I2", "IT1"], + ["I1", "IT1", "IT3", "IL"], ["I3", "I4", "IT2"], ["IT2", "I4", "P1", "IT3"], + ["IL", "IT3", "P1", "Q"], + # palm under the index (W1 sits on the P1-P2 line) + ["Q", "P1", "W1"], ["Q", "W1", "P2"], ["Q", "P2", "HC"], + # middle finger + ["MT1", "MT2", "P2"], ["MT1", "M2", "M3", "MT2"], ["M1", "M2", "MT1"], + ["M1", "MT1", "P2", "W1"], ["M3", "M4", "MT2"], ["MT2", "M4", "MB2", "MB1", "P2"], + # ring finger + ["RT1", "RT2", "RB"], ["RT1", "R2", "R3", "RT2"], ["R1", "R2", "RT1"], + ["R1", "RT1", "RB", "RL"], ["R3", "R4", "RT2"], ["RT2", "R4", "RB2", "RB1", "RB"], + # pinky + ["KT1", "KT2", "PK"], ["KT1", "K2", "K3", "KT2"], ["K1", "K2", "KT1"], + ["K1", "KT1", "PK"], ["K3", "K4", "KT2"], ["KT2", "K4", "K5", "PR", "PK"], + # palm + ["P2", "MB1", "MB2", "RL", "RB", "RB1", "RB2", "PK", "HC"], + ["HC", "PK", "BR"], ["PK", "PR", "BR"], ["HC", "BR", "BL"], ["Q", "HC", "BL"], + # thumb + ["TT1", "TH", "TT2"], ["TL", "TT", "TT1", "TT2"], ["TT", "TR", "TH", "TT1"], + ["TT2", "TH", "TB"], ["TH", "Q", "TB"], ["Q", "BL", "TB"], + ], +} + + +def area(poly): + p = np.asarray(poly, float) + return 0.5 * np.sum(p[:, 0] * np.roll(p[:, 1], -1) - np.roll(p[:, 0], -1) * p[:, 1]) + + +def check(shape, name): + """The facets tile the outline exactly: no gap, no overlap, no stray vertex.""" + v = shape["verts"] + outline = abs(area([v[k] for k in shape["outline"]])) + facets = sum(abs(area([v[k] for k in f])) for f in shape["facets"]) + unused = set(v) - {k for f in shape["facets"] for k in f} + assert abs(facets / outline - 1) < 1e-9 and not unused, (name, facets / outline, unused) + + +def facet_colors(shape, img): + """Median colour of each facet in the art, away from its edges.""" + v = shape["verts"] + out = [] + for f in shape["facets"]: + mask = np.zeros(img.shape[:2], np.uint8) + cv2.fillPoly(mask, [np.array([v[k] for k in f], np.int32)], 255) + inner = cv2.erode(mask, np.ones((7, 7), np.uint8)) + px = img[(inner if inner.any() else mask) > 0][:, :3] + b, g, r = np.median(px, axis=0) + out.append("#%02x%02x%02x" % (int(r), int(g), int(b))) + return out + + +def silhouette(img, x0, x1): + """Outer edge of the navy rim, from the art's alpha.""" + a = (img[:, :, 3] > 127).astype(np.uint8) + a[:, :x0] = 0 + a[:, x1:] = 0 + contours, _ = cv2.findContours(a, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE) + c = max(contours, key=cv2.contourArea) + return cv2.approxPolyDP(c, 1.2, True)[:, 0, :].tolist() + + +if __name__ == "__main__": + img = cv2.imread(str(HERE.parent / "source.png"), cv2.IMREAD_UNCHANGED) + out = {} + for shape, name, (x0, x1) in ((ARROW, "arrow", (0, 887)), (HAND, "hand", (887, 1774))): + check(shape, name) + z = shape["z"] + out[name] = { + "hotspot": shape["hotspot"], + "verts": {k: [*p, float(z.get(k, 0))] for k, p in shape["verts"].items()}, + "outline": shape["outline"], + "facets": shape["facets"], + "colors": facet_colors(shape, img), + "silhouette": silhouette(img, x0, x1), + } + (HERE / "prism.json").write_text(json.dumps(out, indent=1) + "\n", newline="\n") + print({k: f"{len(v['facets'])} facets" for k, v in out.items()}) diff --git a/design/cursors/requirements.md b/design/cursors/requirements.md index 3274014c7f..1cf03d9879 100644 --- a/design/cursors/requirements.md +++ b/design/cursors/requirements.md @@ -21,16 +21,17 @@ their 3D models. - Every replacement theme and supported cursor state must work with the existing 3D cursor option. -- Every replacement arrow and hand is a real volume, sculpted in the +- Every replacement arrow and hand is a real volume, modelled in the compositor's shaders (`crates/compositor/src/sculpt.rs`): lighting, normals, self shadows and contact shadows respond to cursor tilt and rotation. A PNG face or a height map is not a model. - A rounded extrusion of the PNG silhouette remains the fallback for cursor - states that do not have a dedicated theme model. The five replacement packs - add style-specific volume: a glove with separate fingers, cut crystal facets, - puffy rubber, beveled voxels, and ceramic accents. -- Model the arrow and hand separately for every theme. Do not apply one - contour, material, or roundness rule to all designs. + states that do not have a dedicated theme model. Each pack adds + style-specific volume: gloves with separate fingers, beveled voxels, flat + cut-paper sheets, cushions set in the rim of their drawn outline, and a + faceted glass crystal that refracts the picture under it. +- Model the arrow and hand separately for every sculpted theme. Do not apply + one contour, material, or roundness rule to all designs. - Preserve the cursor hotspot through hover, tilt, yaw, click, and size changes. - Reuse the compositor's 3D lighting and contact shadows. The surface must change with the compositor camera; a static image of a 3D render is not a @@ -40,11 +41,11 @@ their 3D models. | Theme | Arrow | Hand | | --- | --- | --- | -| Studio Ink | Matte black sculpted body; ivory mark becomes a piping set in from the edge. | Ivory glove with a convex palm and separate fingers; remove the dark outline. | -| Prism Glow | Translucent faceted crystal; no navy ink contour. | Crystal palm and distinct faceted fingers; no navy ink contour. | -| Pop Coral | Rounded coral rubber; omit the yellow offset and navy rim. | Plump yellow rubber glove; no navy outline, accents become physical details. | -| Pixel Candy | Keep the stepped voxel form; purple is side/back blocks, not a flat outline. | Stepped voxel fingers and palm; preserve purple as physical side/back blocks. | -| Star Sprout | Soft mint ceramic; no heavy blue outline, raised star and leaves. | Rounded ivory glove with no blue outline; separate mint cuff, star, and leaves. | +| Studio Ink | Black rim, raised ivory band, recessed satin black field. | Ivory glove cushion; the black outline stays as a physical rim, with black ridges between the fingers. | +| Prism Glow | A glass crystal cut along the drawing's facets, in a thin navy rim; the picture under it shows through, refracted with a slight dispersion, privacy blurs included; the facets glow faintly in their drawn colours. Modelled from the 2D drawing alone, keeping its polygonal simplicity. | The same, a crystal hand. | +| Pop Coral | Cut paper: a coral sheet on a navy sheet, flat tops, matte; the yellow offset becomes a flat sheet behind; the click dashes become flat pieces. | A yellow sheet on a navy sheet, flat tops, matte; the coral offset becomes a flat sheet behind; the coral click dashes become flat pieces. | +| Pixel Candy | Simplified pixel art that reads at 20 px as at full size; the 3D model is one cube per pixel of the same grid as the sprite. | A pixel hand from the same kind of grid: index, three knuckles, thumb. | +| Star Sprout | Puffy mint cushion; the navy outline stays as a physical rim (a navy tray and a rounded bead) around it, the star and its leaves in front, each in its own navy rim. | Puffy ivory glove in the same navy rim, with navy ridges between the fingers; separate mint cuff framed in navy; the star and leaves in front, as in `star-sprout/3d-reference.png`. | The 2D PNG may keep a drawn contour where it helps readability. Its contour must not be copied blindly onto the 3D material: decide per model whether it @@ -53,7 +54,11 @@ becomes an inlay, side material, separate piece, or disappears. ## Review references - `contact-sheet.png`: 2D PNG themes. -- `3d-concept.png`: visual direction only, not rendered by the app. +- `3d-concept.png`: the 3D reference sheet the eight sculpted models follow + (Prism Glow's crystal follows its 2D drawing instead), not rendered by the + app. +- `star-sprout/3d-reference.png`: a closer view of the Star Sprout hand; also + not rendered by the app. - `3d-direction.md`: per-theme modeling notes. - `README.md`: source PNG and preparation workflow. diff --git a/design/cursors/star-sprout/3d-reference.png b/design/cursors/star-sprout/3d-reference.png new file mode 100644 index 0000000000..760ed6b62c Binary files /dev/null and b/design/cursors/star-sprout/3d-reference.png differ diff --git a/docs/3d-effects-v2.md b/docs/3d-effects-v2.md index 978cd1a809..119c5b3bbd 100644 --- a/docs/3d-effects-v2.md +++ b/docs/3d-effects-v2.md @@ -233,12 +233,11 @@ inconnues sont ignorées). ### B.2 Le réglage **Un seul interrupteur**, `cursor.model3d` (« 3D cursor », **éteint par défaut**) : il passe -**chaque état** du curseur en 3D. La flèche et la main des cinq thèmes d'origine deviennent leur -modèle sculpté (B.3 bis) ; les autres états, et tous ceux du thème par défaut (les seize de -`DEFAULT_CURSOR_SPRITES` : flèche, I, main, croix, mains ouverte et fermée, redimensionnements, -déplacement, interdit, attente…), leur sprite extrudé. Curseur masqué, l'interrupteur est grisé et -son info-bulle dit pourquoi. Éteint, la frame est celle d'avant **à l'octet** (vérifié à plat et -incliné contre le commit de base). +**chaque état** du curseur en 3D. La flèche et la main des cinq thèmes d'origine deviennent +leur modèle (B.3 bis) ; les autres états, et tous ceux du thème par défaut (les seize de `DEFAULT_CURSOR_SPRITES` : flèche, I, main, croix, mains +ouverte et fermée, redimensionnements, déplacement, interdit, attente…), leur sprite extrudé. +Curseur masqué, l'interrupteur est grisé et son info-bulle dit pourquoi. Éteint, la frame est +celle d'avant **à l'octet** (vérifié à plat et incliné contre le commit de base). Tuyauterie : `CursorVisualSettings.model3d`, clé legacy `cursorModel3d`, préréglages (absent → éteint), `SceneCursor.model3d` (`serde(default)`), `LiveParams.cursor_model3d`, paramètre live @@ -287,24 +286,44 @@ du bord (0,48 au pire), sous 1 texel au-delà (le flou arrondit les crêtes). ### B.3 bis Les curseurs sculptés -La flèche et la main des cinq thèmes d'origine sont **modélisées à la main**, en fonctions de -distance signée écrites dans les trois shaders (`sculpt_proto`, `sculpt_material`) : capsules et -unions lissées pour les gants, extrusion arrondie et bombée pour les flèches, voxels biseautés -pour Pixel Candy, polyèdres taillés pour le cristal de Prism Glow. Le PNG du thème reste l'art en -2D. En 3D, la scène nomme le modèle ; `sculpt.rs` en tient la boîte, qui pose le hotspot (pointe -de la flèche, bout de l'index), règle la garde au sol et borne la boîte de dessin. Emplacement du -cbuffer : `trail_a` = [modèle, épaisseur sous z = 0, hauteur au-dessus, 0] (`cursor_model_cb`). +La flèche et la main des thèmes d'origine sont **modélisées à la main**, en fonctions de +distance signée écrites dans les trois shaders (`sculpt_proto`, `sculpt_material`) : voxels +pour Pixel Candy ; pour Studio Ink et Star Sprout des pièces qui gardent le trait de +leur dessin : un plateau et un jonc de la couleur du trait, un coussin de couleur dedans +(`s_rimmed`) ; pour Pop Coral les mêmes formes en papier découpé, des feuilles au dessus plat +(`s_paper`). Prism Glow est à part : un **cristal en maillage**, ses facettes tracées sur son +seul dessin 2D (`design/cursors/prism-glow/model/`), serti dans sa silhouette extrudée, marine. +Le shader le lance de rayons, triangles rangés par boîtes englobantes (un rayon saute les +boîtes qu'il rate) : réfraction par canal (une légère dispersion), réflexions totales internes, +et sortie par son fond plat sur l'image sous le curseur ; chaque facette luit un peu de sa +couleur du dessin, ses plis d'un liseré clair. Cette image est une copie de la frame composée +prise juste avant le curseur : le métrage, puis ses flous de confidentialité, puis le curseur, +net par-dessus. Seuls des pixels déjà floutés passent donc à travers le verre. +`export_compositor.py` écrit le maillage dans `prism_mesh.rs` et dans les trois shaders. Le PNG +du thème reste l'art en 2D. En 3D, la scène nomme le modèle ; `sculpt.rs` en tient la boîte, qui +pose le hotspot (pointe de la flèche, bout de l'index), règle la garde au sol et borne la boîte +de dessin. Emplacement du cbuffer : `trail_a` = [modèle, épaisseur sous z = 0, hauteur +au-dessus, 0] (`cursor_model_cb`). +Pixel Candy est un pixel art dessiné une fois, en grille, dans +`scripts/generate-pixel-candy-voxels.mjs` : le script en tire ses PNG 2D et les tables des trois +shaders, un cube par pixel plein, tous de même hauteur, colorés comme leur pixel (contour prune, +rose, reflet rose pâle, ombre rose foncé). - **Éclairage**, sprites extrudés compris : une lampe proche en haut à gauche, qui met un dégradé et un reflet même sur une face plane ; une lumière d'appoint faible ; le côté ombré teinté par la matière ; occlusion ambiante, ombre propre douce vers la lampe, studio dans les reflets, liseré de Fresnel ; tone map Khronos PBR Neutral. -- **Ombre sur l'écran** : celle de B.3. Le cristal et les voxels donnent de mauvaises distances - loin de leur surface (bornes de plans, champ de grille) : l'ombre prend la forme lisse de Pop - Coral pour le cristal et le contour extrudé pour les voxels. +- **Ombre sur l'écran** : celle de B.3. Les voxels donnent de mauvaises distances loin de leur + surface (champ de grille) : l'ombre de Pixel Candy vient de sa forme extrudée. +- **Antialiasing** : la silhouette l'est par la marche (la distance minimale frôlée, en pixels, + donne la couverture). Les bords intérieurs (deux matières, une arête, un arrondi serré, un + joint de voxels) sont suréchantillonnés : quatre sondes à un demi-pixel du point touché, dans + son plan tangent, et si la matière change ou que la surface s'en écarte, trois rayons de plus + dans le pixel, ombrés avec l'occlusion et les ombres du premier (`STAGE_EDGE`, `STAGE_SHADE`). + Sur un grand curseur, ~+50 % du coût du curseur (rendu logiciel), rien ailleurs. - **Coût de compilation** : FXC recopie chaque appel. La marche, les normales, l'occlusion et les - ombres forment donc une seule boucle à étapes avec un seul appel au modèle, et l'étoile de Star - Sprout un seul appel pour les deux formes. + ombres forment donc une seule boucle à étapes avec un seul appel au modèle, et les thèmes + cerclés un seul appel pour leurs deux formes. ### B.4 La caméra et l'ancrage @@ -490,7 +509,7 @@ C'est le cadre qui s'adapte au métrage — jamais l'inverse : un téléphone au est un téléphone COUCHÉ, avec son œil de caméra sur le bord qui est devenu son haut. **Roundness sous un cadre.** Le slider parcourt 0 → le plafond du cadre choisi (C.5) et se lit -en **%** de cette course, avec l'infobulle « La course s'adapte au cadre » (`roundnessFrameHelp`, +en **%** de cette course, avec l'infobulle « La course dépend du cadre » (`roundnessFrameHelp`, 15 langues). La valeur stockée reste en px ; le natif en relit la position (`roundnessFrac` × petit côté de la sortie ÷ `ROUNDNESS_SLIDER_MAX_PX` = 64, miroir de `paramUnits.ts`). Sans cadre, le slider reste en px et le rendu est celui d'avant, à l'octet. diff --git a/docs/assets/demo.gif b/docs/assets/demo.gif index 9c2a0270ec..23ab2fa658 100644 Binary files a/docs/assets/demo.gif and b/docs/assets/demo.gif differ diff --git a/docs/superpowers/plans/2026-10-05-multi-camera-compositor.md b/docs/superpowers/plans/2026-10-05-multi-camera-compositor.md new file mode 100644 index 0000000000..35f978d853 --- /dev/null +++ b/docs/superpowers/plans/2026-10-05-multi-camera-compositor.md @@ -0,0 +1,456 @@ +# Multi-Camera Compositor (Sub-project 2) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Preview and export draw up to four camera layers per frame — arranged by layout regions stored in the project, gliding between regions, each camera with its own rotation/mirror/crop and an optional perspective (homography) correction — on Windows, macOS and Linux. + +**Architecture:** The app (TypeScript) resolves each stored layout region into concrete camera layers (rects in output-frame fractions) and each camera's perspective into a 3×3 matrix, and sends both in the scene. The Rust compositor plans the layers per frame (`camera_layers.rs`, a pure function: region in effect, glide/fade between neighbours), decodes the extra cameras next to camera 1, and every backend draws each layer with the existing camera shader, whose `LayerCB` gains a homography and a transparency lane. With no layout regions and no camera settings, every code path is the one that runs today. + +**Tech Stack:** Rust (crates/compositor: D3D11 HLSL, Metal MSL, wgpu WGSL; ffmpeg decoders), napi-rs (`crates/compositor-view-napi`), TypeScript (scene builder, layout math), Vitest, cargo test. + +**Spec:** `docs/superpowers/specs/2026-10-05-multi-camera-layouts-design.md` (sections 1, 2 and the perspective parts; the editor in section 3 is sub-project 3, a separate plan). + +**Facts file (read it before any task):** `.superpowers/camera-path-facts.md` — file:line map of today's single-camera path (scene contract, `plan_frame`, each backend's webcam block, live decoders, export walk, tests). Line numbers in this plan come from it and are approximate (`~`). + +**Worktree / branch:** `C:\osc-cams`, branch `feat/multi-camera-layouts` (sub-project 1 + desk view #989 merged), base for this plan `c1a8aabb`. `node_modules` and `crates/thirdparty` are junctions to `C:\osc-multicam`. Rust tests: `crates\x.bat test -p openscreen-compositor --lib` (puts the ffmpeg DLLs on PATH), or from `crates/compositor` with `crates/thirdparty/ffmpeg-n8.1.2-win64-lgpl-shared/bin` on PATH. Expected on this host: 8 `pipeline_windows::tests` fail with or without these changes (no D3D11VA H.264 decode / no AV1 encoder) — any other failure is yours. + +## Global Constraints + +- Camera index convention everywhere: **0 = camera 1** (`asset.cameraTrack`, `SceneClip.webcam_path`), **k ≥ 1 = `asset.additionalCameraTracks[k-1]`** (`SceneClip.additional_cameras[k-1]`). At most 4 cameras. +- **Unchanged-when-unused:** a scene without `cameraLayoutRegions` and without `cameras` renders exactly as today (same `FrameGeometry` for camera 1, same pixels). Every new scene field is `#[serde(default)]`; existing JSON fixtures must parse unchanged. +- A layout region of template `camera-full` whose only camera is 0 is **always emitted as a `cameraFullscreenRegion`** (keeps the desk view: rotation, cover blur, label). Only other templates/cameras become `cameraLayoutRegions`. +- Transition envelope: reuse the Full Camera envelope — lead-in `min(TRANSITION_WINDOW_S, len/2)`, lead-out `min(FULLSCREEN_LEAD_OUT_WINDOW_S, len/2)`, easing `ease_out_screen_studio` (`regions.rs` ~:274, ~:331). (Spec: "dieselbe Fensterlänge und Kurve wie das Wachsen der Full Camera"; its ease-out covers ~90 % of the move in ≈0.4 s, the spec's "~0,4 s".) +- Background effects (segmentation, cutout, blur, custom bg) stay on **camera 0 only**; extra cameras draw with effect code 0. +- Camera 0 settings in this sub-project: only `perspective` (homography) applies to camera 0 — also when there are no layout regions (today's camera-0 block then sets the Task 5 lanes from `Scene::camera(0)`). Camera 0's rotation/mirror stay the project mirror + desk-view regions as today; extras use their own `rotation`/`mirror`. +- With a perspective set, the camera's `rotation`/`mirror` are ignored (the corner order already defines the upright, unmirrored result) and `crop` is ignored. +- `LayerCB` layout is shared by Rust, HLSL, WGSL and MSL; append new lanes **last**, keep 16-byte alignment, update the size/offset test. A transparency lane uses `0 = opaque` so a zero-initialised field changes nothing. +- macOS (Metal) is compiled only by CI here: keep MSL edits mechanical and mirror HLSL; say in the report that Metal was not run. +- Comments in English (match the surrounding file even where it is French). Biome style for TS; no new `any`. Commit messages Conventional-Commits style, **no `Co-Authored-By` / Claude trailer**, do not push. +- Native addon: after Rust changes that the app should load, `npm run build:native:compositor`, then copy `electron/native/compositor-view/build/compositor_view.node` to `electron/native/bin/win32-arm64/` and run the app with `OPENSCREEN_COMPOSITOR_VIEW_NODE` pointing at it (this branch has no ARM PR; the script copies to win32-x64 only). + +## Review Focus + +1. **A layout region names a camera whose file is missing or fails to open** → that layer is skipped, the rest (screen, other cameras) is drawn; preview and export keep running. Pinned in Task 7 (`a missing extra camera skips its layer`) and Task 8 (`export skips an extra camera that will not open`). +2. **An extra camera shorter than the clip** (stopped early) → after its last frame the layer disappears; the clip is *not* truncated (only camera 0 clamps the clip end, as today). Pinned in Task 8 (`a short extra camera does not shorten the clip`). +3. **No layout regions** → bit-identical to today. Pinned in Task 4 (`without layout regions the plan has no camera layers`) and Task 6 (`identity lanes leave the camera pixels unchanged`). +4. **Two adjacent regions** → the cameras glide directly from region A's layout to region B's, not via the default PiP in between. Pinned in Task 4 (`adjacent regions glide directly`). +5. **Degenerate perspective corners** (collinear, crossed) → no matrix is sent; the camera draws uncorrected. Pinned in Task 1 (`rejects collinear and crossed quads`) and Task 3 (`a degenerate perspective sends no homography`). + +--- + +### Task 1: Camera layout model, normalizers and perspective math (TS) + +**Files:** +- Modify: `src/components/video-editor/types.ts` (next to `CameraFullscreenRegion` ~:326) — types below. +- Create: `src/lib/cameraLayouts.ts` + `src/lib/cameraLayouts.test.ts` — normalizers. +- Create: `src/lib/cameraPerspective.ts` + `src/lib/cameraPerspective.test.ts` — homography. + +**Interfaces:** +- Produces: + ```ts + // types.ts + export type CameraLayoutTemplate = "screen-pip" | "camera-full" | "camera-full-pip" | "side-by-side"; + export interface NormalizedRect { x: number; y: number; width: number; height: number } // 0..1 of the output frame + export interface CameraLayoutSlot { camera: number; rect?: NormalizedRect } + export interface CameraLayoutRegion { + id: string; startMs: number; endMs: number; + template: CameraLayoutTemplate; slots: CameraLayoutSlot[]; + rotation?: CameraRotation; mirror?: CameraMirrorMode; deskLabel?: false; // camera-full only (desk view) + } + export interface CameraPoint { x: number; y: number } // 0..1 of the camera image + export interface CameraPerspective { corners: [CameraPoint, CameraPoint, CameraPoint, CameraPoint]; aspect: number; margin?: number } + export interface CameraSettings { rotation?: CameraRotation; mirror?: boolean; crop?: CropRegion; perspective?: CameraPerspective } + // cameraLayouts.ts + export const MAX_CAMERAS = 4; + export const TEMPLATE_SLOTS: Record; + export function normalizeCameraLayoutRegions(raw: unknown): CameraLayoutRegion[]; + export function normalizeCameraSettings(raw: unknown): (CameraSettings | null)[]; // index = camera + // cameraPerspective.ts + export function homographyFromUnitSquare(c: CameraPerspective["corners"]): number[] | null; // row-major 3x3, maps (u,v,1) target→source + export function perspectiveMatrix(p: CameraPerspective): number[] | null; // margin folded in + ``` + (`CropRegion` is the existing webcam crop type used by `settings.webcamCropRegion` — reuse it; check its name in `types.ts`.) + +Normalizer rules: drop non-objects; `template` must be one of the four; `slots` filtered to integer `camera` in `0..MAX_CAMERAS-1`, duplicates removed (first wins), length clamped to `TEMPLATE_SLOTS[template].max`, region dropped if fewer than `min`; `rect` kept only if all four numbers finite, width/height > 0, inside `[-0.5, 1.5]`; `startMs < endMs` (else dropped); regions sorted by `startMs`, overlaps resolved by dropping the later one; desk fields only on `camera-full` (normalize with the existing `normalizeCameraRotation` / `normalizeCameraMirror`). `TEMPLATE_SLOTS = { "screen-pip": {min:1,max:3}, "camera-full": {min:1,max:1}, "camera-full-pip": {min:2,max:3}, "side-by-side": {min:2,max:2} }`. `normalizeCameraSettings`: array of length ≤ 4, each entry an object or `null`; `rotation` via `normalizeCameraRotation`; `mirror` boolean only; `perspective` kept only if 4 finite corners in `[-0.5,1.5]`, `aspect` finite in `[0.1, 10]`, `margin` clamped to `[0, 0.2]`. + +- [ ] **Step 1: Failing tests** — `cameraPerspective.test.ts`: + +```ts +import { describe, expect, it } from "vitest"; +import { homographyFromUnitSquare, perspectiveMatrix } from "./cameraPerspective"; + +const apply = (h: number[], u: number, v: number) => { + const w = h[6] * u + h[7] * v + h[8]; + return [(h[0] * u + h[1] * v + h[2]) / w, (h[3] * u + h[4] * v + h[5]) / w]; +}; +const quad = [ + { x: 0.2, y: 0.3 }, { x: 0.85, y: 0.25 }, { x: 0.95, y: 0.9 }, { x: 0.1, y: 0.8 }, +] as const; + +describe("homographyFromUnitSquare", () => { + it("maps the unit square's corners onto the four points", () => { + const h = homographyFromUnitSquare([...quad] as never); + expect(h).not.toBeNull(); + const corners = [[0, 0], [1, 0], [1, 1], [0, 1]]; + corners.forEach(([u, v], i) => { + const [x, y] = apply(h as number[], u, v); + expect(x).toBeCloseTo(quad[i].x, 9); + expect(y).toBeCloseTo(quad[i].y, 9); + }); + }); + it("is the identity for the full image", () => { + const h = homographyFromUnitSquare([{ x: 0, y: 0 }, { x: 1, y: 0 }, { x: 1, y: 1 }, { x: 0, y: 1 }]); + expect(h?.map((n) => Number(n.toFixed(12)))).toEqual([1, 0, 0, 0, 1, 0, 0, 0, 1]); + }); + it("rejects collinear and crossed quads", () => { + expect(homographyFromUnitSquare([{ x: 0, y: 0 }, { x: 0.5, y: 0 }, { x: 1, y: 0 }, { x: 0, y: 1 }])).toBeNull(); + // bow-tie: top-right and bottom-right swapped + expect(homographyFromUnitSquare([{ x: 0, y: 0 }, { x: 1, y: 1 }, { x: 1, y: 0 }, { x: 0, y: 1 }])).toBeNull(); + }); +}); + +describe("perspectiveMatrix", () => { + it("folds the margin in: the target edge lands outside the quad", () => { + const h = perspectiveMatrix({ corners: [...quad] as never, aspect: 297 / 210, margin: 0.1 }); + const inner = perspectiveMatrix({ corners: [...quad] as never, aspect: 297 / 210 }); + expect(h).not.toBeNull(); + // the target point that maps to the quad's top-left corner moves inward by the margin + const [x, y] = apply(h as number[], 0.1 / 1.2, 0.1 / 1.2); + expect(x).toBeCloseTo(quad[0].x, 9); + expect(y).toBeCloseTo(quad[0].y, 9); + expect(apply(inner as number[], 0, 0)[0]).toBeCloseTo(quad[0].x, 9); + }); +}); +``` + +`cameraLayouts.test.ts` (normalizers): a valid region round-trips; unknown template dropped; duplicate camera in slots removed; `camera-full-pip` with one slot dropped; overlapping regions → later dropped; `rect` with NaN removed; desk fields stripped from `screen-pip`; settings: `[null, { rotation: 180, perspective: {...valid} }, "junk", {…}, {…}]` → length 4, junk → `null`, a perspective with `aspect: 0` removed. Write each as its own `it` with explicit input/expected objects. + +- [ ] **Step 2:** `npx vitest --run src/lib/cameraPerspective.test.ts src/lib/cameraLayouts.test.ts` → FAIL (modules missing). + +- [ ] **Step 3: Implement.** `homographyFromUnitSquare` — Heckbert's square→quad closed form (corners in order (0,0),(1,0),(1,1),(0,1)): + +```ts +export function homographyFromUnitSquare(c: CameraPerspective["corners"]): number[] | null { + if (!isConvexClockwiseOrCounter(c)) return null; + const [p0, p1, p2, p3] = c; + const dx1 = p1.x - p2.x, dx2 = p3.x - p2.x, dx3 = p0.x - p1.x + p2.x - p3.x; + const dy1 = p1.y - p2.y, dy2 = p3.y - p2.y, dy3 = p0.y - p1.y + p2.y - p3.y; + let g = 0, h = 0; + if (Math.abs(dx3) > 1e-12 || Math.abs(dy3) > 1e-12) { + const den = dx1 * dy2 - dx2 * dy1; + if (Math.abs(den) < 1e-12) return null; + g = (dx3 * dy2 - dx2 * dy3) / den; + h = (dx1 * dy3 - dx3 * dy1) / den; + } + const a = p1.x - p0.x + g * p1.x, b = p3.x - p0.x + h * p3.x; + const d = p1.y - p0.y + g * p1.y, e = p3.y - p0.y + h * p3.y; + return [a, b, p0.x, d, e, p0.y, g, h, 1]; +} +``` + +`isConvexClockwiseOrCounter`: the four cross products of consecutive edges are all non-zero (|z| > 1e-9) and share one sign. `perspectiveMatrix`: `H · S` where `S` maps the output's local `[0,1]²` to `[-m, 1+m]²` (`S = [[1+2m,0,-m],[0,1+2m,-m],[0,0,1]]`), row-major multiply; `null` if `H` is null. (`aspect` is not part of the matrix — it sets the layer's box ratio in Task 2.) + +- [ ] **Step 4:** both test files PASS; `npx tsc --noEmit`; `npm run lint`. +- [ ] **Step 5:** commit `feat(camera): layout regions, camera settings and the perspective matrix`. + +--- + +### Task 2: Resolve layout templates into camera layers (TS) + +**Files:** +- Create: `src/lib/cameraLayoutTemplates.ts` + `src/lib/cameraLayoutTemplates.test.ts`. + +**Interfaces:** +- Consumes: Task 1 types. +- Produces: + ```ts + export interface ResolvedCameraLayer { + camera: number; + rect: NormalizedRect; // output-frame fractions + radiusFrac: number; // corner radius as a fraction of min(rect w, h) in pixels + shape: "rectangle" | "rounded" | "circle" | "square"; // same vocabulary as SceneLayout.webcamShape + fillsFrame: boolean; // covers the screen (no shadow) + } + export interface CameraLayoutContext { + frame: { width: number; height: number }; // output pixels + cameraAspect: (camera: number) => number; // width/height of that camera's box (perspective aspect if set) + pipShape: ResolvedCameraLayer["shape"]; // project's webcam shape + pipRadiusFrac: number; // project's webcam radius fraction + } + export const PIP_WIDTH_FRAC = 0.22; + export const PIP_MARGIN_FRAC = 0.025; + export const PIP_GAP_FRAC = 0.02; + export function resolveCameraLayout(region: CameraLayoutRegion, ctx: CameraLayoutContext): ResolvedCameraLayer[]; + ``` + +Template rules (slot order = `region.slots` order; a slot's own `rect` overrides the template rect but keeps the template's `fillsFrame`/shape): +- `camera-full`: slot 0 → `{x:0,y:0,width:1,height:1}`, radius 0, shape `"rectangle"`, `fillsFrame: true`. +- `side-by-side`: slot 0 → left half `{0,0,0.5,1}`, slot 1 → right half `{0.5,0,0.5,1}`, radius 0, `"rectangle"`, `fillsFrame: true`. +- `screen-pip`: every slot is a PiP. +- `camera-full-pip`: slot 0 as `camera-full`, slots 1.. are PiPs. +- PiP j (0-based among the PiPs): width `w = PIP_WIDTH_FRAC` of the frame width; height fraction `h = w * frame.width / cameraAspect(camera) / frame.height`; right edge at `1 - PIP_MARGIN_FRAC - j * (w + PIP_GAP_FRAC)`; bottom at `1 - PIP_MARGIN_FRAC * frame.width / frame.height`; `x = right - w`, `y = bottom - h`; shape `ctx.pipShape`, radius `ctx.pipRadiusFrac`, `fillsFrame: false`. For `circle`/`square` the box is square (`h = w * frame.width / frame.height`). +- Layers are returned in draw order: `fillsFrame` layers first (slot order), then PiPs (slot order). + +- [ ] **Step 1: Failing tests** (frame 1920×1080, aspect 16/9 for all cameras unless stated): + +```ts +import { describe, expect, it } from "vitest"; +import { PIP_MARGIN_FRAC, PIP_WIDTH_FRAC, resolveCameraLayout } from "./cameraLayoutTemplates"; + +const ctx = { frame: { width: 1920, height: 1080 }, cameraAspect: () => 16 / 9, pipShape: "rounded" as const, pipRadiusFrac: 0.12 }; +const region = (template: never, cameras: number[], rects: Record = {}) => + ({ id: "r", startMs: 0, endMs: 5000, template, slots: cameras.map((camera, i) => ({ camera, ...(rects[i] ? { rect: rects[i] } : {}) })) }) as never; + +describe("resolveCameraLayout", () => { + it("camera-full fills the frame", () => { + expect(resolveCameraLayout(region("camera-full" as never, [2]), ctx)).toEqual([ + { camera: 2, rect: { x: 0, y: 0, width: 1, height: 1 }, radiusFrac: 0, shape: "rectangle", fillsFrame: true }, + ]); + }); + it("side-by-side splits the frame in halves", () => { + const layers = resolveCameraLayout(region("side-by-side" as never, [0, 1]), ctx); + expect(layers.map((l) => l.rect)).toEqual([ + { x: 0, y: 0, width: 0.5, height: 1 }, { x: 0.5, y: 0, width: 0.5, height: 1 }, + ]); + }); + it("screen-pip stacks pips from the bottom-right corner leftwards", () => { + const [a, b] = resolveCameraLayout(region("screen-pip" as never, [0, 1]), ctx); + expect(a.rect.x + a.rect.width).toBeCloseTo(1 - PIP_MARGIN_FRAC, 9); + expect(a.rect.width).toBeCloseTo(PIP_WIDTH_FRAC, 9); + expect(a.rect.height).toBeCloseTo(PIP_WIDTH_FRAC, 9); // 16:9 camera in a 16:9 frame + expect(b.rect.x + b.rect.width).toBeLessThan(a.rect.x); + expect([a.fillsFrame, b.fillsFrame]).toEqual([false, false]); + }); + it("camera-full-pip draws the full camera first, then the pip", () => { + const layers = resolveCameraLayout(region("camera-full-pip" as never, [1, 0]), ctx); + expect(layers.map((l) => [l.camera, l.fillsFrame])).toEqual([[1, true], [0, false]]); + }); + it("a user rect overrides the template position", () => { + const rect = { x: 0.1, y: 0.1, width: 0.3, height: 0.3 }; + const [layer] = resolveCameraLayout(region("screen-pip" as never, [1], { 0: rect }), ctx); + expect(layer.rect).toEqual(rect); + }); + it("a portrait camera gets a taller pip box", () => { + const [layer] = resolveCameraLayout(region("screen-pip" as never, [1]), { ...ctx, cameraAspect: () => 9 / 16 }); + expect(layer.rect.height).toBeGreaterThan(layer.rect.width); + }); +}); +``` + +- [ ] **Step 2:** run → FAIL. **Step 3:** implement per the rules. **Step 4:** PASS, tsc, lint. **Step 5:** commit `feat(camera): resolve layout templates into camera layers`. + +--- + +### Task 3: Scene contract for extra cameras, camera settings and layout regions (TS + Rust parse) + +**Files:** +- Modify: `src/native/contracts.ts` (`CompositorClipInput` ~:196) — `additionalCameras?: CompositorClipCamera[]`. +- Modify: `src/native/sceneDescription.ts` — types (~:132-146, ~:473) and builder (`buildSceneDescription` ~:840-1545). +- Modify: `src/lib/ai-edition/timeline/camera.ts` (~:45) — add `assetAdditionalCameraSources`. +- Modify: `crates/compositor/src/scene.rs` — structs + `Scene::for_clip_window` (~:780) + parse tests (~:957). +- Test: `src/native/sceneDescription.test.ts`, `src/lib/ai-edition/timeline/camera.test.ts` (create if absent), `scene.rs` tests. + +**Interfaces:** +- Consumes: Task 1 normalizers + `perspectiveMatrix`, Task 2 `resolveCameraLayout`. +- Produces (TS, mirrored in Rust with `#[serde(rename_all = "camelCase")]`, every new field `#[serde(default)]`): + ```ts + export interface CompositorClipCamera { path: string; offsetSec: number } // contracts.ts + // CompositorClipInput.additionalCameras?: CompositorClipCamera[] — index k-1 = camera k + export interface SceneCamera { index: number; rotation?: 180; mirror?: boolean; crop?: SceneCrop; homography?: number[] /* 9, row-major, target uv → camera uv */; aspect?: number } + export interface SceneCameraLayer { camera: number; rect: SceneRect; radiusFrac: number; shape: string; fillsFrame: boolean } + export interface SceneCameraLayoutRegion { startSec: number; endSec: number; clipIndex?: number; underTrim?: boolean; layers: SceneCameraLayer[] } + // SceneDescription.cameras?: SceneCamera[]; SceneDescription.cameraLayoutRegions?: SceneCameraLayoutRegion[] + export function assetAdditionalCameraSources(asset: AxcutAsset): CompositorClipCamera[]; // camera.ts + ``` + Rust: `SceneClipCamera { path: String, offset_sec: f64 }`; `SceneClip.additional_cameras: Vec`; `SceneCamera { index: usize, rotation: u16, mirror: Option, crop: Option, homography: Option<[f32; 9]>, aspect: Option }`; `SceneCameraLayer { camera: usize, rect: SceneRect, radius_frac: f32, shape: String, fills_frame: bool }`; `SceneCameraLayoutRegion { clip_index: Option, start_sec: f64, end_sec: f64, layers: Vec }`; `Scene.cameras: Vec`, `Scene.camera_layout_regions: Vec`; `Scene::camera(&self, index) -> Option<&SceneCamera>`. + +Builder rules: +- `assetAdditionalCameraSources(asset)`: for each `additionalCameraTracks` entry, `{ path: visible && sourcePath ? sourcePath : "", offsetSec: (startMs + offsetMs) / 1000 }` (same formula as `assetCameraSource`); `[]` when absent. +- Clips (~:1004-1027): set `additionalCameras` only when non-empty. +- Read `legacyEditor.cameraLayoutRegions` through `normalizeCameraLayoutRegions` and `legacyEditor.cameraSettings` through `normalizeCameraSettings` (raw legacyEditor access as done for `cameraFullscreenRegions` ~:1101). +- Regions whose template is `camera-full` and whose single slot is camera 0 → appended to the Full Camera list **before** `projectRegionsToSource` (with `rotation/mirror/deskLabel`), so the desk view and its label apply unchanged. +- All other regions → `projectRegionsToSource` exactly like Full Camera regions (same clip windowing, `underTrim`), then each projected piece gets `layers = resolveCameraLayout(region, ctx)` with `ctx.frame` = output size, `ctx.cameraAspect(i)` = the camera's perspective `aspect` if set, else its track `width/height` (camera 0: the probed `webcamSourceSize` / `cameraTrack`; others: `additionalCameraTracks[i-1].width/height`, fallback 16/9), `pipShape`/`pipRadiusFrac` from the resolved camera-1 layout (`settings.webcamMaskShape`, `webcamRadiusFrac`). Layers whose camera does not exist on the asset (no track) are dropped; a region left with no layers is dropped. +- `cameras`: one entry per non-null normalized settings entry: `rotation: 180` only if 180, `mirror` only if set, `crop` only if set and no perspective, `homography` = `perspectiveMatrix(p)` when it is non-null (null → omit, Review Focus 5), `aspect` = `p.aspect` when perspective set. +- Omit `cameras` / `cameraLayoutRegions` entirely when empty (unchanged scene for old projects). +- Rust `Scene::for_clip_window`: retain `camera_layout_regions` for the clip with the same rule as `camera_fullscreen_regions` (by `clip_index`, fallback overlap). + +- [ ] **Step 1: Failing tests.** + - `sceneDescription.test.ts`: (a) `a project without camera layouts produces the same scene as before` — build a scene for an existing fixture document and assert `cameras` and `cameraLayoutRegions` are `undefined` and every clip has no `additionalCameras` key; (b) `extra camera tracks become clip cameras` — asset with `additionalCameraTracks: [{ sourcePath: "/w-2.mp4", startMs: 0, offsetMs: 120, visible: true, label: "Desk" }]` → `clips[0].additionalCameras` equals `[{ path: "/w-2.mp4", offsetSec: 0.12 }]`; (c) `a camera-full region for camera 1 stays a Full Camera region` — `legacyEditor.cameraLayoutRegions: [{ id:"a", startMs:1000, endMs:4000, template:"camera-full", slots:[{camera:0}], rotation:180 }]` → it appears in `cameraFullscreenRegions` with `rotation: 180` and `cameraLayoutRegions` is undefined; (d) `a camera-full-pip region becomes resolved layers` — template `camera-full-pip`, slots `[{camera:1},{camera:0}]` → one `cameraLayoutRegions` entry with `startSec 1, endSec 4` and layers `[camera 1 fillsFrame, camera 0 pip]`; (e) `a degenerate perspective sends no homography` — settings `[null, { perspective: { corners: collinear, aspect: 1.5 } }]` → `cameras` is `[{ index: 1 }]` or undefined, but never has `homography`; (f) `a layer for a camera the asset does not have is dropped`. + - Build the fixtures with the helpers already used in `sceneDescription.test.ts` (find the document factory there; add `additionalCameraTracks` / `legacyEditor` fields to it). + - `scene.rs`: `parses_extra_cameras_settings_and_layout_regions` (a JSON with all new fields → values), `old_scene_json_parses_without_the_new_fields` (reuse an existing fixture string → empty vectors), `for_clip_window_keeps_only_this_clips_layout_regions`. +- [ ] **Step 2:** run the TS tests and `cargo test -p openscreen-compositor --lib scene::` → FAIL. +- [ ] **Step 3:** implement (TS builder + Rust structs). +- [ ] **Step 4:** PASS; both `tsc`; lint; full `cargo test --lib` shows only the 8 known failures. +- [ ] **Step 5:** commit `feat(compositor): carry extra cameras, camera settings and layout regions in the scene`. + +--- + +### Task 4: Plan camera layers per frame (Rust, pure) + +**Files:** +- Create: `crates/compositor/src/camera_layers.rs` (register in `lib.rs`). +- Modify: `crates/compositor/src/frame_geometry.rs` — `FrameGeometry` gains `camera_layers: Vec`; `plan_frame` (~:2953-3556) fills it. + +**Interfaces:** +- Consumes: Task 3 `SceneCameraLayoutRegion`, `SceneCamera`; `regions.rs` `TRANSITION_WINDOW_S`, `FULLSCREEN_LEAD_OUT_WINDOW_S`, `ease_out_screen_studio`, `ScreenClock` (how `camera_fullscreen_region_phase` turns `t` into region time — copy that). +- Produces: + ```rust + #[derive(Debug, Clone, Copy, PartialEq)] + pub struct CameraLayerPlan { + pub camera: usize, + pub dst: [f32; 4], // x, y, w, h in output fractions + pub radius_frac: f32, // of min(dst w, h) in pixels + pub shape: u32, // webcam_shape_code vocabulary + pub opacity: f32, // 0..1 + pub fills_frame: bool, + } + /// `default_cam0`: camera 0's layer as today's plan places it (None = camera 0 not drawn). + pub fn camera_layers_at(regions: &[SceneCameraLayoutRegion], t: f32, clock: &ScreenClock, + default_cam0: Option) -> Vec; + ``` + `FrameGeometry.camera_layers` is **empty when `scene.camera_layout_regions` is empty** (backends then run today's camera-0 path untouched). Otherwise it holds every layer to draw, in draw order, including camera 0's (whose `dst`/opacity backends must then take from the plan instead of `w_dst`). + +Algorithm: +1. Find the region `r` with `start ≤ t ≤ end` (region time per the clock). None → return `default_cam0` as a one-element vec (or empty). +2. `len = end - start`; `win_in = min(TRANSITION_WINDOW_S, len/2)`; `win_out = min(FULLSCREEN_LEAD_OUT_WINDOW_S, len/2)`. +3. `prev` = layers of a region whose `end` is within 1 ms of `r.start` (adjacent), else the default set. `next` = adjacent region starting within 1 ms of `r.end`, else default. +4. If `t - start < win_in`: `blend(prev, r, ease((t-start)/win_in))`. Else if `end - t < win_out` **and there is no adjacent next region**: `blend(r, default, 1 - ease((end-t)/win_out))`. Else `r`'s layers at full opacity. (With an adjacent next region the hand-over happens in that region's lead-in — Review Focus 4.) +5. `blend(a, b, k)`: for each camera in `a ∪ b`: in both → lerp `dst`, `radius_frac`; `shape`/`fills_frame` from `b` when `k ≥ 0.5` else `a`; opacity 1. Only in `a` → `a`'s geometry, opacity `1-k`. Only in `b` → `b`'s geometry, opacity `k`. Layers with opacity ≤ 1e-3 are dropped. +6. Draw order: `fills_frame` layers first, then the rest; within each group, the order of the target set (`b`, else `a`). +`shape` from the scene string via `webcam_shape_code`; `radius_frac` from `SceneCameraLayer.radius_frac`. + +`plan_frame` integration: when regions are non-empty, `default_cam0 = Some(CameraLayerPlan { camera: 0, dst: w_dst, radius_frac: w_radius / min(w_px.x, w_px.y), shape: lp.webcam_shape, opacity: 1.0, fills_frame: cam_progress >= 1.0 })` if `lp.has_webcam`, else `None`; then `camera_layers = camera_layers_at(&scene.camera_layout_regions, t, &clock, default_cam0)` with the same `t`/clock the Full Camera code uses. + +- [ ] **Step 1: Failing tests** in `camera_layers.rs` `mod tests` (helper `fn region(start, end, layers: &[(usize, [f32;4], bool)]) -> SceneCameraLayoutRegion` and a default camera-0 PiP at `[0.75,0.7,0.22,0.22]`): + - `outside_every_region_only_the_default_camera_is_drawn` + - `inside_a_region_after_the_lead_in_its_layers_are_drawn_fully` (cam 1 full + cam 0 PiP, t mid-region → exact dsts, opacity 1, cam 1 first) + - `entering_a_region_glides_camera_0_and_fades_in_a_new_camera` (t = start + win_in/2 → cam 0 dst strictly between default and target; cam 1 opacity strictly between 0 and 1) + - `leaving_a_region_returns_to_the_default` + - `adjacent_regions_glide_directly` (A: cam 1 full, B: cam 2 full, B.start == A.end; at A.end − 0.01 cam 1 opacity 1 and **no default PiP geometry appears**; at B.start + win_in/2 cam 1 fading out, cam 2 fading in; at no sampled t in [A.end−0.2, B.start+0.2] does camera 0's default PiP dst appear unless it is in A or B) + - `a_short_region_halves_its_windows` + - `fills_frame_layers_are_drawn_first` + - In `frame_geometry.rs` tests: `without_layout_regions_the_plan_has_no_camera_layers` (golden_scene → `camera_layers.is_empty()` and every existing webcam field unchanged vs a snapshot taken by the same test before the call path — compare against `plan_frame` on the scene JSON without the new keys) and `with_a_layout_region_camera_0_comes_from_the_plan` (patch `golden_scene` JSON with a `cameraLayoutRegions` entry → a layer for camera 0 at the region's rect mid-region). +- [ ] **Step 2:** `cargo test -p openscreen-compositor --lib camera_layers` → FAIL. +- [ ] **Step 3:** implement. **Step 4:** PASS + full lib run (only the 8 known failures). **Step 5:** commit `feat(compositor): plan camera layers with a glide between layout regions`. + +--- + +### Task 5: `LayerCB` homography and transparency lanes + shaders (all backends) + +**Files:** +- Modify: `crates/compositor/src/frame_geometry.rs` — `LayerCB` (~:45-68), size/offset test (~:7345). +- Modify: `crates/compositor/src/shaders.hlsl`, `crates/compositor/src/vk_shaders/layer.wgsl`, `crates/compositor/src/shaders.metal` — the layer constant buffer and the camera (video, mode 0) fragment path. +- Test: `frame_geometry.rs` (size/offset, naga test ~:8918 still green), Linux pixel test module (`compositor_linux.rs` `mod tests` ~:4053, helpers `compose_pip_scene`, `camera_pixels`) — **runs on Linux CI**; on Windows add the equivalent check only if a D3D11 pixel test harness already exists (search `compositor_windows.rs` tests), else say so. + +**Interfaces:** +- Produces: `LayerCB` gains, appended last: + ```rust + pub persp: [[f32; 4]; 3], // rows of H (target uv → camera uv): [h0,h1,h2,0],[h3,h4,h5,0],[h6,h7,h8,0] + pub layer_fx: [f32; 4], // x = transparency (0 = opaque), y = 1 when persp applies, z,w = 0 + ``` + New size **256 bytes** (192 + 48 + 16). Every existing `LayerCB { .. }` literal must compile with the new fields zeroed (add them where literals are exhaustive, or rely on an existing `Default`/`..` pattern — check how literals are written and follow it). + +Shader rule (camera / video mode only; all other modes ignore `persp`): +- `local` = the fragment's position inside the layer's `dst` quad, 0..1 (the shader already derives the texture uv from `dst`→`src`; compute `local` the same way it computes the quad-relative coordinate). +- If `layer_fx.y > 0.5`: `q = (dot(persp[0].xyz, (local,1)), dot(persp[1].xyz, (local,1)), dot(persp[2].xyz, (local,1)))`; `cam = q.xy / q.z`; if `q.z <= 0` or `cam` outside `[0,1]²` → output transparent; texture uv = `cam * fx.xy` (`fx.xy` = valid fraction of the decoder texture, as today). The segmentation mask (camera 0) samples the same uv. +- Final colour alpha (premultiplied as the shader already does) is multiplied by `1 - layer_fx.x` for **every** mode. +- With `layer_fx = 0` the shader output is bit-identical to today. + +- [ ] **Step 1: Failing tests:** size/offset test updated to 256 and `offset_of!(LayerCB, persp) == 192`, `offset_of!(LayerCB, layer_fx) == 240`; Linux pixel tests: `identity_lanes_leave_the_camera_pixels_unchanged` (compose the PiP scene twice — once as today, once with `layer_fx = 0` explicitly — equal RGBA), `transparency_half_halves_the_camera_over_the_screen` (camera white over black screen, `layer_fx.x = 0.5` → camera pixels ≈ 128 ± 2), `a_homography_maps_the_camera_quad_to_the_layer` (camera texture with a red left half / blue right half, H = horizontal flip `[-1,0,1, 0,1,0, 0,0,1]` → left half of the layer blue). Wire the test-only values by building the `LayerCB` in the test through the same function the webcam block uses, or by a small test hook — follow the existing test style. +- [ ] **Step 2:** run → FAIL (size assertion, missing fields). +- [ ] **Step 3:** implement Rust struct + the three shaders. Keep HLSL/WGSL/MSL field order identical to Rust. +- [ ] **Step 4:** `cargo test --lib` (size/offset, naga validation of `layer.wgsl` with `LAYER_MODELS` false/true) PASS; HLSL compiles on this host (the crate's build compiles shaders — confirm in the report how); Linux pixel tests are compiled here (`cargo test --lib --no-run` on Windows builds only the Windows backend — say which pixel tests ran where). +- [ ] **Step 5:** commit `feat(compositor): homography and transparency lanes for camera layers`. + +--- + +### Task 6: Draw the planned camera layers (all backends) + +**Files:** +- Modify: `crates/compositor/src/compositor_windows.rs` (webcam block ~:2424-2534, `compose_frame` ~:1983, `nv12_srvs` ~:874), `compositor_linux.rs` (~:2420-2530, webcam-pass ~:3382), `compositor_macos.rs` (~:2660-2761). +- Test: Linux pixel tests (`compositor_linux.rs` tests), `frame_geometry.rs` if helpers move there. + +**Interfaces:** +- Consumes: Task 4 `FrameGeometry.camera_layers`, Task 5 lanes, Task 3 `Scene::camera(i)`. +- Produces (each backend's `Compositor`): + ```rust + /// Frames for cameras 1..=3 (index 0 = camera 1 of the extras, i.e. scene camera index 1). + /// Null or missing → that camera is not drawn this frame. Valid until the next call. + pub unsafe fn set_extra_camera_frames(&self, frames: &[*const AVFrame]); + ``` + Shared helper (put it in `frame_geometry.rs` so all three backends call the same code): + ```rust + pub(crate) fn camera_layer_cb(plan: &CameraLayerPlan, camera: Option<&SceneCamera>, visible_px: [f32; 2], + tex_px: [f32; 2], render: [f32; 2], base: &LayerCB) -> LayerCB; + ``` + It returns `base` (camera 0's today-built CB, or a zeroed video CB for extras) with `dst = plan.dst`, `quad_px`, `radius_px = plan.radius_frac * min(quad_px)`, `src` from `webcam_source_rect(visible, tex, crop, box_ar)` with the camera's crop (none when a homography is set or `fills_frame` desk full-frame applies) and flips from the camera's rotation/mirror (`flip_u = mirror ^ turned`, `flip_v = turned`; extras: mirror default **false**), `persp`/`layer_fx.y` from `homography`, `layer_fx.x = 1 - plan.opacity`, motion-blur `dst_prev = dst` (no trail for extras). + +Rules: +- `camera_layers.is_empty()` → today's code path, byte-for-byte unchanged — except that when `Scene::camera(0)` carries a `homography`, the camera-0 CB gets `persp` + `layer_fx.y = 1` (and no crop). Test: `camera_0_perspective_applies_without_layout_regions` (Linux pixel test, red/blue halves + flip matrix). +- Otherwise draw `camera_layers` in order, in the place where the webcam is drawn today (after cursor, before annotations). Camera 0's layer uses today's frame/texture, shadow and effects (segmentation, cutout, blur, custom bg); its `dst`/radius/opacity come from its plan layer; if camera 0 has no layer it is not drawn. Extra cameras: no effects (effect code 0), shadow only when `!fills_frame` and shadows are enabled (same constants as camera 0), texture from `set_extra_camera_frames` — **tolerant** (`.ok()`; null → skip, Review Focus 1), even on Windows. +- Windows `nv12_srvs` caches by decoder texture address; extra frames go through the same cache (clear it where it is cleared today). +- Linux: one uniform buffer + bind group per layer (as today per draw); inside the existing "webcam-pass". +- macOS: same structure inside the webcam encoder; compile-only. + +- [ ] **Step 1: Failing Linux pixel tests:** `two_cameras_draw_in_their_planned_rects` (screen black, camera 0 white, camera 1 green; scene with a `camera-full-pip` region: camera 1 full, camera 0 PiP → center pixel green, PiP rect white), `a_missing_extra_camera_skips_its_layer` (no frame set for camera 1 → center pixel = screen), `a_fading_extra_camera_is_half_transparent` (plan opacity 0.5 via a mid lead-in time). Plus a Windows unit test of `camera_layer_cb` (pure): homography sets `layer_fx.y = 1` and ignores crop; mirror flips `src` u-bounds; opacity 0.25 → `layer_fx.x = 0.75`. +- [ ] **Step 2:** run → FAIL. **Step 3:** implement the three backends (Windows and Linux fully, macOS mirrored). **Step 4:** `cargo test --lib` (only the 8 known failures); `cargo build` for the Windows backend; report that the Linux tests ran only if a Linux toolchain was available (otherwise CI) and that Metal is compile-checked only in CI. **Step 5:** commit `feat(compositor): draw every planned camera layer`. + +--- + +### Task 7: Live preview decodes the extra cameras (Rust live + napi + TS) + +**Files:** +- Modify: `crates/compositor/src/live.rs` (`PrefetchedClip` ~:65, `open_and_seek_clip` ~:151, `PooledClip` ~:184, `Player` ~:296, `step` ~:540-607, `present_frame` ~:643, recompose ~:634, render thread clip switch ~:1274/1400, `set_live_params` ~:1670). +- Modify: `crates/compositor-view-napi/src/lib.rs` (`set_active_clip` ~:372, `create_view` ~:142 if it takes clip paths). +- Modify: `src/components/ai-edition/NativeCompositorOverlay.tsx` (~:275, ~:351) and the napi typings it uses (find the `setActiveClip` declaration in `src/native/` / `electron/` preload). +- Test: `live.rs` tests (~:2325). + +**Interfaces:** +- Consumes: Task 3 `SceneClip.additional_cameras`, `Scene.camera_layout_regions`; Task 6 `set_extra_camera_frames`. +- Produces: `Player.extra: Vec>` with `struct ExtraCamera { dec: Decoder, offset_sec: f64 }` (index k-1 = camera k); napi `set_active_clip(id, screen_path, webcam_path, webcam_offset_sec, clip_index, source_time_sec, additional_cameras: Option>)` where `ClipCameraInput { path: String, offset_sec: f64 }` (camelCase in JS); TS passes `assetAdditionalCameraSources(asset)`. + +Rules: +- A clip opens an extra camera's decoder only if the clip's windowed `camera_layout_regions` reference that camera (`scene.for_clip_window(..)`), and only for a non-empty path; a failed open → `None` with one warning line (Review Focus 1). Pool key extends to the extra paths/offsets (so a pooled pair is reused only for the same set). +- Each step/seek: advance/seek each open extra like `wdec` (`webcam_seek_time(t, offset)`, `frame_step` by pts) **only when** a layout region referencing that camera is within `[region.start - PREFETCH_LEAD_SEC, region.end]` of `t`; otherwise leave it idle. Then `comp.set_extra_camera_frames(&frames)` (null for idle/None/ended) right before every `compose_frame` call (step, present_frame, recompose). +- A finished extra (EOF) yields null — the layer disappears (Review Focus 2). +- Without extras and without layout regions: no new decoder, no new call cost beyond an empty `set_extra_camera_frames(&[])`. + +- [ ] **Step 1: Failing tests** (pure helpers so they run without a GPU): `extra_cameras_to_open_are_those_the_clips_regions_reference` (fn `extra_cameras_for_clip(scene, clip_index) -> Vec`), `an_extra_camera_is_decoded_only_near_its_regions` (fn `extra_camera_active(regions, camera, t) -> bool`), `a_missing_extra_camera_skips_its_layer` (fn that maps a failed open to `None` and the frame list to null — test the mapping helper), `the_pool_key_includes_the_extra_cameras`. +- [ ] **Step 2:** `cargo test --lib live::` → FAIL. **Step 3:** implement live + napi + TS. **Step 4:** PASS; `npx tsc --noEmit`; build the addon (`npm run build:native:compositor`, copy to `bin/win32-arm64`) and confirm it loads in the dev app with a normal one-camera project (no regressions; log line `[compositor-view]` without "not present"). **Step 5:** commit `feat(preview): decode the extra cameras a layout shows`. + +--- + +### Task 8: Export decodes the extra cameras (timeline walk + producers) + +**Files:** +- Modify: `crates/compositor/src/timeline_walk.rs` (`walk_composited_timeline` ~:149-363), the three `run_composited_multi` callers (`pipeline_windows.rs` ~:1293/1652/1724, `pipeline_macos.rs` ~:1133/1161/1235, `pipeline_linux.rs` ~:845/947/1032), `ClipSource` (~:1278 / ~:659 / ~:50), `gif_export.rs` caller. +- Modify: `crates/compositor-view-napi/src/lib.rs` (`ClipInput` ~:498, `export_multi` ~:697, GIF ~:887). +- Modify: `src/components/ai-edition/ExportDialog.tsx` (`buildNativeClipList` ~:112), `src/cli/CliExportRunner.tsx` (~:82) — both add `additionalCameras: assetAdditionalCameraSources(asset)` (only when non-empty). +- Test: `timeline_walk.rs` tests (~:432). + +**Interfaces:** +- Consumes: Task 3, Task 6 `set_extra_camera_frames`, Task 7 `extra_cameras_for_clip` / `extra_camera_active` (reuse, don't duplicate — move them to a shared module if they live in `live.rs`). +- Produces: `ClipSource.additional_cameras: Vec` (`ClipCamera { path: String, offset_sec: f64 }`); `walk_composited_timeline(.., extra_decs: &mut HashMap, ..)` (keyed by path like `webcam_decs`). + +Rules: +- Per clip, open (once per path) the extra cameras the clip's regions reference; a failed open → warning, camera skipped for the whole export (Review Focus 1). +- **Do not** clamp the clip end to an extra camera's duration (only camera 0 clamps, as today — Review Focus 2). +- Per output frame: for each active extra (`extra_camera_active`), `advance_decoder_to(dec, t, offset)`; `cur_frame()` or null after EOF; `comp.set_extra_camera_frames(..)` before `compose_frame`. +- Deterministic mode unchanged; extras have no segmentation. + +- [ ] **Step 1: Failing tests** (pure where possible): `a_short_extra_camera_does_not_shorten_the_clip` (the clip-end computation helper with an extra camera shorter than camera 0 → unchanged end), `export_skips_an_extra_camera_that_will_not_open` (open-result mapping), `extra_offsets_move_into_the_screen_clock` (like `offset_moves_the_webcam_into_the_screen_clock`). TS: `ExportDialog`/`CliExportRunner` clip lists include `additionalCameras` for an asset with an extra track (extend their existing tests if present; otherwise add a small unit test of `buildNativeClipList`). +- [ ] **Step 2:** run → FAIL. **Step 3:** implement. **Step 4:** PASS; both `tsc`; lint; `cargo test --lib` (only the 8 known failures). **Step 5:** commit `feat(export): decode the extra cameras a layout shows`. + +--- + +### Task 9: End-to-end measurement, performance and checklist + +**Files:** +- Create: `scripts/multicam-fixture.mjs` (generates synthetic sources + a project file; dev tool, documented in its header). +- Modify: `technical-documentation/testing/manual-e2e-checklist.md` — a "Several cameras in the picture" item + results-log row. + +- [ ] **Step 1: Fixture.** With the vendored ffmpeg (`crates/thirdparty/ffmpeg-n8.1.2-win64-lgpl-shared/bin/ffmpeg.exe`): screen 1920×1080 dark grey 8 s; camera 0 solid red 1280×720; camera 1 solid green 1280×720; camera 2 a checkerboard 1280×720 photographed "in perspective" (`ffmpeg -vf perspective=` with known corners, so the true corners are known). Write a `.openscreen` project (copy the shape of an existing saved project under `%APPDATA%\openscreen\projects\`, or build it from the schema) with `additionalCameraTracks` for cameras 1–2, `legacyEditor.cameraLayoutRegions`: `screen-pip [0,1]` 1–3 s, `camera-full-pip [1,0]` 3–5 s (adjacent), `side-by-side [0,1]` 5.5–7 s, and `legacyEditor.cameraSettings[2].perspective` = the known corners with the checkerboard's aspect, plus a `camera-full [2]` region 7–8 s. +- [ ] **Step 2: Headless export** `electron . export -o out.mp4` (dev build, `OPENSCREEN_COMPOSITOR_VIEW_NODE` set). Extract frames with `ffmpeg -ss -i out.mp4 -frames:v 1 -c:v ppm` at 2.0 s, 4.0 s, 6.0 s, 7.5 s and every 1/30 s across 2.9–3.6 s; parse the P6 PPMs (≤ 20-line Node parser) and assert: at 2.0 s two PiPs (red, green) at the template rects ±2 px; at 4.0 s green fills the frame with a red PiP; across the 3 s boundary green grows from its PiP rect to the full frame without passing through "no green" (Review Focus 4); at 6.0 s left half red / right half green; at 7.5 s the checkerboard's squares are axis-aligned (edge positions on a scanline equally spaced ±2 px). Put the assertions in the script; print a pass/fail table. +- [ ] **Step 3: Preview = export.** Grab the dev app's preview at 4.0 s (computer-use not available → use the compositor's frame capture if one exists, else skip and say so) and compare. +- [ ] **Step 4: Performance.** Export time for the fixture vs the same project with layout regions removed (one camera): report both; target ≤ 2× (spec). Live preview: in the dev app, play the fixture and read the compositor's frame-pacing log (find the live loop's timing log line) — report dropped/late frames with 1 vs 3 cameras. If targets are missed, report the numbers; do not tune silently. +- [ ] **Step 5: Checklist + commit.** Checklist item (English, next to the camera items): layouts with real cameras (desk full + face PiP, screen + 2 PiP, side by side), glide, perspective with an A4 sheet, export vs preview, performance with 3 cameras; results-log row marked **Partial** (real-device run with the user's cameras outstanding). Commit `test(compositor): measure several camera layers end to end`. diff --git a/docs/superpowers/plans/2026-10-05-multi-camera-editor.md b/docs/superpowers/plans/2026-10-05-multi-camera-editor.md new file mode 100644 index 0000000000..0eaf962456 --- /dev/null +++ b/docs/superpowers/plans/2026-10-05-multi-camera-editor.md @@ -0,0 +1,258 @@ +# Multi-Camera Editor (Sub-project 3) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** In Studio, the user places layout sections on the timeline, picks a template and which camera goes into each place, moves/resizes camera windows in the preview, sets rotation/mirror/crop per camera, and calibrates a perspective correction with four corner handles (optionally pre-placed by detected ArUco markers from a printable sheet). + +**Architecture:** Full Camera regions (camera 1, incl. the desk view) stay exactly where and how they are stored today (`legacyEditor.cameraFullscreenRegions`, all existing writers untouched); every *other* layout (templates `screen-pip`, `camera-full-pip`, `side-by-side`, and `camera-full` with a camera other than 1) lives in `legacyEditor.cameraLayoutRegions`, clip-anchored exactly like Full Camera regions. The two lists are disjoint by construction and share one timeline lane with a cross-list non-overlap rule. Per-camera settings live in `legacyEditor.cameraSettings`. The compositor side (sub-project 2) already renders all of it. + +**Tech Stack:** React 19 / TypeScript (v4 editor), Zustand project store with `saveDocument(…, {history:true})` undo, Vitest (+ jsdom, Testing Library), Playwright (`_electron`), `js-aruco2` 2.0.0 (MIT, pure JS) for marker detection. + +**Spec:** `docs/superpowers/specs/2026-10-05-multi-camera-layouts-design.md` — sections 3 and 6, data model in section 1. **Deviation (recorded in Task 1, spec updated there):** section 1's "load converts every Full Camera region into a layout region; save writes both lists" is replaced by the disjoint two-list storage above, because six existing writers of `cameraFullscreenRegions` (`useTimeline` add/update/remove, copy/paste in `NewEditorShell`, clip re-anchoring in `document/timeline.ts`, and the main-process agent tools in `electron/ai-edition/agent-tools.ts`) would otherwise go silently dead in preview and export. Spec section 6's first bullet changes accordingly. + +**Facts file (read before any task):** `.superpowers/editor-facts.md` — file:line map of today's Full Camera path through store, undo, persistence, timeline, inspector, preview drag, dialogs, shortcuts, i18n and tests. Line numbers are approximate (`~`). + +**Worktree / branch:** `C:\osc-cams`, branch `feat/multi-camera-layouts`, plan base = the commit that adds this plan. `crates/thirdparty` is a junction to `C:\osc-multicam`. `node_modules` is a junction too — Task 9 replaces it with a real install (see there) before adding the dependency. + +## Global Constraints + +- Camera index convention: **0 = camera 1** (`asset.cameraTrack`), **k ≥ 1 = `asset.additionalCameraTracks[k-1]`**. Labels: camera 1 → "Kamera 1" (no label exists); camera k → its track `label`, else "Kamera k+1". Max 4 cameras. +- **Storage rule (deviation above):** a layout section that is `camera-full` with camera 0 is a `CameraFullscreenRegion` in `cameraFullscreenRegions`; everything else is a `CameraLayoutRegion` in `cameraLayoutRegions`. Never both. Both lists are clip-anchored (`clipId`, `assetId`, `sourceStartSec`, `sourceEndSec` + derived `startMs/endMs`) via `anchorRegionsWithDerivedMs`. +- **One shared lane, no overlap across the two lists**: add refuses (existing "cannot place" notice pattern), move/resize clamps against neighbours of *both* lists (`clampSpanAgainstNeighbours` semantics). +- **Camera 1 settings stay in the existing fields** (`webcamMirrored`, desk-view regions, `webcamCropRegion`); `cameraSettings[0]` only ever carries `perspective`. Extras use `cameraSettings[k]` (`rotation`, `mirror`, `crop`, `perspective`). +- Desk-view switches (rotation/mirror/label per section) only on Full Camera regions (camera 1). The normalizer already strips them elsewhere. +- Templates with PiP places (`screen-pip`, `camera-full-pip`) are disabled with a hint under the block layout presets `dual-frame` / `vertical-stack`; templates needing ≥2 cameras are disabled with a hint when the clip's asset has only one camera. +- A circle/square place keeps a square rect; dragging/resizing such a place keeps the aspect. +- Every editor change is **one undo step** via `saveDocument(doc, { history: true })` (live drags: `setDocument(…, {history:false})` + one commit with `historyBase`). **Every new writer gets a row in `src/lib/ai-edition/store/documentWriteAudit.test.ts`.** (The spec's mention of `useEditorHistory` is obsolete — v4 does not use it.) +- Default/unused values are not stored (a reset field is deleted), so untouched projects stay byte-identical. +- i18n: new strings in **all 15 locales** (`src/i18n/locales//*.json`) with real translations; `npm run i18n:check` passes; no hard-coded English in new UI (the existing hard-coded `"Full Camera"` pill label is replaced by a translated label in Task 4). +- Tests per AGENTS.md: focused `npx vitest --run ` while working; jsdom only where a DOM is needed; both `tsc` (`npx tsc --noEmit`, `npx tsc -p tsconfig.test.json --noEmit`); `npm run lint` 0 errors (warnings stay 26). +- Comments in English; Biome style; no new `any`; hooks at top level. Commit messages Conventional-Commits style, **no `Co-Authored-By` / Claude trailer**, do not push. + +## Review Focus + +1. **A Full Camera section edited by any existing path** (C key, inspector desk switches, copy/paste, clip trim/move, AI agent tool) keeps working in preview/export after layout sections exist in the same project. Pinned in Task 1 (`legacy full camera regions still reach the scene next to layout regions`). +2. **A clip is trimmed or moved under a layout section** → the section follows the clip like a Full Camera section (no stale times). Pinned in Task 1 (`mapAllRegionCollections re-anchors layout regions`). +3. **Adding a layout section where any section (either list) already is** → refused with the existing notice, nothing silently dropped. Pinned in Task 2 (`adding over a full camera region is refused`). +4. **A camera chosen in a section is later missing** (track hidden/relinked away) → the place is empty, the inspector shows the camera as unavailable, the project opens. Pinned in Task 5 (`a slot whose camera is gone shows as unavailable`). +5. **Calibration with markers not all found** → message, handles unchanged; **degenerate quad** (crossed/collinear handles) → "Übernehmen" disabled with a hint. Pinned in Task 8 (`apply is disabled for a crossed quad`) and Task 9 (`fewer than four markers leaves the corners unchanged`). + +--- + +### Task 1: Storage, anchoring and the scene (disjoint lists) + +**Files:** +- Modify: `src/lib/cameraLayouts.ts` (`normalizeCameraLayoutRegions` ~:96) — keep anchor fields; never accept `camera-full` + camera 0 (drop such rows — they belong to Full Camera). +- Modify: `src/lib/ai-edition/document/timeline.ts` — `RegionKind` gains `"cameraLayout"` (~:39); `mapAllRegionCollections` (~:355) and `anchoredRegionsOf` (~:722) include `cameraLayoutRegions` (prefix `"camlayout"`); `removeRegion` (~:1080) case; `EDIT_REGIONS` (~:1153) entry. +- Modify: `src/lib/ai-edition/store/regionClipboard.ts` (~:13) — union gains `{ kind: "cameraLayout"; region }`. +- Modify: `src/native/sceneDescription.ts` (~:1150-1190) — always use `cameraFullscreenRegions`; project `cameraLayoutRegions` through the **anchored** branch of `projectRegionsToSource` like Full Camera; remove the rule "ignore the legacy list when `cameraLayoutRegions` is an array" (it was only right for the replaced storage); keep converting a hand-written `camera-full`+camera 0 layout row only if no Full Camera region covers the same span (de-dup by span). +- Modify: `docs/superpowers/specs/2026-10-05-multi-camera-layouts-design.md` — section 1 "Bestehende Projekte" and section 6 first bullet rewritten to the disjoint two-list rule (German, like the spec). +- Test: `src/lib/cameraLayouts.test.ts`, `src/lib/ai-edition/document/timeline.test.ts`, `src/native/sceneDescription.test.ts`. + +**Interfaces:** +- Produces: + ```ts + // cameraLayouts.ts + export type AnchoredCameraLayoutRegion = CameraLayoutRegion & { clipId?: string; assetId?: string; sourceStartSec?: number; sourceEndSec?: number }; + export function normalizeCameraLayoutRegions(raw: unknown): AnchoredCameraLayoutRegion[]; // keeps finite anchor fields + export function isFullCameraLayout(region: Pick): boolean; // camera-full with camera 0 + // document/timeline.ts + export type RegionKind = "zoom" | "trim" | "annotation" | "speed" | "cameraFullscreen" | "cameraLayout" | "audio"; + ``` + Overlap in the normalizer: keep dropping a later overlapping row **only within the layout list**; cross-list overlaps are prevented by the editor (Task 2), not repaired here. + +- [ ] **Step 1: Failing tests.** + - `cameraLayouts.test.ts`: `keeps the clip anchor fields`; `drops a camera-full row for camera 1 (it belongs to full camera)`. + - `timeline.test.ts`: `mapAllRegionCollections re-anchors layout regions` (a layout row anchored to clip A; move clip A → row's `startMs/endMs` follow, as the existing camfull test at ~:1574 does); `removeRegion removes a layout pill`; `clearEditRegions clears layout regions`. + - `sceneDescription.test.ts`: `legacy full camera regions still reach the scene next to layout regions` (doc with one Full Camera region 1–3 s and one `screen-pip` layout region 4–6 s → `cameraFullscreenRegions` has the first, `cameraLayoutRegions` the second); `a layout region follows its clip` (anchored row + a clip whose timeline start moved → projected seconds follow the anchor); replace the existing test that asserted "legacy list ignored when cameraLayoutRegions present" with `a hand-written camera-1 camera-full layout row is not doubled` (same span in both → one Full Camera region, one desk label). +- [ ] **Step 2:** run those files → FAIL. +- [ ] **Step 3:** implement; update the spec text. +- [ ] **Step 4:** PASS; both `tsc`; lint. +- [ ] **Step 5:** commit `feat(editor): store layout sections next to full camera regions`. + +--- + +### Task 2: Layout section mutators in `useTimeline` + +**Files:** +- Modify: `src/lib/ai-edition/store/useTimeline.ts` (pattern: `addCameraFullscreen` ~:547, `updateCameraFullscreenSpan` ~:1032, `updateCameraFullscreenOrientation` ~:1084, `removeRegion(s)` ~:1134/1150, live/commit pattern `updateZoomFocusLive`/`commitZoomFocus` ~:779/811). +- Modify: `src/lib/ai-edition/store/documentWriteAudit.test.ts` — one row per new writer. +- Test: `src/lib/ai-edition/store/useTimeline.test.ts` (jsdom, `renderHook` + `I18nProvider`, bridge mocked — see facts §10). + +**Interfaces:** +- Consumes: Task 1 types, `isFullCameraLayout`, `TEMPLATE_SLOTS`. +- Produces (on the `useTimeline()` return): + ```ts + cameraLayoutRegions: AnchoredCameraLayoutRegion[]; + addCameraLayout(template: CameraLayoutTemplate, cameras: number[], durationSec?: number): Promise<"added" | "occupied" | "no-camera">; + updateCameraLayoutSpan(id: string, startMs: number, endMs: number): Promise; // clamps against BOTH lists + setLayoutTemplate(handle: { kind: "cameraFullscreen" | "cameraLayout"; id: string }, template: CameraLayoutTemplate, cameras: number[]): Promise<{ kind: "cameraFullscreen" | "cameraLayout"; id: string }>; + setLayoutSlotCamera(id: string, slotIndex: number, camera: number): Promise; // unique per section: swaps if already used + updateLayoutSlotRectLive(id: string, slotIndex: number, rect: NormalizedRect): void; // setDocument history:false + commitLayoutSlotRect(): Promise; // one undo step + resetLayoutSlotRects(id: string): Promise; + ``` +Rules: +- `addCameraLayout` places at the playhead like `addCameraFullscreen` (same default duration, same `hasAnyClipWithCamera` gate → `"no-camera"`); if the span overlaps any row of either list → `"occupied"` and no write (the caller shows the existing "cannot place" notice). `camera-full` with `[0]` delegates to `addCameraFullscreen`. +- `setLayoutTemplate` converts across lists in **one** `saveDocument`: Full Camera → layout (drop the FC pill's rows, add layout rows on the same anchors/span, desk fields dropped), layout → Full Camera when the new template is `camera-full` with camera 0 (desk fields default), otherwise patch template+slots (slots: keep cameras in order, fill missing from the given `cameras`, cut to `TEMPLATE_SLOTS.max`). Returns the new handle so the selection can follow. +- Slot rect changes only touch `slots[i].rect`; reset deletes every `rect`. +- Pill semantics: like Full Camera, a section crossing a clip cut is stored as one row per clip and edited as one pill (`patchPillById` / `replacePillSpan` with the `"camlayout"` id prefix). + +- [ ] **Step 1: Failing tests** in `useTimeline.test.ts` (new `describe("useTimeline camera layouts")`): `adds a screen-pip section at the playhead`; `adding over a full camera region is refused` (→ `"occupied"`, document unchanged); `camera-full with camera 1 is added as a full camera region`; `moving a layout section clamps at a full camera neighbour`; `switching a full camera section to camera-full-pip moves it to the layout list in one undo step` (undo restores the Full Camera region); `switching back to camera-full with camera 1 restores a full camera region`; `choosing a camera already in the section swaps the two places`; `a slot rect drag is one undo step`; `reset removes every slot rect`. Add the audit rows (Step 3) — the audit test fails first. +- [ ] **Step 2:** run → FAIL. **Step 3:** implement. **Step 4:** PASS, `tsc` ×2, lint. **Step 5:** commit `feat(editor): add, move and convert layout sections`. + +--- + +### Task 3: Camera list and per-camera settings + +**Files:** +- Create: `src/lib/ai-edition/timeline/cameraList.ts` + `cameraList.test.ts` (node). +- Modify: `src/lib/ai-edition/store/useTimeline.ts` (or a new small `useCameraSettings.ts` hook next to `useEditorSettings.ts` — choose the one that keeps the writer in one place) — `setCameraSettings`. +- Modify: `documentWriteAudit.test.ts`. + +**Interfaces:** +- Produces: + ```ts + export interface ProjectCamera { index: number; label: string; path: string; available: boolean; width?: number; height?: number } + export function projectCameras(asset: AxcutAsset | undefined, t: (key: string, vars?: Record) => string): ProjectCamera[]; + // available = visible track with a non-empty sourcePath; label per Global Constraints (`t("cameras.cameraN", { n })` fallback) + export function camerasForClipAt(document: AxcutDocument, timelineSec: number): ProjectCamera[]; // asset of the clip under the playhead + // hook + setCameraSettings(index: number, patch: Partial | null): Promise; // null = reset; index 0 accepts only `perspective` + cameraSettings: (CameraSettings | null)[]; + ``` +Rules: writes `legacyEditor.cameraSettings` (array, length ≤ 4, `null` holes, trailing nulls trimmed, empty → key deleted); defaults deleted; for index 0 any non-`perspective` key is ignored (spec section 6). + +- [ ] **Step 1: Failing tests:** `labels camera 1 and falls back for empty labels`; `a hidden track is unavailable`; `cameras of the clip under the playhead`; hook: `setting camera 2 mirror stores only that`; `resetting the last camera removes the key`; `camera 1 accepts only a perspective`. +- [ ] **Step 2–5:** FAIL → implement → PASS (+ tsc, lint) → commit `feat(editor): list cameras and store their settings`. + +--- + +### Task 4: Timeline "Layout" lane + +**Files:** +- Modify: `src/components/ai-edition/v4/V4Timeline.tsx` (pills ~:757, drag apply ~:986, `laneOf` ~:1416, `pillIcon` ~:1426, lanes ~:2140-2180, toolbar ~:2020-2034, `LanePill.kind` ~:595). +- Modify: `src/components/ai-edition/NewEditorShell.tsx` (copy ~:1200-1207 / paste ~:1140 — make the kind switch explicit for `cameraLayout`; shortcut handler ~:1387 unchanged for C). +- Modify: `src/lib/shortcuts.ts` only if a new shortcut is added (none planned). +- Modify: locale files (`timeline.json`) — lane title "Layout", pill labels per template ("Bildschirm + 2 Kameras", "Tisch voll + Gesicht", …: label = template name + camera labels), add-menu items. +- Test: `src/components/ai-edition/v4/V4Timeline.geometry.test.tsx` style (fake `tl`), new `V4Timeline.layout.test.tsx`. + +Rules: +- The Full Camera lane becomes the "Layout" lane: it renders Full Camera pills (as today, label translated, rotate icon kept) **and** layout pills (icon per template: `screen-pip` → PictureInPicture-style icon, `camera-full` → Maximize2, `camera-full-pip` → layers icon, `side-by-side` → Columns2; use icons already available from `lucide-react` in the repo). +- Pill label: translated template name + " · " + camera labels of its places (from `projectCameras`). +- Drag/resize of a layout pill → `tl.updateCameraLayoutSpan`; click → `tl.selectRegion("cameraLayout", id)`. +- Toolbar: the existing "Add Full Camera" button stays; next to it an "Add layout" menu button listing the four templates (each item disabled with a tooltip when its rule from Global Constraints applies). Choosing an item calls `tl.addCameraLayout(template, defaultCameras)` where `defaultCameras` = the first N available cameras of the clip under the playhead (N = template min; for `camera-full-pip` desk-first: `[1, 0]` if camera 2 exists). `"occupied"` → the existing "cannot place" notice. +- Copy/paste supports `cameraLayout` (no fallthrough into camera full). + +- [ ] **Step 1: Failing tests** (`V4Timeline.layout.test.tsx`, jsdom, fake `tl`): `renders layout pills in the full camera lane`; `a layout pill drag calls updateCameraLayoutSpan`; `the add-layout menu disables multi-camera templates for a one-camera project`; `the pill label names its cameras`. Shell copy/paste: unit-test the kind switch if it is extracted into a pure helper (do that). +- [ ] **Step 2–5:** FAIL → implement (+ all 15 locales) → PASS, `tsc` ×2, lint, `npm run i18n:check` → commit `feat(editor): layout lane on the timeline`. + +--- + +### Task 5: Inspector for layout sections + +**Files:** +- Modify: `src/components/ai-edition/v4/FloatingInspector.tsx` (`SelectionPane` ~:745; Full Camera branch ~:1277-1350). +- Create: `src/components/ai-edition/v4/LayoutSectionPane.tsx` (+ test) — keep the new pane out of the 1400-line file. +- Modify: locale files (`settings.json` → `cameraLayout.*`). + +Rules: +- Selection `cameraLayout` → `LayoutSectionPane`: header (template icon + name), template `ChoiceRow` (disabled entries with hint per Global Constraints), one row per place: label ("Platz 1 · groß" / "Platz 2 · klein" by template) + a select of `projectCameras(...)` (unavailable cameras listed but disabled, the current one shown with "nicht verfügbar"), "Fenster zurücksetzen" (`resetLayoutSlotRects`, disabled when no rect is set), delete. +- Selection `cameraFullscreen` → today's pane **plus** the same template `ChoiceRow` at the top (current = `camera-full`); changing it calls `setLayoutTemplate` and moves the selection to the returned handle. +- Choosing a camera already used in the section swaps (handled by the store). + +- [ ] **Step 1: Failing tests** (`LayoutSectionPane.test.tsx`, jsdom, i18n mocked as in `FloatingInspector.test.tsx`): `shows one camera select per place`; `choosing a template calls setLayoutTemplate`; `a slot whose camera is gone shows as unavailable`; `side-by-side is disabled with one camera`; `pip templates are disabled under dual-frame`; and in `FloatingInspector.test.tsx`: `the full camera pane offers the template choice`. +- [ ] **Step 2–5:** FAIL → implement (+ locales) → PASS, tsc ×2, lint, i18n:check → commit `feat(editor): inspector for layout sections`. + +--- + +### Task 6: Move and resize camera windows in the preview + +**Files:** +- Modify: `src/components/ai-edition/PreviewCanvas.tsx` (layout memo ~:261-311, Full Camera hitbox ~:317-339, drag ~:385-431, slot element ~:482-496). +- Modify: `src/components/ai-edition/NewEditorShell.tsx` (~:1620-1630, pass props like the zoom-focus pair). +- Create: `src/lib/ai-edition/timeline/layoutSlotDrag.ts` + test (pure geometry: move/resize a rect inside the frame, keep square for circle/square, min size, clamp). + +Rules: +- When a `cameraLayout` section is selected and the playhead is inside it (not playing): draw one hitbox per **PiP** place from `resolveCameraLayout(region, ctx)` (same ctx as the scene: frame size, camera aspects incl. perspective aspect, pip shape/radius, `defaultPipRect`); dragging moves, a corner handle resizes (aspect locked for every PiP: width/height ratio of the camera box; square for circle/square); live → `updateLayoutSlotRectLive`, release → `commitLayoutSlotRect`. Frame-filling places have no handles. +- Outside a selected layout section the existing camera-1 anchor drag is unchanged. + +- [ ] **Step 1: Failing tests:** `layoutSlotDrag.test.ts` — `moves inside the frame`, `resize keeps the box aspect`, `a circle stays square`, `clamps to a minimum size`; `PreviewCanvas` jsdom test (if the existing test setup allows; else via the pure helper only — say so): `a selected layout section shows a handle per pip place`. +- [ ] **Step 2–5:** FAIL → implement → PASS (+ tsc, lint) → commit `feat(editor): move and resize camera windows of a layout section`. + +--- + +### Task 7: "Kameras" section in the layout pane + +**Files:** +- Create: `src/components/ai-edition/CamerasSection.tsx` + test; render it inside `LayoutPane` (`src/components/ai-edition/RightPanes.tsx` ~:2782) below the existing camera controls. +- Create: `src/lib/ai-edition/timeline/grabFrame.ts` + test (pattern `probeVideoDimensions`, `src/lib/ai-edition/timeline/duration.ts:89`): `grabFrame(src: string, timeSec: number, maxSide?: number): Promise` and `grabFrameDataUrl(...)`. +- Modify: locale files (`settings.json` → `cameras.*`). + +Rules: +- One row per project camera (`projectCameras` of the clip under the playhead): thumbnail (`grabFrameDataUrl` at the camera time of the playhead, 96 px), label, availability. +- Camera 1 row: only "Perspektive korrigieren…" (+ hint that rotation/mirror/crop for camera 1 are the existing controls above). +- Camera k ≥ 1 rows: rotation 0°/180° `ChoiceRow`, mirror `Toggle`, "Zuschnitt…" and "Perspektive korrigieren…" buttons (open the Task 8 dialog in the matching mode), "Zurücksetzen" when settings exist. +- Camera time = `max(0, sourceTimeSec - offsetSec)` from `locateVirtualPosition` (facts §5). + +- [ ] **Step 1: Failing tests:** `CamerasSection.test.tsx` — `lists every camera of the clip`, `camera 1 offers only the perspective`, `rotating camera 2 calls setCameraSettings`; `grabFrame.test.ts` — with a mocked `HTMLVideoElement`/canvas (jsdom has no media): `seeks to the time and draws the frame`, `rejects on error with a clear message`. +- [ ] **Step 2–5:** FAIL → implement (+ locales) → PASS, tsc ×2, lint, i18n:check → commit `feat(editor): camera list with per-camera settings`. + +--- + +### Task 8: Calibration dialog (perspective + crop) + +**Files:** +- Create: `src/components/ai-edition/CameraCalibrationModal.tsx` + test; `src/lib/ai-edition/timeline/calibrationGeometry.ts` + test (pure: handle hit-testing, loupe source rect, quad validity via `homographyFromUnitSquare`, crop rect clamp, rectified-preview sampling). +- Modify: `CamerasSection.tsx` to open it; locale files (`dialogs.json` → `cameraCalibration.*`). + +**Interfaces:** +- Consumes: `grabFrame` (Task 7), `homographyFromUnitSquare` / `perspectiveMatrix` (`src/lib/cameraPerspective.ts`), `setCameraSettings` (Task 3). +- Produces: ``; `onApply(settingsPatch)` → one `setCameraSettings` call. + +Rules (built on `ModalShell`, `src/components/ai-edition/Modals.tsx:35`, so editor shortcuts and undo are blocked while open): +- Still image of the camera at the playhead (`grabFrame`, full resolution), shown fit-to-dialog. +- **Perspective mode:** four handles (TL, TR, BR, BL; initial = existing corners or an inset rectangle), drag with pointer, arrow keys nudge the focused handle (Shift = 10×), a **loupe** (4× zoom of the area under the active handle) in a corner; target format choice (A4 hoch 210/297, A4 quer 297/210, 16:9, 4:3, 1:1, frei = numeric ratio input), margin slider 0–20 %; a **live rectified preview** (320 px canvas sampled through the homography, nearest-neighbour is enough); (the "Marker erkennen" / "Markerblatt drucken" buttons are added by Task 9); "Zurücksetzen" (removes the perspective), "Übernehmen" (disabled + hint for a degenerate quad), "Abbrechen". +- **Crop mode** (cameras k ≥ 1): a rectangle with corner/edge handles in normalized coords; "Übernehmen" stores `crop`. +- Apply = one undo step; cancel changes nothing. + +- [ ] **Step 1: Failing tests:** `calibrationGeometry.test.ts` — `hit-tests the nearest handle within the radius`, `the loupe rect stays inside the image`, `a crossed quad is invalid`, `the rectified preview maps the quad corners to the canvas corners`; `CameraCalibrationModal.test.tsx` (jsdom, `grabFrame` mocked to a small ImageData): `apply is disabled for a crossed quad`, `apply stores corners, aspect and margin in one call`, `arrow keys move the focused handle`, `crop mode stores a crop`, `cancel stores nothing`. +- [ ] **Step 2–5:** FAIL → implement (+ locales) → PASS, tsc ×2, lint, i18n:check → commit `feat(editor): calibrate a camera's perspective and crop`. + +--- + +### Task 9: ArUco markers — detection and printable sheet + +**Files:** +- Replace the `node_modules` junction in `C:\osc-cams` by a real install first: `Remove-Item C:\osc-cams\node_modules` (removes only the junction — verify with `(Get-Item C:\osc-cams\node_modules).LinkType` = `Junction` before), then `npx -y npm@10.9.4 ci --ignore-scripts` and `node node_modules/electron/install.js`; then `npm install js-aruco2@2.0.0 --save-exact` (adds to `package.json` + `package-lock.json`). +- Create: `src/lib/arucoMarkers.ts` + `src/lib/arucoMarkers.test.ts`; `src/types/js-aruco2.d.ts` (minimal local declarations). +- Create: `src/lib/markerSheet.ts` + test — A4 SVG with markers ID 0–3 (4×4 dictionary), 40 mm each, cut lines, a one-line instruction, generated from the same dictionary the detector uses. +- Modify: `CameraCalibrationModal.tsx` — wire "Marker erkennen" / "Markerblatt drucken"; locale strings. + +**Interfaces:** +- Produces: + ```ts + export type MarkerCorners = [CameraPoint, CameraPoint, CameraPoint, CameraPoint]; // normalized 0..1 + /** Finds markers 0–3 (ARUCO 4x4) and returns the four inner corners as TL (id 0), TR (1), BR (2), BL (3); null unless all four are found. */ + export function detectCornerMarkers(image: ImageData): MarkerCorners | null; + export function markerSheetSvg(): string; // A4 portrait, mm units + export function printMarkerSheet(): void; // hidden iframe + window.print (OS dialog offers "Save as PDF") + ``` + "Inner corner" = the marker corner closest to the centroid of the four markers. + +Rules: import js-aruco2 so it works under Vite (renderer) and Vitest (node) — verify both early (CJS with top-level `this`); if the CJS interop fails, vendor `src/aruco.js`, `src/cv.js` and the 4×4 dictionary under `src/vendor/js-aruco2/` with the MIT license header and an ESM wrapper (say which route was taken). Printing via hidden iframe; no new IPC. + +- [ ] **Step 1: Failing tests:** `arucoMarkers.test.ts` — render the four markers (from the dictionary bits) into an RGBA `ImageData`-like buffer at known positions with a slight perspective (plain JS rasterizer in the test), then `detects four markers and orders the inner corners` (±1.5 px), `fewer than four markers leaves the corners unchanged` (→ null), `a rotated sheet still maps id 0 to top-left`; `markerSheet.test.ts` — `the sheet contains markers 0–3 at 40 mm` (parse the SVG). Modal: `detect fills the handles`, `detect without four markers shows the message and keeps the handles`. +- [ ] **Step 2–5:** FAIL → implement (+ locales) → PASS, tsc ×2, lint, i18n:check, **full `npm run test` once** (dependency change) → commit `feat(editor): detect corner markers and print a marker sheet`. + +--- + +### Task 10: End to end in the app, checklist + +**Files:** +- Create: `scripts/editor-multicam-smoke.mjs` (Playwright `_electron`, dev tool; header documents usage). +- Modify: `technical-documentation/testing/manual-e2e-checklist.md` — "Several cameras in the editor" item + results row. + +- [ ] **Step 1:** Build (`npm run build-vite`), addon from this tree (`npm run build:native:compositor`, copy to `electron/native/bin/win32-arm64/` next to the ARM64 ffmpeg DLLs), launch with `_electron.launch({ args: [] })` (the repo root, **not** `dist-electron/main.js` — otherwise userData becomes `Roaming\Electron`), `OPENSCREEN_COMPOSITOR_VIEW_NODE` set and that dir first on PATH. Use a **copy** of the user's two-camera project in a temp user-data dir *only if* its media resolves there; otherwise use the real user data with the projects folder backed up first and restored byte-identical afterwards (verify by hash) — record which. +- [ ] **Step 2:** In the editor (via Playwright): add a `camera-full-pip` section through the layout menu, choose cameras in the inspector, convert a Full Camera section (key C) to `screen-pip` and back, undo/redo, drag a PiP place in the preview, open the camera calibration dialog and apply a perspective, save, reopen the project. Assert document state after each step (read via `window` store hooks or the saved file) and capture preview screenshots inside the sections (the preview now renders layout sections live — this is the first in-app run of that path). +- [ ] **Step 3:** Checklist item (real cameras: desk camera full + face PiP, screen + 2 PiP, side by side, perspective with the printed marker sheet and an A4 sheet, export matches preview) and a results row (Partial: real-camera run by the user outstanding). Commit `test(editor): drive the multi-camera editor end to end`. diff --git a/docs/superpowers/plans/2026-10-06-desk-lane.md b/docs/superpowers/plans/2026-10-06-desk-lane.md new file mode 100644 index 0000000000..f13b0f25e1 --- /dev/null +++ b/docs/superpowers/plans/2026-10-06-desk-lane.md @@ -0,0 +1,386 @@ +# Tischkamera-Spur (D) — Implementation Plan (Teil B) + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Eine Kamera wird im Projekt als Tischkamera festgelegt; die Taste D legt einen Abschnitt auf einer eigenen Timeline-Zeile an, in dem diese Kamera weich ins Vollbild überblendet — ohne Änderung am Compositor. + +**Architecture:** D-Abschnitte sind eine eigene, schlanke, clip-verankerte Liste `legacyEditor.deskRegions` (wie `cameraFullscreenRegions`), dazu die Projekteinstellung `legacyEditor.deskCamera`. Erst `buildSceneDescription` übersetzt sie in Zeilen, die der Compositor schon kennt: `camera-full`-Layout der Tischkamera (Kamera ≥ 2) bzw. eine Full-Camera-Zeile (Kamera 1). Die Timeline behandelt C-, Layout- und D-Abschnitte als drei Listen, die sich zeitlich gegenseitig ausschließen. + +**Tech Stack:** TypeScript (strict), React 19, Vitest (node + jsdom), Biome, i18next (15 Locales), Playwright `_electron` für den App-Lauf. + +**Spec:** `docs/superpowers/specs/2026-10-06-desk-lane-and-free-crop-design.md` (Teil B) + +## Global Constraints + +- Arbeitsverzeichnis `C:\osc-cams`, Branch `feat/multi-camera-layouts`; nichts pushen. +- Keine Änderung an Rust/Shadern/Compositor; keine Schema-Version (`deskRegions`/`deskCamera` optional in `legacyEditor`). +- Alte Projekte ohne `deskRegions`/`deskCamera`: Szene byte-gleich wie vorher; `legacyEditor` bleibt unverändert, solange nie ein D-Abschnitt angelegt wurde. +- D-Abschnitte schließen sich mit C- und Layout-Abschnitten aus (keine Überlappung, Berühren erlaubt) — beim Anlegen, Einfügen, Ziehen und Längenändern, in beide Richtungen. +- Übergang: Überblenden mit den Zeiten von C (kommt aus dem Compositor-Layoutpfad, nichts Eigenes bauen). +- Drehung, Spiegelung, Zuschnitt, Perspektive der Tischkamera kommen aus `cameraSettings` dieser Kamera, nicht aus dem Abschnitt. +- Code-Kommentare Englisch; Biome; kein `any`; jeder UI-Text in allen 15 Locales; `npm run i18n:check` grün. +- Keine Claude-Signatur und kein `Co-Authored-By` in Commits. +- Prüfen pro Task: betroffene Testdateien mit `npx vitest --run `, `npx tsc --noEmit`, `npx tsc -p tsconfig.test.json --noEmit`; volle Suite (`npm run test`) einmal im letzten Code-Task. + +## Review Focus + +- **Tischkamera im Projekt nicht (mehr) vorhanden** (Datei verloren, Index zu groß): D-Abschnitte werden in der Szene übersprungen, nicht gelöscht; Inspector sagt „nicht verfügbar“ — Tests in Task 1 und Task 3. +- **D-Abschnitt direkt an C-Abschnitt** (Berühren): erlaubt beim Anlegen und Ziehen; Szene enthält beide — Tests in Task 2 und Task 3. +- **Clip-Bearbeitung (Schneiden, Verschieben, Löschen eines Clips) mit D-Abschnitten**: werden mitverankert wie Full-Camera-Abschnitte — Test in Task 2. +- **Einfügen (Strg+V) eines kopierten C- oder Layout-Abschnitts über einen D-Abschnitt** und umgekehrt: abgelehnt mit „occupied“ — Test in Task 2. +- **Tischkamera = Kamera 1**: D wird zur Full-Camera-Zeile ohne Drehungs-Override; kein doppeltes Label — Test in Task 3. + +--- + +### Task 1: Kern — Typen, Lesen, Tischkamera auflösen, Übersetzung + +**Files:** +- Modify: `src/components/video-editor/types.ts` (Typ `DeskRegion`) +- Create: `src/lib/deskSections.ts` +- Create: `src/lib/deskSections.test.ts` + +**Interfaces:** +- Produces: + - `export interface DeskRegion { id: string; startMs: number; endMs: number; deskLabel?: false }` (types.ts, neben `CameraFullscreenRegion`) + - `export type AnchoredDeskRegion = DeskRegion & { clipId?: string; assetId?: string; sourceStartSec?: number; sourceEndSec?: number }` + - `export function normalizeDeskRegions(raw: unknown): AnchoredDeskRegion[]` — ungültige verwerfen, nach `startMs` sortieren, spätere Überlappung innerhalb der Liste verwerfen (wie `normalizeCameraLayoutRegions`). + - `export function resolveDeskCamera(input: { deskCamera: unknown; cameraCount: number; cameraSettings: CameraSettings[] }): number | null` — Index (0 = Kamera 1). + - `export function deskRowsForScene(rows: AnchoredDeskRegion[], deskCamera: number | null): { layout: AnchoredCameraLayoutRegion[]; full: Array }` (`AnchoredRow`-Felder: `clipId`, `assetId`, `sourceStartSec`, `sourceEndSec` — als lokaler Typ in diesem Modul). + +- [ ] **Step 1: Failing tests** — `src/lib/deskSections.test.ts`: + +```ts +import { describe, expect, it } from "vitest"; +import { deskRowsForScene, normalizeDeskRegions, resolveDeskCamera } from "./deskSections"; + +describe("normalizeDeskRegions", () => { + it("keeps valid rows with their anchor, sorted, and drops broken or overlapping ones", () => { + const rows = normalizeDeskRegions([ + { id: "b", startMs: 5000, endMs: 7000, clipId: "c1", sourceStartSec: 5, sourceEndSec: 7 }, + { id: "a", startMs: 1000, endMs: 3000, deskLabel: false }, + { id: "x", startMs: 6000, endMs: 8000 }, + { id: "bad", startMs: 4000, endMs: 4000 }, + { startMs: 1, endMs: 2 }, + "nope", + ]); + expect(rows.map((r) => r.id)).toEqual(["a", "b"]); + expect(rows[0].deskLabel).toBe(false); + expect(rows[1]).toMatchObject({ clipId: "c1", sourceStartSec: 5, sourceEndSec: 7 }); + }); + + it("reads nothing from a missing list", () => { + expect(normalizeDeskRegions(undefined)).toEqual([]); + }); +}); + +describe("resolveDeskCamera", () => { + const none = [{}, {}, {}]; + + it("takes the chosen camera when the project has it", () => { + expect(resolveDeskCamera({ deskCamera: 2, cameraCount: 3, cameraSettings: none })).toBe(2); + expect(resolveDeskCamera({ deskCamera: 0, cameraCount: 3, cameraSettings: none })).toBe(0); + }); + + it("falls back to the first camera after camera 1 with a perspective", () => { + const withPerspective = [ + {}, + {}, + { perspective: { corners: [{ x: 0, y: 0 }, { x: 1, y: 0 }, { x: 1, y: 1 }, { x: 0, y: 1 }], aspect: 1 } }, + ] as never; + expect(resolveDeskCamera({ deskCamera: undefined, cameraCount: 3, cameraSettings: withPerspective })).toBe(2); + // A chosen camera the project no longer has falls back the same way. + expect(resolveDeskCamera({ deskCamera: 7, cameraCount: 3, cameraSettings: withPerspective })).toBe(2); + }); + + it("falls back to camera 2, and to nothing with a single camera", () => { + expect(resolveDeskCamera({ deskCamera: undefined, cameraCount: 2, cameraSettings: [{}, {}] })).toBe(1); + expect(resolveDeskCamera({ deskCamera: undefined, cameraCount: 1, cameraSettings: [{}] })).toBeNull(); + expect(resolveDeskCamera({ deskCamera: "2", cameraCount: 1, cameraSettings: [{}] })).toBeNull(); + }); +}); + +describe("deskRowsForScene", () => { + const rows = normalizeDeskRegions([ + { id: "d1", startMs: 1000, endMs: 3000, clipId: "c1", assetId: "a1", sourceStartSec: 1, sourceEndSec: 3 }, + ]); + + it("turns a section of camera k >= 2 into a camera-full layout row with the same anchor", () => { + const out = deskRowsForScene(rows, 2); + expect(out.full).toEqual([]); + expect(out.layout).toEqual([ + { + id: "d1", + startMs: 1000, + endMs: 3000, + clipId: "c1", + assetId: "a1", + sourceStartSec: 1, + sourceEndSec: 3, + template: "camera-full", + slots: [{ camera: 2 }], + }, + ]); + }); + + it("turns a section of camera 1 into a Full Camera row without an orientation override", () => { + const out = deskRowsForScene(rows, 0); + expect(out.layout).toEqual([]); + expect(out.full).toEqual([ + { id: "d1", startMs: 1000, endMs: 3000, clipId: "c1", assetId: "a1", sourceStartSec: 1, sourceEndSec: 3 }, + ]); + }); + + it("draws nothing without a desk camera", () => { + expect(deskRowsForScene(rows, null)).toEqual({ layout: [], full: [] }); + }); +}); +``` + +- [ ] **Step 2:** `npx vitest --run src/lib/deskSections.test.ts` → FAIL (Modul fehlt). + +- [ ] **Step 3: Implementieren.** `DeskRegion` in `types.ts` mit Doc-Kommentar („A desk section: the project's desk camera fills the frame. Only `false` is stored for `deskLabel`.“). `src/lib/deskSections.ts`: + +```ts +// Desk sections ("D"): spans in which the project's desk camera fills the frame. They are +// stored as their own clip-anchored list and become rows the compositor already draws only +// when the scene is built: a camera-full layout of the desk camera, or a Full Camera row when +// the desk camera is camera 1. + +import type { + CameraFullscreenRegion, + CameraSettings, + DeskRegion, +} from "@/components/video-editor/types"; +import type { AnchoredCameraLayoutRegion } from "./cameraLayouts"; + +interface Anchor { + clipId?: string; + assetId?: string; + sourceStartSec?: number; + sourceEndSec?: number; +} + +export type AnchoredDeskRegion = DeskRegion & Anchor; + +function isRecord(v: unknown): v is Record { + return typeof v === "object" && v !== null; +} + +function anchorOf(raw: Record): Anchor { + const out: Anchor = {}; + if (typeof raw.clipId === "string") out.clipId = raw.clipId; + if (typeof raw.assetId === "string") out.assetId = raw.assetId; + if (typeof raw.sourceStartSec === "number") out.sourceStartSec = raw.sourceStartSec; + if (typeof raw.sourceEndSec === "number") out.sourceEndSec = raw.sourceEndSec; + return out; +} + +/** Stored desk sections, read defensively: broken rows and later overlapping rows are dropped. */ +export function normalizeDeskRegions(raw: unknown): AnchoredDeskRegion[] { + if (!Array.isArray(raw)) return []; + const valid: AnchoredDeskRegion[] = []; + for (const r of raw) { + if (!isRecord(r) || typeof r.id !== "string") continue; + const { startMs, endMs } = r; + if (typeof startMs !== "number" || typeof endMs !== "number") continue; + if (!Number.isFinite(startMs) || !Number.isFinite(endMs) || startMs >= endMs) continue; + valid.push({ + id: r.id, + startMs, + endMs, + ...anchorOf(r), + ...(r.deskLabel === false ? { deskLabel: false as const } : {}), + }); + } + valid.sort((a, b) => a.startMs - b.startMs); + const kept: AnchoredDeskRegion[] = []; + for (const row of valid) { + const last = kept[kept.length - 1]; + if (last && row.startMs < last.endMs) continue; + kept.push(row); + } + return kept; +} + +/** + * The desk camera (0 = camera 1): the chosen one while the project has it; else the first + * camera after camera 1 with a perspective; else camera 2; else none. + */ +export function resolveDeskCamera(input: { + deskCamera: unknown; + cameraCount: number; + cameraSettings: CameraSettings[]; +}): number | null { + const { deskCamera, cameraCount, cameraSettings } = input; + if ( + typeof deskCamera === "number" && + Number.isInteger(deskCamera) && + deskCamera >= 0 && + deskCamera < cameraCount + ) { + return deskCamera; + } + for (let i = 1; i < cameraCount; i++) { + if (cameraSettings[i]?.perspective) return i; + } + return cameraCount >= 2 ? 1 : null; +} + +/** The scene's view of the desk sections: rows of the two lists the compositor draws. */ +export function deskRowsForScene( + rows: AnchoredDeskRegion[], + deskCamera: number | null, +): { + layout: AnchoredCameraLayoutRegion[]; + full: Array; +} { + if (deskCamera === null) return { layout: [], full: [] }; + const span = ({ deskLabel: _label, ...row }: AnchoredDeskRegion) => row; + if (deskCamera === 0) return { layout: [], full: rows.map(span) }; + return { + layout: rows.map((row) => ({ + ...span(row), + template: "camera-full" as const, + slots: [{ camera: deskCamera }], + })), + full: [], + }; +} +``` + +- [ ] **Step 4:** Test grün; `npx tsc --noEmit`. +- [ ] **Step 5:** Commit `feat(camera): desk sections and the desk camera, read and resolved`. + +--- + +### Task 2: Dokument und Store — Liste, Ausschluss, Verankerung, Kopieren + +**Files:** +- Modify: `src/lib/ai-edition/document/timeline.ts` (`RegionKind`, Neuverankerung, `anchoredRegionsOf`, Einzel-Löschen) +- Modify: `src/lib/ai-edition/store/regionClipboardKinds.ts`, `src/lib/ai-edition/store/regionClipboard.ts` +- Modify: `src/lib/ai-edition/store/useTimeline.ts` +- Modify: `src/lib/ai-edition/timeline/cameraSectionNotice.ts`, `src/i18n/locales/*/timeline.json` (15) +- Modify: `electron/ai-edition/agent-tools.ts` (Belegt-Prüfung des Agenten) +- Tests: die vorhandenen Testdateien dieser Module (`*.test.ts` daneben), dort ergänzen + +**Interfaces:** +- Consumes: `normalizeDeskRegions`, `resolveDeskCamera`, `AnchoredDeskRegion` (Task 1). +- Produces (auf dem Rückgabeobjekt von `useTimeline`, neben den Full-Camera-Gegenstücken): + - `deskRegions: AnchoredDeskRegion[]` + - `deskCamera: number | null` (aufgelöst) und `deskCameraChosen: number | null` (gespeichert) + - `addDeskSection(durationSec?: number): Promise` + - `updateDeskSpan(id: string, startMs: number, endMs: number): Promise` + - `updateDeskLabel(id: string, show: boolean): Promise` + - `setDeskCamera(index: number | null): Promise` + - `RegionKind`/Selection-Kind `"desk"`; `removeRegions` versteht `"desk"`. + - `CameraSectionOutcome` bekommt `"no-desk-camera"` → Notice `errors.noDeskCamera` („Keine Tischkamera — lege eine im Abschnitt Kameras fest“ / „No desk camera — choose one in the Cameras section“). + +**Verhalten:** +- Speicherort `legacyEditor.deskRegions` (Id-Präfix `"desk"`), clip-verankert wie `cameraFullscreenRegions` (`anchorRegionsWithDerivedMs`, `replacePillSpan`, `coalesceRegionsForRuler`, `dropPillsByIds`). Eine nie angelegte Liste bleibt im Dokument abwesend. +- `addDeskSection`: ohne Dokument oder ohne aufgelöste Tischkamera → `"no-desk-camera"`; überlappt die Spanne einen Abschnitt einer der drei Listen → `"occupied"`; sonst anlegen, ein Undo-Schritt. +- `addCameraFullscreen` lehnt zusätzlich bei Überlappung mit `deskRegions` ab; `addCameraLayout` ebenso. (Full Camera über Full Camera bleibt erlaubt wie heute.) +- Ziehen/Längenändern: `clampCameraSpan` bekommt als „andere“ Pillen die Zeilen *beider* übrigen Listen; das gilt für Full Camera, Layout und Desk. +- `setDeskCamera(null)` entfernt `deskCamera` aus `legacyEditor`; ein Undo-Schritt. +- `timeline.ts`: `RegionKind` um `"desk"` erweitern; die TypeScript-Fehler an den erschöpfenden `switch`-Stellen zeigen jede Stelle, die einen Fall braucht — jeweils wie `"cameraFullscreen"` mit `legacyEditor.deskRegions`. Die Neuverankerung bei Clip-Bearbeitung (Funktion um Zeile 372–430) und `anchoredRegionsOf` (um Zeile 735) nehmen `deskRegions` mit (Präfix `"desk"`). +- Zwischenablage: `"desk"` wie `"cameraFullscreen"` (Store `legacy`, Key `deskRegions`, Präfix `desk`); das Einfügen prüft den Ausschluss über alle drei Listen (die bestehende Prüfung in `NewEditorShell.tsx` um Zeile 1160–1175 benutzt dafür eine Hilfsfunktion — sie muss `deskRegions` einschließen, in beide Richtungen). +- Agent (`electron/ai-edition/agent-tools.ts`): wo er Kamera-Abschnitte auf Belegung prüft (um Zeile 279 und 386–393), zählen `deskRegions` als belegt; Beschreibungstexte, die die Listen aufzählen, nennen `deskRegions`. + +- [ ] **Step 1: Failing tests** in den vorhandenen Testdateien (Store-Tests nach dem Muster der bestehenden `addCameraFullscreen`/`addCameraLayout`-Tests; Dokument-Tests nach dem Muster der bestehenden `cameraFullscreenRegions`-Tests in `timeline.test.ts`): + 1. `addDeskSection` legt an der Abspielposition an (Standarddauer), ein Undo-Schritt, `deskRegions` erscheint erst dann im Dokument. + 2. `addDeskSection` → `"occupied"` über einem Full-Camera- und über einem Layout-Abschnitt; direkt anschließend (Berühren) → `"added"`. + 3. `addCameraFullscreen` und `addCameraLayout` → `"occupied"` über einem D-Abschnitt. + 4. `addDeskSection` ohne Tischkamera (Projekt mit einer Kamera) → `"no-desk-camera"`, Dokument unverändert. + 5. `updateDeskSpan` wird an einem Full-Camera-Nachbarn begrenzt; `updateCameraFullscreenSpan` an einem D-Nachbarn. + 6. `removeRegions` mit `{kind: "desk"}` entfernt den Abschnitt in einem Undo-Schritt. + 7. `setDeskCamera(2)` / `setDeskCamera(null)` schreiben bzw. entfernen `legacyEditor.deskCamera`; `deskCamera` (aufgelöst) folgt. + 8. `timeline.ts`: Clip schneiden/verschieben verankert einen D-Abschnitt mit wie einen Full-Camera-Abschnitt (gleicher Testaufbau, Liste `deskRegions`). + 9. Zwischenablage: einen kopierten Full-Camera-Abschnitt über einem D-Abschnitt einfügen → abgelehnt; einen kopierten D-Abschnitt an freier Stelle einfügen → angelegt. + 10. `cameraSectionNotice("no-desk-camera", t)` → `{ title: "errors.noDeskCamera" }`. +- [ ] **Step 2:** Tests laufen lassen → FAIL. +- [ ] **Step 3:** Implementieren wie unter „Verhalten“; Texte `errors.noDeskCamera` in allen 15 `timeline.json`. +- [ ] **Step 4:** Tests grün; `npx tsc --noEmit`, `npx tsc -p tsconfig.test.json --noEmit`, `npm run i18n:check`. +- [ ] **Step 5:** Commit `feat(camera): desk sections in the document, exclusive with camera sections`. + +--- + +### Task 3: Szene — D wird zu Ebenen, die der Compositor kennt + +**Files:** +- Modify: `src/native/sceneDescription.ts` (um Zeile 1273–1325 und `deskLabelTextRegions` um Zeile 970) +- Test: `src/native/sceneDescription.test.ts` + +**Interfaces:** +- Consumes: `normalizeDeskRegions`, `resolveDeskCamera`, `deskRowsForScene` (Task 1); `normalizeCameraSettings` (vorhanden); die Kamerazahl des Projekts (dieselbe Quelle, die die Szene für `cameras[]` benutzt). + +**Verhalten:** +- Vor der Projektion: `const desk = deskRowsForScene(normalizeDeskRegions(legacyRaw?.deskRegions), resolveDeskCamera({...}))`. `desk.layout` kommt zu `layoutRegions` (nach Start sortiert), `desk.full` zu den Full-Camera-Zeilen (`[...fullscreenRegions, ...handWrittenFullCamera, ...desk.full]`, jeweils mit `sectionId: region.id`). +- Label: für jeden **projizierten** Teil eines D-Abschnitts mit `deskLabel !== false` eine Textregion wie in `deskLabelTextRegions` (gleicher Text, Stil, Position, Id-Schema `desk--`), aber mit `textAnimation: "fade"` statt `DESK_COVER_ANIMATION` — ein D-Abschnitt hat keine Kipp-Abdeckung, deren Stärke das Label sonst folgt. Dafür `deskLabelTextRegions` um einen Parameter für die Animation erweitern oder eine zweite kleine Funktion danebenstellen, die dieselbe Region baut; keine Kopie des ganzen Objekts. Die D-Teile werden über ihre Projektion (Layout- bzw. Full-Liste) per `sectionId` wiedergefunden. +- Ist die Tischkamera Kamera 1, entsteht die Full-Camera-Zeile **ohne** `rotation` → `deskLabelTextRegions` (nur gedrehte Abschnitte) erzeugt kein zweites Label; das D-Label kommt allein aus der neuen Funktion. +- Ohne `deskRegions` ist die Szene unverändert (gleiches Objekt, keine leeren neuen Felder). + +- [ ] **Step 1: Failing tests** in `sceneDescription.test.ts` (Dokument-Fixtures wie in den vorhandenen Layout-Tests der Datei): + 1. Zwei Kameras, `deskRegions: [{ id: "d1", startMs: 1000, endMs: 3000 }]`, keine Festlegung → `cameraLayoutRegions` enthält einen `camera-full`-Eintrag 1–3 s für Kamera 2 (Slot `camera: 1`); ein Text-Annotation `desk-d1-…` mit `textAnimation: "fade"`. + 2. `deskCamera: 0` → kein Layout-Eintrag, eine `cameraFullscreenRegions`-Zeile 1–3 s ohne `rotation`; genau **ein** Label. + 3. `deskLabel: false` → kein Label. + 4. Eine Kamera im Projekt → D-Abschnitt erzeugt nichts; Szene gleich der ohne `deskRegions`. + 5. D-Abschnitt 3–5 s direkt nach Full-Camera-Abschnitt 1–3 s → beide in der Szene. + 6. Dokument ohne `deskRegions`/`deskCamera` → `toEqual` mit der bisherigen Szene (bestehende Fixture). +- [ ] **Step 2:** FAIL prüfen. +- [ ] **Step 3:** Implementieren. +- [ ] **Step 4:** `npx vitest --run src/native/sceneDescription.test.ts src/lib/deskSections.test.ts`; tsc beide. +- [ ] **Step 5:** Commit `feat(camera): the scene draws desk sections through the layout path`. + +--- + +### Task 4: Bedienung — Taste D, Timeline-Zeile, Inspector, Tischkamera wählen + +**Files:** +- Modify: `src/lib/shortcuts.ts` (+ Test daneben), `src/i18n/locales/*/shortcuts.json` +- Modify: `src/components/ai-edition/NewEditorShell.tsx` (Tastenbehandlung neben `addCameraFullscreen` um Zeile 1411; Inspector-Props) +- Modify: `src/components/ai-edition/v4/V4Timeline.tsx` (Pillen um Zeile 797–831, Zieh-Verteilung um Zeile 1052–1064, Werkzeugknopf um Zeile 2089–2107, neue Zeile nach der Kamera-Zeile um Zeile 2296–2308, `LanePill.kind`) +- Modify: `src/components/ai-edition/v4/FloatingInspector.tsx` (Auswahl `"desk"`, neben dem `cameraFullscreen`-Zweig um Zeile 1223) +- Modify: `src/components/ai-edition/CamerasSection.tsx` (Auswahl „Tischkamera“ pro Kamera) +- Modify: `src/i18n/locales/*/timeline.json`, `*/settings.json` (15 je) +- Tests: vorhandene Komponententests dieser Dateien (jsdom) ergänzen; `shortcuts.test.ts` + +**Interfaces:** +- Consumes: alles aus Task 2 auf `tl`. + +**Verhalten:** +- Aktion `addDeskSection` in `SHORTCUT_ACTIONS`, Standard `{ key: "d" }`, Label „Add Desk Section“; kollidiert nicht mit Strg+D (`findConflict` prüft Modifier). Taste → `tl.addDeskSection(newRegionDurationSec())` → `showCameraSectionOutcome(outcome, tt)`. +- Timeline: eigene Zeile direkt unter der Kamera-Zeile, Pillen `kind: "desk"`, Beschriftung „Tisch“ / „Desk“ (+ Kameraname der aufgelösten Tischkamera, z. B. „Tisch · Logitech BRIO“), Symbol wie Full Camera mit Tisch-Andeutung (vorhandenes Icon-Set, z. B. `Monitor`/`LayoutPanelTop` aus lucide — vorhandenes Muster `pillIcon` erweitern). Leere Zeile zeigt „Drücke D“ (`hints.pressDesk`, mit `formatBinding`) — ohne Tischkamera stattdessen `errors.noDeskCamera`-Kurztext. Ziehen/Längenändern → `tl.updateDeskSpan`. Die Zeile ist nur sichtbar, wenn das Projekt mindestens zwei Kameras hat **oder** bereits D-Abschnitte enthält. +- Werkzeugleiste: Knopf „Tischabschnitt hinzufügen“ neben Full Camera, nur mit ≥ 2 Kameras. +- Inspector für `"desk"`: Titel „Tisch“; Zeile „Tischkamera: “ (bzw. „nicht verfügbar“, wenn keine aufgelöst) mit Knopf „Ändern“, der den Inspector auf die Facette `layout` (Abschnitt „Kameras“) schaltet; Schalter „Label anzeigen“ → `tl.updateDeskLabel`; Löschen. +- Abschnitt „Kameras“: pro Kamera ein Schalter/Radiopunkt „Tischkamera“ (`aria-pressed`/`role="radio"` nach Muster der Datei); genau eine; die aufgelöste Kamera ist markiert, die *gespeicherte* Festlegung zusätzlich erkennbar (Hinweis „automatisch“, wenn nichts gespeichert ist); erneutes Klicken auf die gespeicherte hebt die Festlegung auf → `tl.setDeskCamera(null)`. +- Texte (alle 15 Locales): `timeline.labels.desk`, `timeline.hints.pressDesk`, `timeline.toolbar.addDesk` (Schlüsselnamen nach dem Muster der vorhandenen Full-Camera-Schlüssel), `settings.cameras.deskCamera`, `settings.cameras.deskCameraAuto`, `settings.desk.title`, `settings.desk.camera`, `settings.desk.change`, `settings.desk.showLabel`, `settings.desk.unavailable`, `shortcuts`-Label. + +- [ ] **Step 1: Failing tests** (jsdom, nach Muster der vorhandenen Tests dieser Komponenten): + 1. `shortcuts.test.ts`: `DEFAULT_SHORTCUTS.addDeskSection` ist `d`; `findConflict` meldet keinen Konflikt mit `deleteSelected` (Strg+D). + 2. Timeline: Projekt mit zwei Kameras und einem D-Abschnitt → Pille mit Beschriftung `labels.desk`; mit einer Kamera und ohne D-Abschnitt → keine Tisch-Zeile. + 3. Inspector: Auswahl `{kind: "desk"}` zeigt Tischkamera-Namen, Label-Schalter ruft `updateDeskLabel(id, false)`. + 4. Kameras: Klick auf „Tischkamera“ bei Kamera 2 ruft `setDeskCamera(1)`; Klick auf die gespeicherte ruft `setDeskCamera(null)`. +- [ ] **Step 2:** FAIL prüfen. +- [ ] **Step 3:** Implementieren. +- [ ] **Step 4:** Fokussierte Tests; tsc beide; `npm run lint`; `npm run i18n:check`; **einmal `npm run test`**. +- [ ] **Step 5:** Commit `feat(camera): the D key and a desk lane in the editor`. + +--- + +### Task 5: Lauf in der echten App mit Playwright (Controller) + +**Files:** +- Create: `scripts/editor-desk-smoke.mjs` (nach dem Muster von `scripts/editor-multicam-smoke.mjs`) +- Modify: `technical-documentation/testing/manual-e2e-checklist.md` (Results-Log-Zeile) + +- [ ] **Step 1:** `npm run build-vite` in `C:\osc-cams`; Compositor-Addon wie im vorhandenen Smoke-Skript über `OPENSCREEN_COMPOSITOR_VIEW_NODE`. +- [ ] **Step 2:** Skript: Projektordner des Nutzers sichern (Hash), mit einer **Kopie** des Zwei-Kamera-Projekts (`recording-1791290781616`, Kamera 2 = Brio am Tisch) arbeiten; App per `_electron.launch` mit dem App-Ordner als Argument starten. Prüfen, jeweils mit Screenshot: + 1. Abschnitt „Kameras“: Kamera 2 als Tischkamera wählen; gespeicherte Datei enthält `deskCamera: 1`. + 2. Abspielkopf auf freie Stelle, Taste **D** → Pille auf der Tisch-Zeile; Datei enthält einen `deskRegions`-Eintrag. + 3. Vorschau in der Mitte des Abschnitts: Kamera 2 füllt das Bild (Pixelvergleich gegen ein Bild außerhalb: Bildschirminhalt verschwunden); Label „Schreibtischmodus“ sichtbar. + 4. **C** direkt vor dem D-Abschnitt anlegen, Abschnitte berühren sich; **C** über dem D-Abschnitt → Hinweis „belegt“, nichts geschrieben. + 5. Inspector: Label aus → kein Label in der Vorschau. + 6. Strg+Z / Strg+Umschalt+Z; speichern, App neu starten, Datei und Vorschau gleich. + 7. Export MP4 1080p eines Ausschnitts über den D-Abschnitt; ein Bild aus der Mitte mit ffmpeg ziehen und gegen den Vorschau-Screenshot vergleichen (Kamera 2 füllt das Bild). + Danach Projektordner gegen die Sicherung prüfen (byte-gleich). +- [ ] **Step 3:** Results-Log-Zeile (Datum, `feat/multi-camera-layouts @ `, Windows 11 ARM64, „Partial — scripted desk-lane pass“, Ergebnisse, „Not covered“: echte Kamera live, HUD, andere Plattformen). Commit `test: log the desk-lane pass`. +- [ ] **Step 4:** Dem Nutzer die Screenshots zeigen; offen bleibt nur, was Playwright nicht erreicht. diff --git a/docs/superpowers/plans/2026-10-06-plane-crop-calibration.md b/docs/superpowers/plans/2026-10-06-plane-crop-calibration.md new file mode 100644 index 0000000000..9ed250c955 --- /dev/null +++ b/docs/superpowers/plans/2026-10-06-plane-crop-calibration.md @@ -0,0 +1,1206 @@ +# Kalibrierdialog: Entzerrung, dann freier Ausschnitt — Implementation Plan (Teil A) + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** „Marker erkennen“ bestimmt nur noch die Tischebene; der Nutzer zieht danach einen Ausschnitt (4:3, 16:9, 9:16, 1:1, frei) auf das entzerrte Bild, und gespeichert werden die Bildecken eines auf dem Tisch wirklich rechtwinkligen Rechtecks. + +**Architecture:** Die Marker-Quadrate liefern eine metrische Entzerrung (zirkuläre Punkte, schon in `planeMeasure.ts`). Daraus wird ein *Basisrechteck* auf der Ebene gebaut — selbst eine gewöhnliche `CameraPerspective` (vier Bildecken + echtes Seitenverhältnis). Alles Weitere ist Geometrie in Bruchteilen einer vergrößerten *Ansicht* dieses Rechtecks (`planeCrop.ts`): der Ausschnitt ist eine `CropRegion` in der Ansicht und wird beim Übernehmen zurück in vier Bildecken gerechnet. Datenmodell, Compositor und Export bleiben unverändert. + +**Tech Stack:** TypeScript (strict), React 19, Vitest (node + jsdom), Biome, i18next-Locales (15 Sprachen). + +**Spec:** `docs/superpowers/specs/2026-10-06-desk-lane-and-free-crop-design.md` (Teil A) + +## Global Constraints + +- Arbeitsverzeichnis `C:\osc-cams`, Branch `feat/multi-camera-layouts`; nichts pushen. +- `CameraPerspective { corners, aspect, margin? }` bleibt unverändert; alte Projekte laden wie bisher. +- Keine Änderung an Rust/Shadern/Compositor. +- Code-Kommentare auf Englisch; Biome (Tabs, doppelte Anführungszeichen, 100 Spalten); kein `any`. +- Jeder neue UI-Text in allen 15 Locale-Dateien `src/i18n/locales/*/dialogs.json`; `npm run i18n:check` grün. +- Keine Claude-Signatur und kein `Co-Authored-By` in Commit-Nachrichten. +- Prüfen pro Task: die betroffenen Testdateien mit `npx vitest --run `; vor dem letzten Commit einmal `npm run test`, `npx tsc --noEmit`, `npx tsc -p tsconfig.test.json --noEmit`, `npm run lint`. + +## Review Focus + +- **Frei verteilte Marker, die kein Rechteck bilden** (der echte Fall des Nutzers): das gespeicherte Viereck muss auf der Ebene rechtwinklig sein — Test in Task 1 (Winkel 90° ± 0,5°). +- **Ansicht über den Horizont hinaus** (steile Kamera, große Vergrößerung): `planeView` darf keine Ecke hinter der Kamera erzeugen — Test in Task 3. +- **Gespeicherte Perspektive erneut öffnen und unverändert übernehmen**: Ecken und Seitenverhältnis bleiben gleich (Rundreise) — Test in Task 4. +- **Alte Perspektive mit Rand oder gekreuztem Viereck**: öffnet im Vier-Griffe-Modus wie bisher — bestehende Tests in Task 4 bleiben grün. +- **Ausschnitt größer als das Kamerabild / Seitenverhältnis außerhalb 0,1…10**: Hinweis bzw. „Übernehmen“ gesperrt — Tests in Task 4. + +--- + +### Task 1: Basisrechteck auf der Ebene (`markerPlaneRect`) + +**Files:** +- Modify: `src/lib/planeMeasure.ts` +- Create: `src/lib/planeMeasure.test.ts` + +**Interfaces:** +- Produces: + - `export interface PlaneRect { corners: Corners; aspect: number; widthMm: number }` + - `export function markerPlaneRect(squares: Corners[], sideMm: number, inner: Corners): PlaneRect | null` — alle Punkte in *einem* Bildkoordinatensystem (Pixel); `corners` im selben System, Reihenfolge TL, TR, BR, BL; das Rechteck ist die achsenparallele Hülle der vier inneren Ecken, ausgerichtet an inner[0] → inner[1]. + - `export function invert3(m: number[]): number[] | null` (bisher privat) + - `measureOnPlane` und `PlaneSize` bleiben vorerst (Task 4 entfernt sie). + +- [ ] **Step 1: Failing test schreiben** — `src/lib/planeMeasure.test.ts`: + +```ts +import { describe, expect, it } from "vitest"; +import type { CameraPerspective } from "@/components/video-editor/types"; +import { markerPlaneRect } from "./planeMeasure"; + +type Corners = CameraPerspective["corners"]; +interface Pt { + x: number; + y: number; +} + +// A camera looking at the desk at a slant: world millimetres -> image pixels. +const H = [1.1, 0.25, 420, 0.04, 0.8, 180, 0.0003, 0.0008, 1]; + +function project(m: number[], p: Pt): Pt { + const w = m[6] * p.x + m[7] * p.y + m[8]; + return { x: (m[0] * p.x + m[1] * p.y + m[2]) / w, y: (m[3] * p.x + m[4] * p.y + m[5]) / w }; +} + +function inverse(m: number[]): number[] { + const [a, b, c, d, e, f, g, h, i] = m; + const A = e * i - f * h; + const B = f * g - d * i; + const C = d * h - e * g; + const det = a * A + b * B + c * C; + return [ + A / det, + (c * h - b * i) / det, + (b * f - c * e) / det, + B / det, + (a * i - c * g) / det, + (c * d - a * f) / det, + C / det, + (b * g - a * h) / det, + (a * e - b * d) / det, + ]; +} + +/** A 40 mm square centred on `c`, turned by `deg`, corners clockwise from its top-left. */ +function square(c: Pt, deg: number): Pt[] { + const r = (deg * Math.PI) / 180; + return [ + { x: -20, y: -20 }, + { x: 20, y: -20 }, + { x: 20, y: 20 }, + { x: -20, y: 20 }, + ].map((p) => ({ + x: c.x + p.x * Math.cos(r) - p.y * Math.sin(r), + y: c.y + p.x * Math.sin(r) + p.y * Math.cos(r), + })); +} + +/** The square's corner nearest to `toward`. */ +function nearest(sq: Pt[], toward: Pt): Pt { + return sq.reduce((best, p) => + Math.hypot(p.x - toward.x, p.y - toward.y) < Math.hypot(best.x - toward.x, best.y - toward.y) + ? p + : best, + ); +} + +function angleAt(a: Pt, b: Pt, c: Pt): number { + const u = { x: a.x - b.x, y: a.y - b.y }; + const v = { x: c.x - b.x, y: c.y - b.y }; + return ( + (Math.acos((u.x * v.x + u.y * v.y) / (Math.hypot(u.x, u.y) * Math.hypot(v.x, v.y))) * 180) / + Math.PI + ); +} + +// Markers laid freely: no rectangle, each turned its own way (the user's real desk). +const CENTERS: Pt[] = [ + { x: 0, y: 0 }, + { x: 620, y: -40 }, + { x: 560, y: 430 }, + { x: -90, y: 300 }, +]; +const TURNS = [5, -12, 30, 80]; +const WORLD_SQUARES = CENTERS.map((c, i) => square(c, TURNS[i])); +const MIDDLE = { + x: CENTERS.reduce((s, c) => s + c.x, 0) / 4, + y: CENTERS.reduce((s, c) => s + c.y, 0) / 4, +}; +const WORLD_INNER = WORLD_SQUARES.map((sq) => nearest(sq, MIDDLE)); + +const toImage = (pts: Pt[]) => pts.map((p) => project(H, p)) as Corners; + +describe("markerPlaneRect", () => { + const rect = markerPlaneRect( + WORLD_SQUARES.map(toImage), + 40, + toImage(WORLD_INNER) as Corners, + ); + const world = rect ? rect.corners.map((p) => project(inverse(H), p)) : []; + + it("is a true rectangle on the desk although the markers are not", () => { + expect(rect).not.toBeNull(); + for (let k = 0; k < 4; k++) { + expect(angleAt(world[(k + 3) % 4], world[k], world[(k + 1) % 4])).toBeCloseTo(90, 0); + } + }); + + it("reports the real width and proportions", () => { + const width = Math.hypot(world[1].x - world[0].x, world[1].y - world[0].y); + const height = Math.hypot(world[3].x - world[0].x, world[3].y - world[0].y); + expect(Math.abs((rect?.widthMm ?? 0) - width) / width).toBeLessThan(0.005); + expect(Math.abs((rect?.aspect ?? 0) - width / height) / (width / height)).toBeLessThan(0.005); + }); + + it("runs along marker 0 -> marker 1 and keeps marker 0 at the top-left", () => { + const along = Math.atan2(world[1].y - world[0].y, world[1].x - world[0].x); + const markers = Math.atan2( + WORLD_INNER[1].y - WORLD_INNER[0].y, + WORLD_INNER[1].x - WORLD_INNER[0].x, + ); + expect(Math.abs(along - markers)).toBeLessThan(0.01); + // The rectangle is the hull of the inner corners: marker 0's corner lies on its top edge + // side, so the top-left is nearer to it than to any other inner corner. + const d = WORLD_INNER.map((p) => Math.hypot(p.x - world[0].x, p.y - world[0].y)); + expect(d.indexOf(Math.min(...d))).toBe(0); + }); + + it("gives nothing without squares", () => { + expect(markerPlaneRect([], 40, toImage(WORLD_INNER) as Corners)).toBeNull(); + }); +}); +``` + +- [ ] **Step 2: Test laufen lassen, muss fehlschlagen** + +Run: `npx vitest --run src/lib/planeMeasure.test.ts` +Expected: FAIL — `markerPlaneRect` ist kein Export. + +- [ ] **Step 3: Implementieren** — in `src/lib/planeMeasure.ts`: + +1. `function invert3` → `export function invert3` (Signatur unverändert). +2. Den Teil von `measureOnPlane`, der aus den Quadraten die Abbildung Bild → Ebene baut, als eigene Funktion herausziehen und `measureOnPlane` darauf umstellen (Verhalten unverändert): + +```ts +/** Image -> metric plane (similarity-correct), from the squares; `anchor` must be off the horizon. */ +function rectifyingHomography(squares: Corners[], anchor: Pt): number[] | null { + const points = squares.map(circularPoint).filter((v): v is CVec => v !== null); + if (points.length === 0) return null; + // The pair is only defined up to a complex factor and conjugation (a square traced the other + // way round sees the conjugate), so each is turned onto the first before summing. + const ref = points[0]; + const sum: CVec = { re: [0, 0, 0], im: [0, 0, 0] }; + for (const v of points) { + const direct = alignTo(v, ref); + const mirrored = alignTo(conj(v), ref); + const { aligned } = direct.residual <= mirrored.residual ? direct : mirrored; + for (let k = 0; k < 3; k++) { + sum.re[k] += aligned.re[k]; + sum.im[k] += aligned.im[k]; + } + } + // M = [re | im | anchor] maps the metric plane onto the image with (1, ±i, 0) landing on the + // circular points, for any anchor off the vanishing line. + return invert3([ + sum.re[0], + sum.im[0], + anchor.x, + sum.re[1], + sum.im[1], + anchor.y, + sum.re[2], + sum.im[2], + 1, + ]); +} + +/** Millimetres per plane unit: the squares' mean side against their printed side. */ +function mmPerUnitOf(toPlane: number[], squares: Corners[], sideMm: number): number | null { + let sideSum = 0; + let sideCount = 0; + for (const square of squares) { + const r = square.map((p) => apply(toPlane, p)); + if (r.some((p) => p === null)) continue; + const q = r as Pt[]; + for (let k = 0; k < 4; k++) sideSum += length(q[k], q[(k + 1) % 4]); + sideCount += 4; + } + return sideCount > 0 && sideSum > 0 ? sideMm / (sideSum / sideCount) : null; +} +``` + + `measureOnPlane` ruft danach `rectifyingHomography(squares, { x: cx, y: cy })` und `mmPerUnitOf(...)` auf. + +3. Neu dazu: + +```ts +/** A true rectangle on the plane, as the image sees it. */ +export interface PlaneRect { + /** Its image corners TL, TR, BR, BL, in the caller's image coordinates. */ + corners: Corners; + /** Real width ÷ height. */ + aspect: number; + /** Real width in millimetres. */ + widthMm: number; +} + +/** + * The rectangle on the plane that just holds the four inner corners, its top edge parallel to + * inner[0] -> inner[1]: a frame that is rectangular on the desk however the markers lie. Null + * when the squares do not pin the plane down. + */ +export function markerPlaneRect( + squares: Corners[], + sideMm: number, + inner: Corners, +): PlaneRect | null { + const anchor = { + x: inner.reduce((s, p) => s + p.x, 0) / 4, + y: inner.reduce((s, p) => s + p.y, 0) / 4, + }; + const toPlane = rectifyingHomography(squares, anchor); + const fromPlane = toPlane ? invert3(toPlane) : null; + if (!toPlane || !fromPlane) return null; + const mmPerUnit = mmPerUnitOf(toPlane, squares, sideMm); + const onPlane = inner.map((p) => apply(toPlane, p)); + if (mmPerUnit === null || onPlane.some((p) => p === null)) return null; + const p = onPlane as Pt[]; + + // Local frame: x along marker 0 -> marker 1. The rectification may mirror the plane; marker 3 + // (bottom-left) must end up below marker 0, so y is flipped when it is not. + const theta = Math.atan2(p[1].y - p[0].y, p[1].x - p[0].x); + const cos = Math.cos(theta); + const sin = Math.sin(theta); + const rotated = p.map((q) => ({ x: q.x * cos + q.y * sin, y: -q.x * sin + q.y * cos })); + const flip = rotated[3].y < rotated[0].y ? -1 : 1; + const local = rotated.map((q) => ({ x: q.x, y: q.y * flip })); + const minX = Math.min(...local.map((q) => q.x)); + const maxX = Math.max(...local.map((q) => q.x)); + const minY = Math.min(...local.map((q) => q.y)); + const maxY = Math.max(...local.map((q) => q.y)); + const w = maxX - minX; + const h = maxY - minY; + if (!(w > 0 && h > 0)) return null; + + const toImage = (q: Pt) => { + const y = q.y * flip; + return apply(fromPlane, { x: q.x * cos - y * sin, y: q.x * sin + y * cos }); + }; + const corners = [ + { x: minX, y: minY }, + { x: maxX, y: minY }, + { x: maxX, y: maxY }, + { x: minX, y: maxY }, + ].map(toImage); + if (corners.some((q) => q === null)) return null; + const [c0, c1, c2, c3] = corners as Pt[]; + return { corners: [c0, c1, c2, c3], aspect: w / h, widthMm: w * mmPerUnit }; +} +``` + +- [ ] **Step 4: Tests laufen lassen, müssen bestehen** + +Run: `npx vitest --run src/lib/planeMeasure.test.ts src/lib/arucoMarkers.test.ts` +Expected: PASS (die bestehenden `arucoMarkers`-Tests zeigen, dass `measureOnPlane` unverändert misst). + +- [ ] **Step 5: Commit** + +```bash +git add src/lib/planeMeasure.ts src/lib/planeMeasure.test.ts +git commit -m "feat(camera): a true rectangle on the desk from freely laid markers" +``` + +--- + +### Task 2: Marker-Erkennung liefert das Basisrechteck + +**Files:** +- Modify: `src/lib/arucoMarkers.ts` +- Modify: `src/lib/arucoMarkers.test.ts` + +**Interfaces:** +- Consumes: `markerPlaneRect`, `PlaneRect` aus Task 1. +- Produces: `MarkedArea` bekommt `plane: PlaneRect | null` mit **normierten** Ecken (0..1 des Bildes); `size` bleibt bis Task 4. + +- [ ] **Step 1: Failing tests** — an den `describe("detectMarkedArea", …)`-Block in `src/lib/arucoMarkers.test.ts` anhängen: + +```ts + it("frames the marked area as a rectangle on the sheet, in image fractions", () => { + const { image, toImage } = photograph(W, H, TILTED, [0, 1, 2, 3]); + const plane = detectMarkedArea(image)?.plane; + expect(plane).toBeTruthy(); + // The test sheet's inner corners already form a rectangle, so the frame is that rectangle. + plane?.corners.forEach((c, i) => { + expectNear(c, toImage(INNER_ON_SHEET[i]), 4); + }); + expectWithin(plane?.aspect ?? 0, AREA_W / AREA_H, 0.04); + expectWithin(plane?.widthMm ?? 0, AREA_W * MM_PER_UNIT, 0.04); + }); +``` + +- [ ] **Step 2: Fehlschlag prüfen** + +Run: `npx vitest --run src/lib/arucoMarkers.test.ts` +Expected: FAIL — `plane` ist `undefined`. + +- [ ] **Step 3: Implementieren** — in `src/lib/arucoMarkers.ts`: + +```ts +import { markerPlaneRect, measureOnPlane, type PlaneRect, type PlaneSize } from "./planeMeasure"; + +/** The four inner corners, the marked area's real size, and a frame that is rectangular on the desk. */ +export interface MarkedArea { + corners: MarkerCorners; + size: PlaneSize | null; + /** Corners normalized 0..1 of the image; null when the markers cannot fix the plane. */ + plane: PlaneRect | null; +} +``` + + und am Ende von `detectMarkedArea` (nach `size`): + +```ts + const squares = all.map((m) => toCorners(m.corners)); + const rect = markerPlaneRect(squares, MARKER_SIZE_MM, toCorners(inner)); + const normalize = (p: CameraPoint) => ({ x: p.x / image.width, y: p.y / image.height }); + return { + corners: toCorners(inner.map(normalize)), + size, + plane: rect ? { ...rect, corners: toCorners(rect.corners.map(normalize)) } : null, + }; +``` + + (`size` nutzt weiterhin `squares`; die bisherige `all.map(...)`-Zeile im `measureOnPlane`-Aufruf durch `squares` ersetzen.) + +- [ ] **Step 4: Tests bestehen** + +Run: `npx vitest --run src/lib/arucoMarkers.test.ts` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/lib/arucoMarkers.ts src/lib/arucoMarkers.test.ts +git commit -m "feat(camera): marker detection frames the desk as a true rectangle" +``` + +--- + +### Task 3: Geometrie des Ausschnitts auf der Ebene (`planeCrop.ts`) + +**Files:** +- Create: `src/lib/ai-edition/timeline/planeCrop.ts` +- Create: `src/lib/ai-edition/timeline/planeCrop.test.ts` + +**Interfaces:** +- Consumes: `homographyFromUnitSquare` (`@/lib/cameraPerspective`), `mapThrough`, `MIN_CROP` (`./calibrationGeometry`), `invert3` (`@/lib/planeMeasure`). +- Produces (alle Bruchteile 0..1 der Ansicht, `viewAspect` = echtes Breite/Höhe der Ansicht): + - `export const PLANE_VIEW_GROW = 1.6;` + - `export function planeView(base: CameraPerspective, grow?: number): CameraPerspective | null` + - `export function cropToPerspective(view: CameraPerspective, crop: CropRegion): CameraPerspective | null` + - `export function perspectiveToCrop(view: CameraPerspective, p: CameraPerspective): CropRegion | null` + - `export function cropAspectOf(viewAspect: number, crop: CropRegion): number` + - `export function fitCrop(viewAspect: number, cropAspect: number | null, box: CropRegion): CropRegion` + - `export type CropCorner = "nw" | "ne" | "sw" | "se";` + - `export function resizeCropLocked(start: CropRegion, corner: CropCorner, dx: number, dy: number, viewAspect: number, cropAspect: number): CropRegion` + - `export function cropLeavesImage(p: CameraPerspective): boolean` + +- [ ] **Step 1: Failing tests** — `src/lib/ai-edition/timeline/planeCrop.test.ts`: + +```ts +import { describe, expect, it } from "vitest"; +import type { CameraPerspective, CropRegion } from "@/components/video-editor/types"; +import { homographyFromUnitSquare } from "@/lib/cameraPerspective"; +import { + cropAspectOf, + cropLeavesImage, + cropToPerspective, + fitCrop, + PLANE_VIEW_GROW, + perspectiveToCrop, + planeView, + resizeCropLocked, +} from "./planeCrop"; + +// Seen straight on: a 0.6 x 0.6 square of the image that is 2:1 in reality. +const FLAT: CameraPerspective = { + corners: [ + { x: 0.2, y: 0.2 }, + { x: 0.8, y: 0.2 }, + { x: 0.8, y: 0.8 }, + { x: 0.2, y: 0.8 }, + ], + aspect: 2, +}; + +// A desk seen steeply: the far edge much narrower than the near one. +const STEEP: CameraPerspective = { + corners: [ + { x: 0.44, y: 0.3 }, + { x: 0.56, y: 0.3 }, + { x: 0.95, y: 0.95 }, + { x: 0.05, y: 0.95 }, + ], + aspect: 1.5, +}; + +function close(a: CropRegion | null, b: CropRegion, digits = 6) { + expect(a).not.toBeNull(); + expect(a?.x).toBeCloseTo(b.x, digits); + expect(a?.y).toBeCloseTo(b.y, digits); + expect(a?.width).toBeCloseTo(b.width, digits); + expect(a?.height).toBeCloseTo(b.height, digits); +} + +describe("planeView", () => { + it("grows the base rectangle about its centre and keeps its real proportions", () => { + const view = planeView(FLAT); + expect(view?.aspect).toBe(2); + expect(view?.corners[0].x).toBeCloseTo(0.2 - 0.3 * 0.6, 9); + expect(view?.corners[2].y).toBeCloseTo(0.8 + 0.3 * 0.6, 9); + }); + + it("puts the base in the middle of the view", () => { + const view = planeView(FLAT); + const side = 1 / PLANE_VIEW_GROW; + close(view && perspectiveToCrop(view, FLAT), { + x: (1 - side) / 2, + y: (1 - side) / 2, + width: side, + height: side, + }); + }); + + it("never reaches past the horizon of a steep view", () => { + const view = planeView(STEEP, 4); + const h = homographyFromUnitSquare(STEEP.corners); + expect(view).not.toBeNull(); + expect(h).not.toBeNull(); + // Every view corner, taken back onto the base, sits in front of the camera. + const box = view && perspectiveToCrop(view, STEEP); + expect(box).not.toBeNull(); + if (!h || !box) return; + const lo = -box.x / box.width; + const hi = (1 - box.x) / box.width; + for (const [u, v] of [ + [lo, lo], + [hi, lo], + [hi, hi], + [lo, hi], + ]) { + expect(h[6] * u + h[7] * v + h[8]).toBeGreaterThan(0); + } + }); +}); + +describe("crop <-> perspective", () => { + it("round-trips a crop", () => { + const view = planeView(STEEP); + const crop = { x: 0.2, y: 0.25, width: 0.5, height: 0.3 }; + const p = view && cropToPerspective(view, crop); + close(view && p && perspectiveToCrop(view, p), crop); + }); + + it("gives the crop its real proportions", () => { + const view = planeView(FLAT); + const p = view && cropToPerspective(view, { x: 0.1, y: 0.1, width: 0.5, height: 0.25 }); + expect(p?.aspect).toBeCloseTo(4, 9); + expect(cropAspectOf(2, { x: 0, y: 0, width: 0.5, height: 0.25 })).toBeCloseTo(4, 9); + }); + + it("notices a crop that leaves the camera image", () => { + const view = planeView(FLAT); + expect(cropLeavesImage(view && cropToPerspective(view, { x: 0, y: 0, width: 1, height: 1 }))).toBe( + false, + ); + const wide = planeView(FLAT, 2); + expect(cropLeavesImage(wide && cropToPerspective(wide, { x: 0, y: 0, width: 1, height: 1 }))).toBe( + true, + ); + }); +}); + +describe("fitCrop", () => { + it("fits the largest crop of a real aspect into a box, centred", () => { + // View 2:1; a 16:9 crop is k = 2 / (16/9) = 1.125 times as tall as wide in fractions. + close(fitCrop(2, 16 / 9, { x: 0, y: 0, width: 1, height: 1 }), { + x: (1 - 1 / 1.125) / 2, + y: 0, + width: 1 / 1.125, + height: 1, + }); + }); + + it("keeps the box for a free format", () => { + const box = { x: 0.1, y: 0.2, width: 0.3, height: 0.4 }; + expect(fitCrop(2, null, box)).toEqual(box); + }); +}); + +describe("resizeCropLocked", () => { + const start = { x: 0.2, y: 0.2, width: 0.4, height: 0.4 * 1.125 }; + + it("keeps the aspect and the opposite corner", () => { + const r = resizeCropLocked(start, "se", 0.1, 0, 2, 16 / 9); + expect(r.x).toBeCloseTo(0.2, 9); + expect(r.y).toBeCloseTo(0.2, 9); + expect(r.width).toBeCloseTo(0.5, 9); + expect(cropAspectOf(2, r)).toBeCloseTo(16 / 9, 9); + }); + + it("follows the axis the pointer moved further along", () => { + const r = resizeCropLocked(start, "nw", 0, -0.2, 2, 16 / 9); + expect(r.x + r.width).toBeCloseTo(0.6, 9); + expect(r.y + r.height).toBeCloseTo(0.2 + 0.45, 9); + expect(r.height).toBeCloseTo(0.45 + 0.2, 9); + }); + + it("stops at the view's edge", () => { + const r = resizeCropLocked(start, "se", 5, 5, 2, 16 / 9); + expect(r.x + r.width).toBeLessThanOrEqual(1 + 1e-9); + expect(r.y + r.height).toBeLessThanOrEqual(1 + 1e-9); + expect(cropAspectOf(2, r)).toBeCloseTo(16 / 9, 9); + }); +}); +``` + +- [ ] **Step 2: Fehlschlag prüfen** + +Run: `npx vitest --run src/lib/ai-edition/timeline/planeCrop.test.ts` +Expected: FAIL — Modul `./planeCrop` fehlt. + +- [ ] **Step 3: Implementieren** — `src/lib/ai-edition/timeline/planeCrop.ts`: + +```ts +// The crop on the rectified desk. A "base" is a CameraPerspective whose corners are a true +// rectangle on the plane (from the markers, or a stored correction). The dialog shows a "view": +// the base grown about its centre, still a true rectangle. The user's crop is a CropRegion in +// fractions of that view; turning it back into four image corners gives a perspective that is +// rectangular on the desk whatever the crop's format. + +import type { CameraPerspective, CropRegion } from "@/components/video-editor/types"; +import { homographyFromUnitSquare } from "@/lib/cameraPerspective"; +import { invert3 } from "@/lib/planeMeasure"; +import { MIN_CROP, mapThrough } from "./calibrationGeometry"; + +/** How much larger than the base the view is, on each side's length. */ +export const PLANE_VIEW_GROW = 1.6; +/** A view corner closer to the horizon than this (homogeneous w) is pulled back. */ +const MIN_W = 0.05; +const GROW_STEPS = 6; + +type Corners = CameraPerspective["corners"]; + +function cornersOf(h: number[], x0: number, y0: number, x1: number, y1: number): Corners | null { + const pts = [ + mapThrough(h, x0, y0), + mapThrough(h, x1, y0), + mapThrough(h, x1, y1), + mapThrough(h, x0, y1), + ]; + if (pts.some((p) => p === null)) return null; + const [a, b, c, d] = pts as NonNullable<(typeof pts)[number]>[]; + return [a, b, c, d]; +} + +/** + * The base grown `grow` times about its centre — less when that would reach past the horizon, + * never less than the base itself. Null when the base is not a usable quad. + */ +export function planeView( + base: CameraPerspective, + grow = PLANE_VIEW_GROW, +): CameraPerspective | null { + const h = homographyFromUnitSquare(base.corners); + if (!h) return null; + for (let k = GROW_STEPS; k >= 0; k--) { + const g = 1 + ((grow - 1) * k) / GROW_STEPS; + const lo = (1 - g) / 2; + const hi = (1 + g) / 2; + const ws = [ + [lo, lo], + [hi, lo], + [hi, hi], + [lo, hi], + ].map(([u, v]) => h[6] * u + h[7] * v + h[8]); + if (ws.some((w) => w <= MIN_W)) continue; + const corners = cornersOf(h, lo, lo, hi, hi); + if (corners) return { corners, aspect: base.aspect }; + } + return null; +} + +/** The view's crop as a perspective: four image corners and the crop's real proportions. */ +export function cropToPerspective( + view: CameraPerspective, + crop: CropRegion, +): CameraPerspective | null { + const h = homographyFromUnitSquare(view.corners); + if (!h) return null; + const corners = cornersOf(h, crop.x, crop.y, crop.x + crop.width, crop.y + crop.height); + return corners ? { corners, aspect: cropAspectOf(view.aspect, crop) } : null; +} + +/** Where a perspective's corners sit in the view, as the rectangle that holds them. */ +export function perspectiveToCrop( + view: CameraPerspective, + p: CameraPerspective, +): CropRegion | null { + const h = homographyFromUnitSquare(view.corners); + const inv = h ? invert3(h) : null; + if (!inv) return null; + const uv = p.corners.map((c) => mapThrough(inv, c.x, c.y)); + if (uv.some((q) => q === null)) return null; + const q = uv as NonNullable<(typeof uv)[number]>[]; + const x0 = Math.min(...q.map((c) => c.x)); + const x1 = Math.max(...q.map((c) => c.x)); + const y0 = Math.min(...q.map((c) => c.y)); + const y1 = Math.max(...q.map((c) => c.y)); + return { x: x0, y: y0, width: x1 - x0, height: y1 - y0 }; +} + +/** Real width ÷ height of a crop in a view of real aspect `viewAspect`. */ +export function cropAspectOf(viewAspect: number, crop: CropRegion): number { + return (viewAspect * crop.width) / crop.height; +} + +/** The largest crop of real aspect `cropAspect` inside `box`, centred; the box itself when free. */ +export function fitCrop(viewAspect: number, cropAspect: number | null, box: CropRegion): CropRegion { + if (cropAspect === null) return { ...box }; + // In view fractions the crop is k times as tall as it is wide. + const k = viewAspect / cropAspect; + const width = Math.min(box.width, box.height / k); + const height = width * k; + return { + x: box.x + (box.width - width) / 2, + y: box.y + (box.height - height) / 2, + width, + height, + }; +} + +export type CropCorner = "nw" | "ne" | "sw" | "se"; + +/** + * `start` resized from `corner` by (dx, dy), its real aspect held and the opposite corner fixed: + * the size follows whichever axis the pointer moved further along, inside the view. + */ +export function resizeCropLocked( + start: CropRegion, + corner: CropCorner, + dx: number, + dy: number, + viewAspect: number, + cropAspect: number, +): CropRegion { + const k = viewAspect / cropAspect; + const east = corner.endsWith("e"); + const south = corner.startsWith("s"); + const anchorX = east ? start.x : start.x + start.width; + const anchorY = south ? start.y : start.y + start.height; + const fromX = start.width + (east ? dx : -dx); + const fromY = (start.height + (south ? dy : -dy)) / k; + const maxWidth = Math.min(east ? 1 - anchorX : anchorX, (south ? 1 - anchorY : anchorY) / k); + const minWidth = Math.max(MIN_CROP, MIN_CROP / k); + const width = Math.min(maxWidth, Math.max(minWidth, Math.max(fromX, fromY))); + const height = width * k; + return { + x: east ? anchorX : anchorX - width, + y: south ? anchorY : anchorY - height, + width, + height, + }; +} + +/** True when a corner of `p` lies outside the camera image (the picture turns black there). */ +export function cropLeavesImage(p: CameraPerspective | null): boolean { + if (!p) return false; + const eps = 0.002; + return p.corners.some((c) => c.x < -eps || c.y < -eps || c.x > 1 + eps || c.y > 1 + eps); +} +``` + +- [ ] **Step 4: Tests bestehen** + +Run: `npx vitest --run src/lib/ai-edition/timeline/planeCrop.test.ts` +Expected: PASS. Falls „never reaches past the horizon“ mit `STEEP` schon bei `grow = 4` kein Zurückziehen braucht, `grow` im Test erhöhen, bis `perspectiveToCrop(view, STEEP).width > 1 / 4` gilt (dann wurde zurückgezogen), und diese Bedingung zusätzlich prüfen. + +- [ ] **Step 5: Commit** + +```bash +git add src/lib/ai-edition/timeline/planeCrop.ts src/lib/ai-edition/timeline/planeCrop.test.ts +git commit -m "feat(camera): crop geometry on the rectified desk" +``` + +--- + +### Task 4: Dialog — Ausschnitt auf dem entzerrten Bild + +**Files:** +- Modify: `src/components/ai-edition/CameraCalibrationModal.tsx` +- Modify: `src/components/ai-edition/CameraCalibrationModal.test.tsx` +- Modify: `src/lib/arucoMarkers.ts`, `src/lib/arucoMarkers.test.ts`, `src/lib/planeMeasure.ts` (Aufräumen: `size`/`measureOnPlane`/`PlaneSize` entfernen) +- Modify: `src/i18n/locales/*/dialogs.json` (15 Dateien) + +**Interfaces:** +- Consumes: `MarkedArea.plane` (Task 2), alles aus `planeCrop.ts` (Task 3). +- Produces: unverändert `onApply({ perspective })`. + +**Verhalten (aus der Spec):** +- *Ebenen-Modus* gilt, wenn `mode === "perspective"`, eine Ebene bekannt ist und der Nutzer nicht auf „Vier Ecken von Hand“ steht. Eine Ebene ist bekannt nach erfolgreicher Erkennung mit `plane`, oder beim Öffnen, wenn eine gespeicherte Perspektive **ohne** `margin` vorliegt und `planeView(stored)` nicht `null` ist (ein gekreuztes Viereck öffnet also wie bisher mit vier Griffen). +- Im Ebenen-Modus zeigt der große Rahmen das entzerrte Bild der Ansicht (`renderRectified(image, view, …)`) im Seitenverhältnis `view.aspect`, darüber das Ausschnitt-Rechteck (Eckgriffe; Kantengriffe nur bei „frei“). Formate: 4:3, 16:9 (Vorgabe), 9:16, 1:1, frei. Kein Rand-Regler. +- Formatwechsel passt den Ausschnitt neu in das Basisrechteck ein (`fitCrop(view.aspect, aspect, perspectiveToCrop(view, plane))`); „frei“ lässt ihn stehen. +- Nach Erkennung mit bekannter Breite: Zeile „Ausschnitt: ca. B × H cm“. +- Liegt eine Ausschnitt-Ecke außerhalb des Kamerabilds: Hinweis `cropOutside`. +- Übernehmen speichert `cropToPerspective(view, planeCrop)` (ohne `margin`); gesperrt, wenn das Seitenverhältnis außerhalb `MIN_ASPECT…MAX_ASPECT` liegt. +- Erkennung ohne `plane`: Ecken auf die Marker setzen, Vier-Griffe-Modus, Meldung `noPlane`. + +- [ ] **Step 1: Failing tests** — in `CameraCalibrationModal.test.tsx`: + + a) Am Dateianfang die echten Geometrie-Funktionen importieren (nicht mocken): + +```ts +import { cropToPerspective, fitCrop, perspectiveToCrop, planeView } from "@/lib/ai-edition/timeline/planeCrop"; +``` + + b) Die beiden Tests „detect fills the handles“ und „a measured area sets the format to its real proportions“ **ersetzen** durch: + +```ts + const FOUND = [ + { x: 0.2, y: 0.25 }, + { x: 0.8, y: 0.2 }, + { x: 0.85, y: 0.9 }, + { x: 0.1, y: 0.8 }, + ]; + // A desk frame seen straight on: 0.6 x 0.6 of the image, 2:1 and 1 m wide in reality. + const PLANE = { + corners: [ + { x: 0.2, y: 0.2 }, + { x: 0.8, y: 0.2 }, + { x: 0.8, y: 0.8 }, + { x: 0.2, y: 0.8 }, + ] as [ + { x: number; y: number }, + { x: number; y: number }, + { x: number; y: number }, + { x: number; y: number }, + ], + aspect: 2, + widthMm: 1000, + }; + const detect = () => + fireEvent.click(screen.getByRole("button", { name: "dialogs.cameraCalibration.detectMarkers" })); + + function expectedPerspective(aspect: number) { + const base = { corners: PLANE.corners, aspect: PLANE.aspect }; + const view = planeView(base); + const box = view && perspectiveToCrop(view, base); + if (!view || !box) throw new Error("test plane must be usable"); + return cropToPerspective(view, fitCrop(view.aspect, aspect, box)); + } + + it("detect without a plane puts the handles on the markers", async () => { + detectMarkedArea.mockReturnValueOnce({ corners: FOUND, plane: null }); + const { onApply } = renderModal("perspective", null); + await stillLoaded(); + detect(); + expect(detectMarkedArea).toHaveBeenCalledWith(IMAGE); + expect(screen.getByRole("status")).toHaveTextContent("dialogs.cameraCalibration.noPlane"); + expect(screen.getByTestId("calibration-handle-0")).toBeInTheDocument(); + fireEvent.click(apply()); + expect(onApply.mock.calls[0][0].perspective.corners).toEqual(FOUND); + }); + + it("detect shows the rectified desk with a 16:9 crop", async () => { + detectMarkedArea.mockReturnValueOnce({ corners: FOUND, plane: PLANE }); + const { onApply } = renderModal("perspective", null); + await stillLoaded(); + detect(); + expect(screen.getByRole("status")).toHaveTextContent("dialogs.cameraCalibration.markersFound"); + expect(screen.getByTestId("calibration-plane")).toBeInTheDocument(); + expect(screen.queryByTestId("calibration-handle-0")).toBeNull(); + fireEvent.click(apply()); + const stored = onApply.mock.calls[0][0].perspective; + const expected = expectedPerspective(16 / 9); + expect(stored.aspect).toBeCloseTo(16 / 9, 9); + expect(stored.margin).toBeUndefined(); + stored.corners.forEach((c: { x: number; y: number }, i: number) => { + expect(c.x).toBeCloseTo(expected?.corners[i].x ?? Number.NaN, 9); + expect(c.y).toBeCloseTo(expected?.corners[i].y ?? Number.NaN, 9); + }); + }); + + it("a format refits the crop on the desk", async () => { + detectMarkedArea.mockReturnValueOnce({ corners: FOUND, plane: PLANE }); + const { onApply } = renderModal("perspective", null); + await stillLoaded(); + detect(); + fireEvent.click(screen.getByRole("button", { name: "9:16" })); + fireEvent.click(apply()); + expect(onApply.mock.calls[0][0].perspective.aspect).toBeCloseTo(9 / 16, 9); + }); + + it("shows the crop's real size after detection", async () => { + detectMarkedArea.mockReturnValueOnce({ corners: FOUND, plane: PLANE }); + renderModal("perspective", null); + await stillLoaded(); + detect(); + expect(screen.getByText("dialogs.cameraCalibration.cropSize")).toBeInTheDocument(); + }); + + it("hand mode brings back the four handles", async () => { + detectMarkedArea.mockReturnValueOnce({ corners: FOUND, plane: PLANE }); + renderModal("perspective", null); + await stillLoaded(); + detect(); + fireEvent.click(screen.getByRole("button", { name: "dialogs.cameraCalibration.handMode" })); + expect(screen.getByTestId("calibration-handle-0")).toBeInTheDocument(); + fireEvent.click(screen.getByRole("button", { name: "dialogs.cameraCalibration.planeMode" })); + expect(screen.getByTestId("calibration-plane")).toBeInTheDocument(); + }); + + it("a stored correction reopens on the desk and applies unchanged", async () => { + const stored = { corners: PLANE.corners, aspect: 2 }; + const { onApply } = renderModal("perspective", { perspective: stored }); + await stillLoaded(); + expect(screen.getByTestId("calibration-plane")).toBeInTheDocument(); + fireEvent.click(apply()); + const out = onApply.mock.calls[0][0].perspective; + expect(out.aspect).toBeCloseTo(2, 9); + out.corners.forEach((c: { x: number; y: number }, i: number) => { + expect(c.x).toBeCloseTo(stored.corners[i].x, 9); + expect(c.y).toBeCloseTo(stored.corners[i].y, 9); + }); + }); + + it("a stored correction with a margin still opens with the four handles", async () => { + renderModal("perspective", { perspective: { corners: PLANE.corners, aspect: 2, margin: 0.1 } }); + await stillLoaded(); + expect(screen.getByTestId("calibration-handle-0")).toBeInTheDocument(); + }); +``` + + c) In den übrigen Tests jedes `detectMarkedArea.mockReturnValueOnce(null)` unverändert lassen. Der Test „reset removes the stored perspective“ bleibt gültig (Zurücksetzen gibt es in beiden Modi). + +- [ ] **Step 2: Fehlschlag prüfen** + +Run: `npx vitest --run src/components/ai-edition/CameraCalibrationModal.test.tsx` +Expected: FAIL in den neuen Tests (kein `calibration-plane`, kein `noPlane`). + +- [ ] **Step 3: Implementieren** + + **3a — Importe und Konstanten** (`CameraCalibrationModal.tsx`): + +```ts +import { + type CropCorner, + cropAspectOf, + cropLeavesImage, + cropToPerspective, + fitCrop, + perspectiveToCrop, + planeView, + resizeCropLocked, +} from "@/lib/ai-edition/timeline/planeCrop"; +``` + + `import type { PlaneSize } from "@/lib/planeMeasure";` entfernen. Neben `FORMATS`: + +```ts +type PlaneFormatId = "standard" | "wide" | "tall" | "square" | "free"; + +/** Formats of a crop on the rectified desk. */ +const PLANE_FORMATS: ReadonlyArray<{ id: PlaneFormatId; aspect: number | null }> = [ + { id: "standard", aspect: 4 / 3 }, + { id: "wide", aspect: 16 / 9 }, + { id: "tall", aspect: 9 / 16 }, + { id: "square", aspect: 1 }, + { id: "free", aspect: null }, +]; + +/** Long side of the rectified desk drawn in the big frame. */ +const PLANE_VIEW_LONG_SIDE_PX = 960; + +function planeFormatOf(aspect: number): PlaneFormatId { + const match = PLANE_FORMATS.find((f) => f.aspect !== null && Math.abs(f.aspect - aspect) < 1e-3); + return match?.id ?? "free"; +} + +/** A stored correction that can reopen as a crop on the desk: no margin, a usable quad. */ +function storedPlane(stored: CameraPerspective | undefined): CameraPerspective | null { + if (!stored || stored.margin) return null; + const base = { corners: copyCorners(stored.corners), aspect: stored.aspect }; + return planeView(base) ? base : null; +} +``` + + `MarkerResult` ersetzen durch: + +```ts +/** What "Detect markers" last reported. */ +type MarkerResult = { kind: "found" } | { kind: "noPlane" } | { kind: "notFound" }; +``` + + **3b — Zustand** (nach den bestehenden `useState`-Zeilen): + +```ts + // The desk plane: a true rectangle on it (from the markers or the stored correction), the crop + // the user places on its rectified view, and the plane's real width when the markers told it. + const [plane, setPlane] = useState(() => storedPlane(stored)); + const [manual, setManual] = useState(() => storedPlane(stored) === null); + const [planeWidthMm, setPlaneWidthMm] = useState(null); + const [planeFormat, setPlaneFormat] = useState(() => + storedPlane(stored) ? planeFormatOf(stored?.aspect ?? 1) : "wide", + ); + const view = useMemo(() => (plane ? planeView(plane) : null), [plane]); + /** Where the plane's base rectangle sits in the view: the default crop's box. */ + const baseBox = useMemo( + () => (view && plane ? perspectiveToCrop(view, plane) : null), + [view, plane], + ); + const [planeCrop, setPlaneCrop] = useState(() => { + const base = storedPlane(stored); + const v = base ? planeView(base) : null; + return (v && base && perspectiveToCrop(v, base)) ?? FULL_CROP; + }); + const planeRef = useRef(null); +``` + + **3c — Abgeleitete Werte** (nach `const perspective = useMemo(...)` die bisherige Variable in `handPerspective` umbenennen und so ergänzen): + +```ts + const planeMode = mode === "perspective" && !manual && view !== null; + const planeAspect = PLANE_FORMATS.find((f) => f.id === planeFormat)?.aspect ?? null; + const planePerspective = useMemo( + () => (planeMode && view ? cropToPerspective(view, planeCrop) : null), + [planeMode, view, planeCrop], + ); + const planeAspectOk = + planePerspective !== null && + planePerspective.aspect >= MIN_ASPECT && + planePerspective.aspect <= MAX_ASPECT; + const perspective = planeMode ? (planeAspectOk ? planePerspective : null) : handPerspective; + const preview = previewSize(perspective?.aspect ?? aspect ?? 1, PREVIEW_LONG_SIDE_PX); +``` + + (Die bisherige Zeile `const preview = previewSize(aspect ?? 1, …)` entfällt.) + + **3d — Entzerrtes Bild zeichnen** (neuer Effekt neben dem Vorschau-Effekt): + +```ts + // The rectified desk under the crop, redrawn when the plane changes. + useEffect(() => { + const canvas = planeRef.current; + const ctx = canvas?.getContext("2d"); + if (!planeMode || !canvas || !ctx || !image || !view) return; + const size = previewSize(view.aspect, PLANE_VIEW_LONG_SIDE_PX); + canvas.width = size.width; + canvas.height = size.height; + const pixels = renderRectified(image, view, size.width, size.height); + if (pixels) ctx.putImageData(new ImageData(pixels, size.width, size.height), 0, 0); + }, [planeMode, image, view]); +``` + + **3e — Ausschnitt-Handler auf das aktive Rechteck umstellen.** `startCropMove`, `startCropResize` und `onCropKeyDown` arbeiten heute auf `crop`/`setCrop`. Davor einfügen und diese drei Funktionen darauf umstellen: + +```ts + // The rectangle the crop handles move: the camera crop, or the crop on the rectified desk. + const activeCrop = planeMode ? planeCrop : crop; + const setActiveCrop = planeMode ? setPlaneCrop : setCrop; + /** Real aspect the crop must keep while resized; null = free. */ + const lockedAspect = planeMode ? planeAspect : null; + + const resizeActive = (start: CropRegion, edges: CropEdges, dx: number, dy: number) => { + const corner = cornerOf(edges); + return lockedAspect !== null && corner && view + ? resizeCropLocked(start, corner, dx, dy, view.aspect, lockedAspect) + : resizeCrop(start, edges, dx, dy); + }; +``` + + mit der Hilfsfunktion auf Modulebene: + +```ts +/** The corner a pair of moved edges names; null for a single edge. */ +function cornerOf(edges: CropEdges): CropCorner | null { + const ns = edges.top ? "n" : edges.bottom ? "s" : null; + const we = edges.left ? "w" : edges.right ? "e" : null; + return ns && we ? (`${ns}${we}` as CropCorner) : null; +} +``` + + In den drei Handlern `crop` → `activeCrop`, `setCrop(` → `setActiveCrop(` und `resizeCrop(start, edges, …)` bzw. `resizeCrop(crop, { right: …, bottom: … }, …)` → `resizeActive(…)` mit denselben Argumenten. In `onCropKeyDown` bei Umschalt die Kanten `{ right: true, bottom: true }` übergeben, wenn `lockedAspect !== null` (Eckzug „se“), sonst wie bisher `{ right: dx !== 0, bottom: dy !== 0 }`. + + **3f — Erkennung** (`detectMarkers` ersetzen): + +```ts + // The markers fix the desk plane; the crop on it starts as the format's largest rectangle in + // the markers' frame. Without a plane they still place the four corners. + const detectMarkers = () => { + if (!image) return; + const area = detectMarkedArea(image); + if (!area) { + setMarkerResult({ kind: "notFound" }); + return; + } + setCorners(copyCorners(area.corners)); + const base = area.plane ? { corners: copyCorners(area.plane.corners), aspect: area.plane.aspect } : null; + const nextView = base ? planeView(base) : null; + const box = nextView && base ? perspectiveToCrop(nextView, base) : null; + if (!base || !nextView || !box) { + setManual(true); + setMarkerResult({ kind: "noPlane" }); + return; + } + setPlane(base); + setPlaneWidthMm(area.plane?.widthMm ?? null); + setPlaneCrop(fitCrop(nextView.aspect, planeAspect, box)); + setManual(false); + setMarkerResult({ kind: "found" }); + }; + + const selectPlaneFormat = (id: PlaneFormatId) => { + setPlaneFormat(id); + const target = PLANE_FORMATS.find((f) => f.id === id)?.aspect ?? null; + if (target !== null && view && baseBox) setPlaneCrop(fitCrop(view.aspect, target, baseBox)); + }; + + // Real size of the crop, when the markers measured the plane. + const cropSizeCm = + planeMode && view && baseBox && planeWidthMm !== null + ? (() => { + const widthMm = (planeCrop.width / baseBox.width) * planeWidthMm; + return { + width: Math.round(widthMm / 10), + height: Math.round(widthMm / cropAspectOf(view.aspect, planeCrop) / 10), + }; + })() + : null; +``` + + **3g — JSX.** + - Hinweistext oben: `planeMode ? t("cameraCalibration.planeHelp") : isPerspective ? t("cameraCalibration.perspectiveHelp") : t("cameraCalibration.cropHelp")`. + - Rahmen: `style={{ ...previewBoxStyle(planeMode && view ? view.aspect : imageAspect), touchAction: "none" }}`, `onPointerDown={isPerspective && !planeMode ? onFramePointerDown : undefined}`. Den Standbild-Canvas mit `display: planeMode ? "none" : undefined` versehen (er bleibt Quelle der Lupe). Dahinter im Ebenen-Modus: + +```tsx + {planeMode ? ( + + ) : null} +``` + + - Die Bedingung des bestehenden Blocks `{isPerspective ? (<> …Polygon, Griffe, Lupe… ) : (

)}` wird zu `{isPerspective && !planeMode ? ( …unverändert… ) : ( …Ausschnitt… )}`. Im Ausschnitt-`div` `crop` durch `activeCrop` ersetzen und die Kantengriffe nur rendern, wenn `lockedAspect === null`: `{lockedAspect === null ? CROP_EDGES.map(…) : null}`. + - Knopfzeile (Erkennen/Drucken): nach „Markerblatt drucken“, nur wenn `view !== null`: + +```tsx + {view !== null ? ( + + ) : null} +``` + + Status-Text: `found` → `markersFound`, `noPlane` → `noPlane`, `notFound` → `markersNotFound`. + - Formatspalte: im Ebenen-Modus statt der bisherigen `ChoiceRow` + Freiverhältnis + Rand: + +```tsx + {planeMode ? ( + <> + + label={t("cameraCalibration.format")} + columns={3} + options={[ + { value: "standard", label: "4:3" }, + { value: "wide", label: "16:9" }, + { value: "tall", label: "9:16" }, + { value: "square", label: "1:1" }, + { value: "free", label: t("cameraCalibration.formats.free") }, + ]} + value={planeFormat} + onChange={selectPlaneFormat} + /> + {cropSizeCm ? ( +

+ {t("cameraCalibration.cropSize", cropSizeCm)} +

+ ) : null} + {cropLeavesImage(planePerspective) ? ( +

+ {t("cameraCalibration.cropOutside")} +

+ ) : null} + + ) : ( + <>…bisherige Format-ChoiceRow, Freiverhältnis, ungültiges Verhältnis, Rand-Regler… + )} +``` + + - Der Hinweis „invalidQuad“ und `aria-describedby` am Übernehmen-Knopf nur im Hand-Modus: Bedingung `isPerspective && !planeMode && !quadValid`. + + **3h — `apply`** bleibt; es benutzt jetzt das neue `perspective` (im Ebenen-Modus ohne `margin`). `canApply` bleibt `isPerspective ? perspective !== null : true`. + + **3i — Aufräumen** (alte Messung, die durch die Ebene ersetzt ist): + - `src/lib/arucoMarkers.ts`: Feld `size` aus `MarkedArea` und den `measureOnPlane`-Aufruf entfernen; Import auf `markerPlaneRect, type PlaneRect`. + - `src/lib/planeMeasure.ts`: `measureOnPlane` und `PlaneSize` entfernen. + - `src/lib/arucoMarkers.test.ts`: die drei Tests, die `area?.size` prüfen („measures the marked rectangle's real size…“, „…steep keystone…“, „…rotated sheet…“), auf `area?.plane` umstellen: Breite `plane.widthMm` statt `size.widthMm`, Höhe `plane.widthMm / plane.aspect` statt `size.heightMm`, Verhältnis `plane.aspect`; Toleranzen unverändert. + + **3j — Texte** in allen 15 `src/i18n/locales/*/dialogs.json` unter `cameraCalibration`: `markersMeasured` entfernen; neu (deutsch/englisch hier, die übrigen 13 sinngemäß übersetzen): + +| Schlüssel | en | de | +|---|---|---| +| `planeHelp` | Drag the crop on the straightened desk. The corner handles resize it. | Ziehe den Ausschnitt auf dem entzerrten Tisch. Die Eckgriffe ändern die Größe. | +| `handMode` | Four corners by hand | Vier Ecken von Hand | +| `planeMode` | Crop on the straightened desk | Ausschnitt auf entzerrtem Tisch | +| `noPlane` | All four markers found, but the desk plane could not be determined; the corners were placed on them. | Alle vier Marker gefunden, aber die Tischebene ließ sich nicht bestimmen; die Ecken wurden auf sie gesetzt. | +| `cropSize` | Crop: about {{width}} × {{height}} cm | Ausschnitt: ca. {{width}} × {{height}} cm | +| `cropOutside` | The crop reaches past the camera picture; that part stays black. | Der Ausschnitt reicht über das Kamerabild hinaus; dort bleibt das Bild schwarz. | + + `perspectiveHelp` bleibt (gilt für den Hand-Modus). + +- [ ] **Step 4: Tests und Prüfungen** + +Run: `npx vitest --run src/components/ai-edition/CameraCalibrationModal.test.tsx src/lib/arucoMarkers.test.ts src/lib/planeMeasure.test.ts src/lib/ai-edition/timeline/planeCrop.test.ts` +Expected: PASS. + +Run: `npx tsc --noEmit; npx tsc -p tsconfig.test.json --noEmit; npm run lint; npm run i18n:check` +Expected: keine Fehler. + +- [ ] **Step 5: Commit** + +```bash +git add -A src/components/ai-edition/CameraCalibrationModal.tsx src/components/ai-edition/CameraCalibrationModal.test.tsx src/lib/arucoMarkers.ts src/lib/arucoMarkers.test.ts src/lib/planeMeasure.ts src/i18n/locales +git commit -m "feat(camera): crop on the straightened desk after marker detection" +``` + +--- + +### Task 5: Gesamtprüfung und echter Lauf + +**Files:** +- Modify: `technical-documentation/testing/manual-e2e-checklist.md` (Results-Log-Zeile) + +- [ ] **Step 1: Volle Suite** + +Run: `npm run test` +Expected: alle Dateien grün (Stand vor dieser Arbeit: 324 Dateien, 4437 Tests). + +- [ ] **Step 2: Echter Lauf mit dem Bild des Nutzers (ohne App)** — die Probe aus der Diagnose wiederholen: Einzelbild aus `recording-1791290781616-webcam-2.mp4` (ffmpeg aus `crates/thirdparty/ffmpeg-*/bin`), `detectMarkedArea` darauf, `planeView` + `fitCrop(…, 16/9, …)` + `cropToPerspective`, Ergebnis mit `renderRectified` als PNG schreiben und ansehen: Tischkante und Tastatur gerade, rechte Winkel. + +- [ ] **Step 3: Dev-Build starten** (`OPENSCREEN_COMPOSITOR_VIEW_NODE=C:\osc-cams\electron\native\bin\win32-arm64\compositor_view.node`, `npm run dev` in `C:\osc-cams`) und dem Nutzer den Test übergeben: Perspektive korrigieren → Marker erkennen → 16:9 → Übernehmen → Vorschau und Export gerade; Größe gegen Zollstock. + +- [ ] **Step 4: Results-Log-Zeile** nach dem Lauf des Nutzers eintragen (Datum, `feat/multi-camera-layouts @ `, Windows 11 ARM64, Partial, was geprüft und was nicht) und committen: + +```bash +git add technical-documentation/testing/manual-e2e-checklist.md +git commit -m "test: log the straightened-desk crop run" +``` diff --git a/docs/superpowers/plans/2026-10-07-main-camera-pip-toggles-hud-desk.md b/docs/superpowers/plans/2026-10-07-main-camera-pip-toggles-hud-desk.md new file mode 100644 index 0000000000..e7e99d199b --- /dev/null +++ b/docs/superpowers/plans/2026-10-07-main-camera-pip-toggles-hud-desk.md @@ -0,0 +1,134 @@ +# Hauptkamera, Kamera-Schalter, Tischkamera im HUD — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** (D) Kameras eines „Bildschirm + Kamera“-Abschnitts per Schalter; (C) eine wählbare Hauptkamera, die die Rolle von Kamera 1 übernimmt; (E) die Tischkamera als HUD-Vorgabe, die neue Projekte übernehmen. + +**Architecture:** D ist ein neuer Store-Schreiber plus Inspector-Liste. C ist eine reine Dokument-Transformation am Eingang des Szenenaufbaus (`withMainCamera(document)` tauscht Kamera 0 und die Hauptkamera in Assets, Kamera-Einstellungen, Layout-Abschnitten und Tischkamera), dazu eine Auswahl im Kamera-Layout-Bereich; Compositor unverändert. E führt die HUD-Wahl als Index durch Aufnahme-Anfrage und Sitzungsdatei bis `projectStore.addAsset`. + +**Tech Stack:** TypeScript (strict), React 19, Electron IPC, Vitest (node + jsdom), Biome, i18next (15 Locales), Playwright `_electron`. + +**Spec:** `docs/superpowers/specs/2026-10-07-main-camera-pip-toggles-hud-desk-design.md` + +## Global Constraints + +- Arbeitsverzeichnis `C:\osc-cams`, Branch `feat/multi-camera-layouts`; nichts pushen. +- Keine Änderung an Rust/Shadern/Compositor; keine Schema-Version (neue Felder optional in `legacyEditor` bzw. Sitzungsdatei/Prefs). +- Projekte ohne `mainCamera` / ohne neue Felder: Szene und Dokument unverändert. +- Kamera-Indizes in `cameraSettings`, Layout-Abschnitten und `deskCamera` meinen das Gerät, nie die Rolle. +- Verfügbarkeit einer Kamera: dieselbe Regel wie die Tischkamera (`projectCameraAvailable` in `cameraList.ts`). +- Ein Schreibvorgang = ein Undo-Schritt; Schreiber aus dem Inspector/Layout-Bereich laufen über die Schreib-Warteschlange der Shell (`enqueueTimelineWrite`, wie `setCameraSettingsQueued`). +- Code-Kommentare Englisch; Biome; kein `any`; jeder UI-Text in allen 15 Locales (Desk-/Kamera-Vokabular der jeweiligen Sprache wiederverwenden); `npm run i18n:check` grün. +- Keine Claude-Signatur und kein `Co-Authored-By` in Commits. +- Prüfen pro Task: fokussierte Tests, beide `tsc`; volle Suite (`npm run test`) einmal je Teil im letzten Code-Task des Teils. + +## Review Focus + +- **Hauptkamera = Tischkamera**: D-Abschnitt wird Full-Camera-Zeile, genau ein Label — Test in Task C2. +- **Hauptkamera nicht verfügbar** (Datei weg / Index zu groß): Szene wie ohne Hauptkamera, Editor zeigt Hinweis — Tests C1, C2, C3. +- **Layout-Abschnitt mit Kamera 0 und der Hauptkamera zugleich** (z. B. „Bildschirm + Kamera“ mit Kamera 1 und 2, Hauptkamera 2): beide Indizes werden getauscht, nicht doppelt — Test C2. +- **Schalter „Bildschirm + Kamera“ bei verschobenem Fenster**: Position bleibt bei seiner Kamera, wenn eine andere ein-/ausgeschaltet wird — Test D1. +- **HUD: gewählte Tischkamera wird nicht mehr aufgenommen** (abgehakt / abgesteckt): als „Keine“ behandelt, Sitzungsdatei ohne `deskCamera` — Test E1. + +--- + +## Teil D — „Bildschirm + Kamera“ mit Schaltern + +### Task D1: Store-Schreiber `setLayoutSectionCameras` + +**Files:** Modify `src/lib/ai-edition/store/useTimeline.ts` (neben `setLayoutSlotCamera`, ~l.1632); Test `src/lib/ai-edition/store/useTimeline.test.ts` (nach Muster der `setLayoutSlotCamera`-Tests); ggf. `documentWriteAudit`-Tabelle ergänzen. + +**Interfaces — Produces:** `setLayoutSectionCameras(handle: { kind: "cameraLayout"; id: string }, cameras: number[]): Promise<"set" | "too-few" | "too-many" | "unchanged">` auf dem Rückgabeobjekt von `useTimeline`. + +**Verhalten:** Neue `slots` = `cameras` dedupliziert, aufsteigend sortiert, je Kamera der vorhandene Slot (mit seinem `rect`) wiederverwendet, sonst `{ camera }`. Weniger als `TEMPLATE_SLOTS[template].min` → `"too-few"`, mehr als `max` → `"too-many"`, nichts geschrieben. Gleiche Liste → `"unchanged"`, nichts geschrieben. Sonst eine Speicherung mit Verlauf (`patchPillById` + `withCameraLanes`, alle Zeilen der Pille), `"set"`. + +- [ ] **Step 1: Failing tests:** (1) screen-pip mit `[0]` → `setLayoutSectionCameras(h, [2, 0])` → slots `[{camera:0},{camera:2}]`; (2) Slot von Kamera 2 mit `rect` bleibt erhalten, wenn Kamera 1 dazukommt (`[0,1,2]` → rect an Kamera 2 unverändert); (3) `[]` → `"too-few"`, Dokument unverändert; (4) vier Kameras → `"too-many"`; (5) gleiche Liste → `"unchanged"`, kein Undo-Eintrag; (6) ein Aufruf = ein Undo-Schritt. +- [ ] **Step 2:** FAIL prüfen. **Step 3:** implementieren. **Step 4:** Tests + tsc. **Step 5:** Commit `feat(camera): set a layout section's cameras as a set`. + +### Task D2: Inspector-Schalter für „Bildschirm + Kamera“ + +**Files:** Modify `src/components/ai-edition/v4/LayoutSectionPane.tsx` (Kamera-Plätze ~l.141-177) und Test daneben; `src/components/ai-edition/NewEditorShell.tsx` (Schreiber über die Warteschlange durchreichen, wie `setCameraSettingsQueued`); Locales `settings.json` (15). + +**Verhalten:** Nur bei `template === "screen-pip"`: statt der Auswahllisten eine Liste „Kameras“ mit einem Schalter (`role="switch"`/vorhandene `Toggle`-Komponente, Name = Kameraname wie im Abschnitt „Kameras“) pro Projektkamera; an = Kamera hat ein Fenster. Gesperrt: der einzige eingeschaltete (Hinweis `layoutSection.minOneCamera` „Mindestens eine Kamera“), alle ausgeschalteten bei drei eingeschalteten (Hinweis `layoutSection.maxThreeWindows` „Höchstens drei Fenster“), ausgeschaltete nicht verfügbare Kameras. Umschalten ruft den Schreiber aus D1 mit der neuen Menge. Andere Vorlagen unverändert. + +- [ ] **Step 1: Failing tests (jsdom):** Schalter je Kamera mit Kameranamen; Einschalten ruft Schreiber mit `[…bisher, neu]`; einziger an → gesperrt + Hinweis; drei an → übrige gesperrt + Hinweis; nicht verfügbare aus → gesperrt; Vorlage `side-by-side` zeigt weiter Auswahllisten. +- [ ] **Step 2:** FAIL. **Step 3:** implementieren (+ Texte 15 Locales). **Step 4:** fokussierte Tests, tsc ×2, `npm run lint`, `npm run i18n:check`, **`npm run test`**. **Step 5:** Commit `feat(camera): switch cameras on and off in a screen + camera section`. + +### Task D3: App-Lauf D (Controller, Playwright) + +- [ ] `npm run build-vite`; neues Skript `scripts/editor-pip-switches-smoke.mjs` nach Muster von `scripts/editor-desk-smoke.mjs` (Kopie von `proj_5989a8b4-6b0f-46e8-b7bf-b22c77703c57`, Projektordner per Hash prüfen): „Bildschirm + Kamera“-Abschnitt anlegen → Inspector → Kamera 2 einschalten (Datei: zwei Slots, Vorschau-Screenshot mit zwei Fenstern) → Kamera 1 ausschalten (ein Slot, Kamera 2) → letzten Schalter gesperrt → Undo/Redo → speichern, neu starten, gleich. Screenshots ansehen. Results-Log-Zeile, Commit `test: scripted screen + camera switch pass`. + +--- + +## Teil C — Hauptkamera + +### Task C1: Kern — `resolveMainCamera` und `withMainCamera` + +**Files:** Create `src/lib/mainCamera.ts`, `src/lib/mainCamera.test.ts`. + +**Interfaces — Produces:** +- `resolveMainCamera(input: { mainCamera: unknown; cameraCount: number; available: (index: number) => boolean }): number` — gewählte Kamera wenn ganzzahlig, `0 <= i < cameraCount` und verfügbar; sonst `0`. +- `withMainCamera(document: AxcutDocument): AxcutDocument` — liest `legacyEditor.mainCamera`, löst mit `projectCameraCount`/`projectCameraAvailable` auf; bei `m === 0` **dasselbe Objekt** zurück; sonst ein neues Dokument, in dem Kamera 0 und *m* vertauscht sind: + - jedes Asset: `cameraTrack` ↔ `additionalCameraTracks[m-1]` (nur Assets, die beide haben; sonst unverändert); + - `legacyEditor.cameraSettings`: Eintrag 0 ↔ Eintrag *m*, **aber** Drehung/Spiegeln/Zuschnitt der Hauptkamera kommen aus den Layout-Reglern (die Legacy-Felder bleiben, wie sie sind, und gelten nun für Index 0): im neuen Eintrag 0 bleibt nur `perspective` aus dem alten Eintrag *m*; der neue Eintrag *m* ist der alte Eintrag 0 (Einstellungen der echten Kamera 1) vollständig; + - `cameraLayoutRegions[].slots[].camera` und `deskCamera`: 0 ↔ *m* (jeder Wert genau einmal abgebildet); + - `mainCamera` entfernt (die Szene sieht nur noch Kamera 0 als Rolle). + +- [ ] **Step 1: Failing tests:** `resolveMainCamera` (gewählt+verfügbar → i; außer Bereich → 0; nicht verfügbar → 0; fehlt/kein Integer → 0). `withMainCamera`: ohne Feld → `toBe(document)`; *m* = 2 → `cameraTrack`/`additionalCameraTracks[1]` getauscht; `cameraSettings` getauscht nach obiger Regel (Perspektive von 2 landet bei 0, Drehung von 2 nicht); Slots `[0,2]` → `[2,0]`; `deskCamera: 2` → `0`, `deskCamera: 0` → `2`, `deskCamera: 1` → `1`; Hauptkamera nicht verfügbar → `toBe(document)`. +- [ ] **Step 2–5:** FAIL, implementieren, grün + tsc, Commit `feat(camera): the main camera as a document transform`. + +### Task C2: Szene benutzt `withMainCamera` + +**Files:** Modify `src/native/sceneDescription.ts` (Eingang von `buildSceneDescription`; Stellen, die das Dokument für Kameras/Layout/Tischkamera lesen) und Test. + +**Verhalten:** `buildSceneDescription` arbeitet auf `withMainCamera(document)`. Prüfen, dass jede Stelle, die Kameras liest (Clip-`webcamPath`/`additionalCameras` ~l.1202-1219, `webcamSourceSizeOf` ~l.1472, `cameras[]` ~l.1594, Layout-Regionen ~l.1557, Tischkamera ~l.1329), das transformierte Dokument benutzt. Clip-/Asset-Ids bleiben gleich, also bleiben Verankerungen gültig. + +- [ ] **Step 1: Failing tests:** Zwei-Kamera-Fixture, `mainCamera: 1` → Clip `webcamPath` = Pfad von Kamera 2, `additionalCameras[0]` = Pfad von Kamera 1; Preset-Quellmaße aus Kamera 2; Layout-Abschnitt mit Slot 1 → in der Szene Slot 0; `deskCamera: 1` + D-Abschnitt → Full-Camera-Zeile, genau ein Label; `deskCamera: 0` + D-Abschnitt → `camera-full`-Layout mit Kamera 1 (Index 1 in der Szene); Hauptkamera nicht verfügbar → Szene `toEqual` der Szene ohne `mainCamera`; ohne Feld → wörtliche Erwartungswerte (vorhandener Leitplanken-Test bleibt grün). +- [ ] **Step 2–5:** FAIL, implementieren, grün + tsc, Commit `feat(camera): the scene draws the main camera in camera 1's place`. + +### Task C3: Editor — Auswahl „Hauptkamera“, Kameras-Abschnitt, Store + +**Files:** Modify `src/lib/ai-edition/store/useTimeline.ts` (`mainCamera` aufgelöst, `mainCameraChosen`, `setMainCamera(index | null)`), `src/components/ai-edition/RightPanes.tsx` (`LayoutPane`, neben „Preset“), `src/components/ai-edition/CamerasSection.tsx` (Regler-Notiz und Einstellungs-Zeilen an der Hauptkamera statt fest an Index 0), `src/components/ai-edition/NewEditorShell.tsx` (Warteschlange), Locales `settings.json` (15); Tests daneben. + +**Verhalten:** +- Store: `mainCamera: number` (aufgelöst, mit `resolveMainCamera`, gleiche Verfügbarkeit wie Szene), `mainCameraChosen: number | null` (gespeichert, roh), `setMainCamera(i)` schreibt/entfernt `legacyEditor.mainCamera` (`null` oder `0` → Feld entfernt), ein Undo-Schritt, nichts bei unveränderter Wahl. +- Kamera-Layout-Bereich: Auswahl „Hauptkamera“ (`settings.layout.mainCamera`), sichtbar ab zwei Projektkameras, Einträge mit Kameranamen (`projectCameraLabel`), nicht verfügbare deaktiviert; gewählt aber nicht verfügbar → Hinweis `settings.layout.mainCameraUnavailable` („Nicht verfügbar — verwendet wird {{camera}}“). +- Abschnitt „Kameras“: die Zeile der **Hauptkamera** zeigt die Notiz „Ihre Regler stehen oben“ (bisheriger Text `cameras.camera1Hint` sinngemäß angepasst auf „Hauptkamera“) und keine Drehung/Spiegeln/Zuschnitt-Zeilen; alle anderen Kameras — auch Kamera 1, wenn nicht Hauptkamera — zeigen sie und schreiben in `cameraSettings[i]`. + +- [ ] **Step 1: Failing tests:** Store (setzen/entfernen/Undo, roh vs aufgelöst bei verlorener Wahl); LayoutPane (Auswahl erst ab zwei Kameras, Aufruf mit Index, Hinweis bei verlorener Wahl); CamerasSection (Hauptkamera 2 → Kamera 1 hat Drehung/Spiegeln/Zuschnitt, Kamera 2 die Notiz). +- [ ] **Step 2–5:** FAIL, implementieren (+ 15 Locales), fokussierte Tests, tsc ×2, lint, i18n, **`npm run test`**, Commit `feat(camera): choose the main camera in the layout pane`. + +### Task C4: App-Lauf C (Controller, Playwright) + +- [ ] Skript `scripts/editor-main-camera-smoke.mjs` (Muster wie D3): Hauptkamera auf Kamera 2 → Datei `mainCamera: 1` → Vorschau außerhalb von Abschnitten zeigt Kamera 2 als PiP (Screenshot gegen vorher deutlich verschieden) → C-Abschnitt zeigt Kamera 2 im Vollbild → Kameras-Abschnitt: Kamera 1 hat Drehung/Spiegeln → Undo/Redo → speichern, neu starten → CLI-Export, Bild im Preset- und im C-Bereich gegen Vorschau. Screenshots ansehen. Results-Log-Zeile, Commit. + +--- + +## Teil E — Tischkamera im HUD + +### Task E1: Prefs, Aufnahme-Anfrage und Sitzungsdatei + +**Files:** Modify `electron/app-settings.ts` (Key `recording.camDeskDevice`, Muster `camAdditionalDevices` ~l.82-131, 227-235), Preload/Typen der Prefs, `src/components/launch/LaunchWindow.tsx` (Zustand + `persistRecordingPrefs`, ~l.862-960), `src/hooks/useScreenRecorder.ts` (~l.1301-1343: `deskCamera`-Index in die Anfrage), `electron/ipc/handlers.ts` (Sitzungsdatei ~l.1409-1425; `findRecordingCamera` ~l.1909-1970 liefert `deskCamera` mit), `src/lib/recordingSession.ts` (normalisieren); Tests daneben. + +**Interfaces — Produces:** +- Pref `camDeskDevice: { id: string | null; name: string } | null`. +- Pure Funktion (neu, z. B. in `src/lib/additionalWebcams.ts`): `deskCameraIndex(desk: {id,name} | null, camera1: {id,name} | null, recorded: Array<{id,name}>): number | undefined` — 0 wenn Kamera 1, *k* wenn die *k*-te zusätzliche aufgenommene, sonst `undefined` (gleiche Gleichheitsregel wie `isSameCamera`: id, sonst Name). +- Sitzungsdatei/`findRecordingCamera`-Ergebnis: optionales `deskCamera: number`. + +- [ ] **Step 1: Failing tests:** `deskCameraIndex` (Kamera 1 → 0; zweite zusätzliche → 2; nicht aufgenommen → undefined; null → undefined); Prefs lesen/schreiben (Muster der `camAdditionalDevices`-Tests); `recordingSession` normalisiert `deskCamera` (gültig/ungültig); `findRecordingCamera` gibt `deskCamera` weiter. +- [ ] **Step 2–5:** FAIL, implementieren, grün + tsc ×2, Commit `feat(recording): remember the desk camera of a recording`. + +### Task E2: HUD-Auswahl und Übernahme ins Projekt + +**Files:** Modify `src/components/launch/HudDeviceSettings.tsx` (unter `AdditionalCamerasList`), ggf. `src/components/ai-edition/v4/RecStage.tsx` (zweiter Editor derselben Prefs — gleiche Auswahl, falls dort die zusätzlichen Kameras gewählt werden), `src/lib/ai-edition/store/projectStore.ts` (`addAsset` ~l.418-436), Locales `launch`/HUD-Namespace (15); Tests daneben. + +**Verhalten:** +- HUD: Abschnitt „Tischkamera“ mit Einträgen „Keine“ + Kamera 1 + angehakte zusätzliche (Namen), Häkchen am gewählten; gleiche Sperre wie die zusätzlichen Kameras (`disabled` + Hinweis), nur sichtbar ab zwei Kameras. Ist die gespeicherte Wahl nicht unter den angebotenen, ist „Keine“ markiert. +- `addAsset`: `legacyEditor.deskCamera = camera.deskCamera`, nur wenn das Dokument noch kein `deskCamera` hat und `deskCamera` 0 oder ≤ Anzahl verknüpfter zusätzlicher Kameras ist; Teil derselben Speicherung (`history: false` wie dort üblich). + +- [ ] **Step 1: Failing tests:** HUD-Liste (Einträge, Klick setzt Pref, Sperre, „Keine“ bei nicht angebotener Wahl); `addAsset` setzt `deskCamera` bei leerem Projekt, nicht bei vorhandenem, nicht bei Index außerhalb. +- [ ] **Step 2–5:** FAIL, implementieren (+ 15 Locales), fokussierte Tests, tsc ×2, lint, i18n, **`npm run test`**, Commit `feat(recording): pick the desk camera in the HUD`. + +### Task E3: App-Lauf E (Controller) + +- [ ] Playwright am HUD-Fenster: Geräte-Einstellungen öffnen, Tischkamera wählen, `recording-settings.json` enthält `camDeskDevice`; zurück auf „Keine“. (Echte Aufnahme mit Kameras: an den Nutzer bzw. computer-use; im Log als nicht abgedeckt.) Results-Log-Zeile, Commit. diff --git a/docs/superpowers/specs/2026-10-05-multi-camera-layouts-design.md b/docs/superpowers/specs/2026-10-05-multi-camera-layouts-design.md new file mode 100644 index 0000000000..54e7d6e1ab --- /dev/null +++ b/docs/superpowers/specs/2026-10-05-multi-camera-layouts-design.md @@ -0,0 +1,289 @@ +# Mehrere Kameras im Bild — Design, Teilprojekte 2 und 3 + +Stand: 2026-10-05 · Branch: `feat/multi-camera-layouts` (= Teilprojekt 1 `feat/multi-camera` + Schreibtisch-Ansicht `feat/desk-view`, PR #989) + +## Ziel + +Teilprojekt 1 nimmt unter Windows bis zu 4 Kameras auf und speichert sie im Projekt +(`asset.cameraTrack` = Kamera 1, `asset.additionalCameraTracks` = Kameras 2–4). Gezeigt wird bisher +nur Kamera 1. Diese beiden Teilprojekte bringen alle Kameras ins Bild: + +- **Teilprojekt 2 — Compositor:** Vorschau und Export zeichnen mehrere Kamera-Ebenen gleichzeitig, + auf allen drei Plattformen (D3D11, Metal, Vulkan). +- **Teilprojekt 3 — Editor:** Layout-Abschnitte auf der Timeline legen fest, welche Kamera wo zu sehen + ist; Kamera-Einstellungen gelten pro Kamera. + +**Erfolg heißt:** Mit einer Aufnahme aus Bildschirm, Tischkamera und Gesichtskamera lassen sich im +Editor Abschnitte bauen wie „Bildschirm + Gesicht klein“, „Tisch voll + Gesicht klein“, „Gesicht | +Tisch nebeneinander“ und „Bildschirm + Gesicht + Tisch klein“; der Wechsel gleitet; Vorschau und Export +zeigen dasselbe. Projekte mit nur einer Kamera und alte Projekte sehen unverändert aus. + +## Entscheidungen (mit dem Nutzer abgestimmt) + +- Gewünscht sind alle vier Ansichten: Tisch groß + Gesicht klein; Kamera im Bild-im-Bild pro Abschnitt + wechseln; zwei Kameras nebeneinander; mehrere Bild-im-Bild gleichzeitig. +- Bedienung über **Layout-Abschnitte** auf der Timeline (wie heute Full Camera), je Abschnitt eine + Vorlage und die Zuordnung Kamera → Platz; außerhalb gilt das Standard-Layout des Projekts. Kein + freies Keyframing. +- Übergang zwischen Abschnitten: **fließend gleiten** (~0,4 s), neu erscheinende Kameras blenden ein, + verschwindende aus. +- Ansatz: **echte Kamera-Ebenen im Compositor** (kein vorab gerendertes Mosaik, keine DOM-Ebene). +- **Hintergrund-Effekte** (Personenmaske: Unschärfe/Entfernen) bleiben vorerst auf Kamera 1; die + übrigen Kamera-Einstellungen gelten für jede Kamera. +- **Perspektivkorrektur per Homographie** für statische Kameras (Tischkamera schräg vom Monitorrand): + einmal kalibrieren, auf jeden Frame anwenden. Kalibrierung über 4 ziehbare Eckpunkte, optional + vorbelegt durch erkannte ArUco-Marker (die App liefert ein druckbares Markerblatt). Ergebnis zeigt + **nur das Rechteck** zwischen den 4 Punkten in einem gewählten Zielformat (A4 hoch/quer, 16:9, 4:3, + quadratisch, frei), optional mit kleinem Rand. +- Reihenfolge: erst Teilprojekt 2, dann 3 — jedes mit eigenem Plan und Review. Der Shader-Teil der + Perspektivkorrektur gehört zu Teilprojekt 2, der Kalibrier-Dialog zu Teilprojekt 3. + +## Nicht in diesem Schritt + +- Hintergrund-Effekte für die Kameras 2–4 (eine Personenmaske pro Kamera). +- Freie Leinwand mit Keyframes; eigene Übergangsarten pro Abschnitt. +- Aufnahme mehrerer Kameras unter macOS/Linux (Teilprojekt 1 ist Windows-only; die Anzeige hier + funktioniert überall, wo die Dateien vorhanden sind). +- Zuordnung des tatsächlich in der HUD gewählten Geräts bei mehreren gleichen Kameras (bekannte + Einschränkung aus Teilprojekt 1; der Editor erlaubt stattdessen, jede Kamera jedem Platz zuzuweisen). +- Mehr als 4 Kameras. + +## 1. Datenmodell + +### Kamera-Einstellungen (pro Kamera) + +Neue optionale Liste im Projekt, Index = Kamera (0 = Kamera 1): + +```ts +interface CameraSettings { + rotation?: 0 | 180; // fehlt = 0 + mirror?: boolean; // fehlt = Projekt-Spiegel (`webcamMirrored`) für Kamera 1, aus für die übrigen + crop?: SceneCrop; // fehlt = ganzes Bild; Kamera 1 nutzt weiterhin den bestehenden Crop + perspective?: CameraPerspective; // fehlt = keine Korrektur +} + +interface CameraPerspective { + // Die 4 Quellpunkte im Kamerabild, normiert 0..1, Reihenfolge oben-links, oben-rechts, + // unten-rechts, unten-links (aus Sicht des fertigen, entzerrten Bildes). + corners: [Point, Point, Point, Point]; + aspect: number; // Zielformat Breite/Höhe, z. B. 297/210 für A4 quer + margin?: number; // Rand um das Rechteck, Anteil der Rechteckgröße, 0..0.2; fehlt = 0 +} +``` + +Gespeichert werden die Punkte, nicht die Matrix: die Matrix (3×3, Ziel → Quelle) wird bei Bedarf aus +den 4 Punktpaaren berechnet (`src/lib/cameraPerspective.ts`, eine Stelle für App und Szene). Mit +gesetzter Perspektive ist der Zuschnitt (`crop`) wirkungslos und im Editor ausgegraut. Ein +degeneriertes Viereck (Punkte kollinear, überschlagen) wird abgelehnt. + +```ts +``` + +Kamera 1 behält ihre heutigen Einstellungen an den heutigen Stellen; die Liste trägt nur +Abweichungen und die Kameras 2–4. + +### Layout-Abschnitte + +```ts +type CameraLayoutTemplate = + | "screen-pip" // Bildschirm + 1–3 kleine Kameras + | "camera-full" // eine Kamera füllt den Rahmen + | "camera-full-pip" // eine Kamera voll + 1–2 kleine Kameras + | "side-by-side"; // zwei Kameras je halber Rahmen, ohne Bildschirm + +interface CameraLayoutSlot { + camera: number; // 0..3, jede Kamera höchstens einmal pro Abschnitt + rect?: NormalizedRect; // vom Nutzer verschoben/skaliert; fehlt = Vorlage +} + +interface CameraLayoutRegion { + id: string; + startMs: number; + endMs: number; + template: CameraLayoutTemplate; + slots: CameraLayoutSlot[]; // Anzahl je Vorlage, Reihenfolge = Plätze der Vorlage + // Schreibtisch-Ansicht aus #989, nur bei camera-full: Drehung/Spiegel dieses Abschnitts + // überschreiben die Kamera-Einstellung; deskLabel steuert das Label der verdeckten Kippung. + rotation?: 0 | 180; + mirror?: "auto" | "on" | "off"; + deskLabel?: false; +} +``` + +Gespeichert in `legacyEditor.cameraLayoutRegions` neben den bestehenden Regionen, mit derselben +Verankerung pro Clip wie `cameraFullscreenRegions` (`migrate.ts`, Kürzel z. B. `"camlayout"`). +Abschnitte überlappen sich nicht (Regel wie bei Full Camera). + +### Bestehende Projekte + +- **Zwei getrennte Listen, nie beide:** Ein Abschnitt `camera-full` mit Kamera 1 ist immer eine + Full-Camera-Region und steht in `cameraFullscreenRegions` (mit `rotation`/`mirror`/`deskLabel`); + jeder andere Abschnitt steht in `cameraLayoutRegions`. Beide Listen sind gleich pro Clip verankert + (`clipId`, `assetId`, `sourceStartSec`, `sourceEndSec`, abgeleitet `startMs`/`endMs`) und folgen + Clip-Änderungen gemeinsam. +- **Beim Laden:** nichts umzuwandeln. Alte Full-Camera-Regionen bleiben, wo sie sind, und ältere + OpenScreen-Versionen zeigen sie weiter; die neuen Vorlagen sehen ältere Versionen nicht (additiv, + kein Schema-Versionssprung). Der Normalizer verwirft eine Zeile `camera-full` mit Kamera 1 in + `cameraLayoutRegions`; der Szenenbau übernimmt eine solche handgeschriebene Zeile nur, wenn keine + Full-Camera-Region dieselbe Spanne abdeckt (nie doppelt). +- **Eine gemeinsame Spur:** Abschnitte beider Listen überlappen sich nicht. Der Editor verhindert das + (Hinzufügen lehnt ab, Verschieben/Skalieren klemmt an Nachbarn beider Listen); der Normalizer + verwirft nur Überlappungen innerhalb von `cameraLayoutRegions`. +- Projekte ohne weitere Kameras und ohne neue Abschnitte bleiben byte-gleich, solange niemand sie + anfasst. + +## 2. Compositor (Teilprojekt 2) + +### Szene + +`SceneClip` bekommt optional die weiteren Kameras des Clips (Pfad und Zeitversatz wie +`webcam_path`/`webcam_offset_sec`). Die Szene bekommt: + +- `cameras`: Einstellungen pro Kamera (Drehung, Spiegel, Crop, Seitenverhältnis) — Kamera 1 bleibt + zusätzlich über die heutigen Felder beschrieben; +- `camera_layout_regions`: pro Abschnitt Start/Ende und die **aufgelösten Zielebenen** — je Ebene + Kamera-Index, Zielrechteck (Anteile des Ausgaberahmens), Eckenradius, Form, Ebenen-Reihenfolge + und ob die Ebene den Rahmen füllt. + +Pro Kamera mit Perspektive sendet die App die fertige 3×3-Matrix (Ziel-UV → Quell-UV) und das +Zielformat mit; das Seitenverhältnis der Kamera-Ebene ist dann das Zielformat. + +Ohne weitere Kameras und ohne Layout-Abschnitte sendet die App diese Felder nicht; der Compositor +verhält sich dann exakt wie heute. + +### Auflösung der Vorlagen (App, TypeScript) + +Eine Stelle in `src/lib/compositeLayout.ts` (neben `computeCompositeLayout`) übersetzt Vorlage + +Plätze + Nutzer-Rechtecke + Seitenverhältnisse in Zielebenen. Das Standard-Layout außerhalb der +Abschnitte ist das heutige (Bildschirm + Kamera 1 nach `webcamLayoutPreset`). `sceneDescription.ts` +projiziert die Abschnitte wie die übrigen Regionen auf die Clips der Timeline. + +### Pro Frame (Rust) + +- `regions.rs`: zur Zeit `t` der aktive Abschnitt und die Übergangsphase (dieselbe Fensterlänge und + Kurve wie das Wachsen der Full Camera). +- `frame_geometry.rs`: eine Liste von Kamera-Ebenen statt des einen Webcam-Rechtecks. Im Übergang + wird jedes Ziel zwischen altem und neuem Layout interpoliert (Rechteck, Radius); eine Kamera, die + nur auf einer Seite vorkommt, blendet über die Deckkraft ein bzw. aus. Kamera 1 ohne Abschnitte + ergibt genau die heutigen Werte (Regressionstest). +- Die Schreibtisch-Ansicht aus #989 (Drehung, Vollbild ohne Crop, verdeckte Kippung) wirkt auf die + Ebene der Kamera im Abschnitt `camera-full` und nur, wenn der Abschnitt die Drehung selbst setzt. + +### Dekodieren + +- Pro Clip ein Decoder je Kamera, mit eigenem Zeitversatz (`live.rs`, Export-Pipelines je Plattform). +- Dekodiert wird nur, was das Layout im aktuellen Frame oder im laufenden/nächsten Übergang zeigt; + ein Decoder öffnet beim ersten Bedarf und bleibt für den Clip offen. +- Fehlt die Datei einer Kamera: die Ebene entfällt, der Rest wird gezeichnet (wie heute die + Ersatz-Kamera für Clips ohne Kamera). + +### Zeichnen + +- Jede Ebene mit dem bestehenden Kamera-Shader und eigenem `LayerCB`; kein neuer Shader. +- **Perspektive:** der Kamera-Shader bekommt eine optionale 3×3-Homographie im `LayerCB` (drei + `float4`-Zeilen, hinten angehängt — Rust, HLSL, WGSL und MSL gleich, vom bestehenden + Größen-/Offset-Test festgehalten). Pro Pixel: `q = H · (u, v, 1)`, Quell-UV = `q.xy / q.z`; außerhalb + des Kamerabildes transparent. Ohne Korrektur ist `H` die Einheitsmatrix und das Ergebnis bit-gleich + zu heute. Die Personenmaske (Kamera 1) wird über dieselbe UV abgetastet und dreht mit. Die drei Backends ersetzen den einen Aufruf durch eine Schleife in der + Ebenen-Reihenfolge: Kameras, die den Rahmen füllen (`camera-full`, `side-by-side`), verdecken den + Bildschirm wie heute die Full Camera; Bild-im-Bild liegt darüber, wie heute Kamera 1. Bildschirm + plus zwei Kameras ist `screen-pip` mit zwei Plätzen. +- Hintergrund-Effekte nur auf Kamera 1 (Personenmaske wie heute). + +### Leistung + +Zielwerte, gemessen und im Plan als Prüfschritt: Vorschau mit Bildschirm + 2 Kameras in 1080p ohne +merkliche Ruckler auf dem Referenzrechner (Windows on ARM, Snapdragon X Elite); Export mit 3 +Kameras nicht mehr als doppelt so lang wie mit einer. Werden die Ziele verfehlt, wird das gemessen +berichtet und nicht stillschweigend hingenommen. + +## 3. Editor (Teilprojekt 3) + +### Perspektive kalibrieren + +- In der Kamera-Liste je Kamera „Perspektive korrigieren…“: Dialog mit Standbild der Kamera (Frame + an der Abspielposition), 4 ziehbare Eckpunkte mit Lupe, Zielformat, Rand, Vorschau des + entzerrten Ergebnisses, „Zurücksetzen“. +- „Marker erkennen“: sucht 4 ArUco-Marker (Wörterbuch 4×4) im Standbild mit einer kleinen + JavaScript-Bibliothek (kein OpenCV) und setzt die Eckpunkte auf die inneren Ecken der Marker; + findet sie nicht alle 4, sagt er das und lässt die Punkte unverändert. +- „Markerblatt drucken“: ein PDF/A4 mit den 4 Markern zum Ausschneiden und einer kurzen Anleitung. +- Ein Undo-Schritt pro bestätigter Kalibrierung. + +### Timeline + +- Die Full-Camera-Zeile wird zur Zeile „Layout“. Taste `C` legt wie bisher einen Abschnitt + `camera-full` mit Kamera 1 an; eine Schaltfläche legt einen Abschnitt mit wählbarer Vorlage an. +- Jeder Abschnitt zeigt ein Symbol der Vorlage und die beteiligten Kameras (z. B. „Tisch + Gesicht“); + gedrehte Abschnitte behalten das Dreh-Symbol aus #989. + +### Inspector + +- Vorlage wählen; je Platz eine Kamera-Auswahl („Kamera 1 · Logitech BRIO“, „Kamera 2 · HD Pro + Webcam C920“ — Name = `label` aus dem Projekt, sonst „Kamera n“). Eine Kamera kann pro Abschnitt + nur einmal vorkommen; beim Wechsel der Vorlage werden die Plätze in Reihenfolge übernommen. +- In der Vorschau verschobene/skalierte Kamerafenster speichern ihr Rechteck in diesem Abschnitt; + „Zurücksetzen“ stellt die Vorlage wieder her. +- Abschnitte `camera-full` behalten die Schalter der Schreibtisch-Ansicht (#989). +- Abschnitt „Kameras“ im Projekt-Inspector: Liste aller Kameras mit Name und kleinem Vorschaubild, + je Kamera Drehung 0°/180°, Spiegeln, Zuschnitt. +- Hat das Projekt nur eine Kamera: Auswahl bietet nur Kamera 1, Vorlagen mit mehreren Kameraplätzen + sind ausgegraut mit Hinweis. + +### Bedienregeln + +- Jede Änderung ist ein Undo-Schritt (`useEditorHistory`), wie bei der Schreibtisch-Ansicht. +- Texte in allen Locales (`npm run i18n:check`). + +## 4. Fehlerfälle + +- Kameradatei fehlt oder lässt sich nicht öffnen: ihre Plätze bleiben leer, der Abschnitt zeigt den + Rest, der Editor zeigt einen Hinweis mit dem Kameranamen. +- Abschnitt verweist auf eine Kamera, die das Projekt nicht hat (z. B. nach Relink): Platz leer, + Hinweis, Projekt öffnet. +- Unbekannte Vorlage aus einer späteren Version: beim Laden verworfen, das Projekt öffnet. +- Kamera ist kürzer als der Abschnitt (z. B. vorzeitig beendet): nach ihrem Ende bleibt der Platz leer. + +## 5. Tests und Abnahme + +- **Perspektive:** Homographie aus 4 Punktpaaren bildet die Ecken exakt aufeinander ab; degenerierte + Vierecke abgelehnt; Einheitsmatrix lässt den Shader-Ausgang bit-gleich; gemessener Export mit + einem schräg aufgenommenen Schachbrett ergibt rechte Winkel und gleiche Feldgrößen; Marker-Erkennung + an einem gerenderten Testbild. Echter Lauf: Tischkamera mit A4-Blatt und Markerblatt. +- **Rust:** Ebenenliste aus Szene und Zeit; Übergang interpoliert Rechteck/Radius und blendet + ein-/ausfallende Kameras; ohne Abschnitte identisch zu heute; Schreibtisch-Ansicht auf der richtigen + Ebene; naga-Validierung unverändert grün. +- **TS:** Auflösung jeder Vorlage (inkl. Nutzer-Rechtecke, Seitenverhältnisse); Laden alter + Full-Camera-Regionen; Schreiben der Rückwärtskompatibilität; Kamera-Einstellungen; Undo; + Inspector (jsdom). +- **Export, gemessen:** Headless-Export mit synthetischen Kameras unterschiedlicher Farbe — Position + und Größe jeder Ebene im Steady-State und im Übergang aus den Pixeln gemessen; Vorschau-Frame und + Export-Frame an derselben Stelle verglichen. +- **Echter Lauf (Nutzer mit 2 Brios, C920, eingebauter Kamera):** Abschnitte „Tisch voll + Gesicht + klein“, „Bildschirm + 2 PiP“, „nebeneinander“, gleitender Wechsel, MP4-Export; Leistung der Vorschau + mit mehreren Kameras. Ergebnis ins Results-Log von + `technical-documentation/testing/manual-e2e-checklist.md`. + +## 6. Verbindliche Vorgaben für Teilprojekt 3 (aus den Reviews von Teilprojekt 2) + +- **Speichern:** Zwei getrennte Listen. Full Camera von Kamera 1 (`camera-full` mit Kamera 1) steht + nur in `cameraFullscreenRegions`, jeder andere Abschnitt nur in `cameraLayoutRegions`; der Szenenbau + liest immer beide. Die frühere Regel „liegt `cameraLayoutRegions` vor, wird die alte Liste + ignoriert“ gilt nicht mehr. +- **Kamera-1-Einstellungen:** Drehung, Spiegeln und Zuschnitt von Kamera 1 stehen weiter in den + bestehenden Projektfeldern (`webcamMirrored`, Schreibtisch-Abschnitte, `webcamCropRegion`), nicht in + `cameraSettings[0]`; für Kamera 1 wertet der Compositor aus `cameraSettings[0]` nur die Perspektive. +- **Schreibtisch-Schalter** nur bei `camera-full` mit Kamera 1 anbieten (der Normalizer verwirft sie + sonst). +- **Vorlagen bei Block-Layouts** (`dual-frame`, `vertical-stack`): Bild-im-Bild-Vorlagen ausgrauen oder + den Bildschirm neu anordnen — sonst bleibt eine Hälfte leer. +- **Form:** Ein runder/quadratischer Platz braucht ein quadratisches Rechteck und Radius 0,5; beim + Verschieben/Skalieren im Editor das Seitenverhältnis halten (das Backend liest die Form nicht). +- **Fehlende Kamera:** Fehlt eine Kamera schon im Projekt (Spur fehlt/unsichtbar), entfällt ihr Platz + und ein Abschnitt ohne Plätze wird verworfen; lässt sich nur die Datei nicht öffnen, bleibt der + Abschnitt und die Kamera fehlt im Bild. Der Hinweis im Editor muss das jeweilige Ergebnis nennen. +- **Nahtstelle Full Camera ↔ Layout-Abschnitt:** gleitet direkt; die Unschärfe eines gedrehten + Abschnitts endet dort mit ihrer eigenen Blende (sichtbarer Schnitt im Vollbild) — im Editor + gegebenenfalls einen Hinweis geben oder die Kippung an solche Nahtstellen nicht legen. +- **Zuordnung gleicher Kameras** folgt der Windows-Reihenfolge (Teilprojekt 1): jede Kamera muss jedem + Platz frei zuweisbar sein. diff --git a/docs/superpowers/specs/2026-10-06-desk-lane-and-free-crop-design.md b/docs/superpowers/specs/2026-10-06-desk-lane-and-free-crop-design.md new file mode 100644 index 0000000000..41f26e644f --- /dev/null +++ b/docs/superpowers/specs/2026-10-06-desk-lane-and-free-crop-design.md @@ -0,0 +1,180 @@ +# Tischkamera-Spur (D) und freier Ausschnitt nach der Entzerrung — Design + +Stand: 2026-10-06 · Branch: `feat/multi-camera-layouts` (Worktree `C:\osc-cams`), danach im Squash von #1026 + +## Ziel + +Zwei Schwächen aus dem ersten echten Lauf am Tisch des Nutzers (Brio als Tischkamera, Kamera 2): + +1. **Verzerrtes Bild nach „Marker erkennen“.** Der Dialog zieht das Viereck zwischen den vier inneren + Marker-Ecken auf ein Rechteck. Das ist nur verzerrungsfrei, wenn die Marker auf dem Tisch selbst + ein Rechteck bilden. Beim Nutzer lagen sie frei verteilt (gemessen: Seiten 46 / 32 / 98 / 71 cm, + Winkel 109° / 126° / 79° / 46°), das Ergebnis war entsprechend verzerrt. +2. **Keine einfache Tischansicht mit eigener Kamera.** „Full Camera“ (Taste C) kennt nur Kamera 1; + eine andere Kamera im Vollbild geht nur über „Layout hinzufügen → Kamera voll“, was man nicht findet. + +**Erfolg heißt:** Marker frei auf den Tisch legen, „Marker erkennen“, einen Ausschnitt in 16:9 (oder +4:3, 9:16, 1:1, frei) auf das entzerrte Tischbild ziehen — das Ergebnis ist rechtwinklig. Eine Kamera +einmal als Tischkamera markieren, dann legt die Taste D einen Abschnitt an, in dem die Tischkamera +weich ins Vollbild überblendet. Wer keine Entzerrung will, bekommt mit D die Tischkamera so, wie sie +aufnimmt. Alte Projekte sehen unverändert aus. + +## Entscheidungen (mit dem Nutzer abgestimmt) + +- **Entzerrung und Ausschnitt sind getrennt.** Die Marker liefern nur die Entzerrung der Tischebene; + der Ausschnitt wird danach frei auf dem entzerrten Bild gewählt, Formate 4:3, 16:9, 9:16, 1:1, frei. +- **Die Entzerrung ist optional.** Eine Tischkamera ohne Perspektive ist ein gültiger Aufbau. +- **Eine Kamera wird im Projekt als Tischkamera festgelegt**; D-Abschnitte zeigen immer diese. +- **Eigene Timeline-Zeile, Taste D.** D-Abschnitte schließen sich mit C- und Layout-Abschnitten aus + (keine Überlappung), dürfen aber direkt anschließen. +- **Übergang: Überblenden**, mit den Zeiten und der Kurve von C. Folgt D auf C, blendet das Gesicht im + Vollbild in den Tisch im Vollbild über. +- **Bauweise:** eigene schlanke Abschnittsliste plus Projekteinstellung; der Szenenaufbau übersetzt + D-Abschnitte in „Kamera voll“-Ebenen. Kein neuer Abschnittstyp im Compositor. +- **Reihenfolge:** erst Teil A (Dialog), dann Teil B (D-Spur). + +## Nicht in diesem Schritt + +- Board-Markerblatt (viele Marker auf A4) und Modus „ganzes Sichtfeld“. +- Kalibrierung vor der Aufnahme, Kamerarollen im HUD, Zuordnung über Seriennummer. +- Übergang „aus dem PiP wachsen“ (lässt sich später ohne Datenmodelländerung nachrüsten). +- Korrektur der Objektivverzeichnung. +- Änderungen am Compositor (Rust, Shader). + +## Teil A — Kalibrierdialog: Entzerrung, dann freier Ausschnitt + +### Datenmodell + +Unverändert: `CameraPerspective { corners, aspect, margin? }` in `legacyEditor.cameraSettings[i]`. +Neu ist nur, was in `corners` steht: die Bildecken eines **auf der Ebene rechtwinkligen** Rechtecks +mit Seitenverhältnis `aspect`. Compositor, Szene und Export bleiben unverändert; alte Projekte laden +wie bisher. + +Beim erneuten Öffnen wird die Ebene aus den gespeicherten Daten zurückgewonnen: die Homographie vom +Rechteck `aspect × 1` auf `corners` *ist* eine metrische Entzerrung. Es muss also nichts zusätzlich +gespeichert werden. (Für eine alte, von Hand gezogene Perspektive gilt dasselbe — sie wird als das +Rechteck gelesen, als das sie gespeichert wurde.) + +### Geometrie (`src/lib/planeMeasure.ts`, ausgebaut) + +- `planeFromMarkers(squares, sideMm)` → Homographie Bild → Ebene (Millimeter), aus den gemittelten + zirkulären Punkten der Marker-Quadrate (heute intern in `measureOnPlane`), mit festgelegter + Ausrichtung: x-Achse waagerecht zur Kamera (die Bildwaagerechte, auf die Ebene übertragen in + der Mitte der Marker), um die Vierteldrehung gedreht, die Marker 0 → Marker 1 am nächsten + kommt — die Marker bestimmen, wo oben ist (Marker 0 oben links, Marker 3 darunter), die Kamera + hält den Rahmen gerade, auch wenn die Marker schief liegen. Ursprung in der Mitte der vier + Marker. +- `planeFromPerspective(perspective)` → dieselbe Homographie aus gespeicherten `corners` + `aspect`. +- `rectToCorners(plane, rect)` → die vier Bildpunkte (normiert 0..1) eines Rechtecks auf der Ebene. +- `measureOnPlane` bleibt als Messfunktion erhalten (Größenangabe im Dialog). + +### Bedienung + +- **Ohne Ebene** (keine Perspektive, keine Marker): wie heute — Standbild mit vier Griffen. +- **„Marker erkennen“**: setzt die Ebene. Die große Fläche zeigt danach das **entzerrte** Tischbild, + auf das Kamerabild begrenzt (Teile außerhalb des Kamerabilds bleiben leer; die Darstellung wird auf + das Mehrfache der Marker-Ausdehnung begrenzt, damit ein Horizont im Bild sie nicht sprengt). +- Darauf liegt ein **Ausschnitt-Rechteck** mit Eck- und Kantengriffen (Bedienung wie der bestehende + Zuschnitt-Modus), Format 4:3 / 16:9 / 9:16 / 1:1 / frei; Vorgabe 16:9, mittig, größtmöglich + innerhalb der Marker. Bei festem Format bleibt das Verhältnis beim Ziehen erhalten. +- Die Größenangabe „ca. 94 × 58 cm“ bezieht sich auf den Ausschnitt. +- „Vier Ecken von Hand“: Umschalter zurück zum heutigen Modus (z. B. für eine Vorlage ohne Marker). +- **Übernehmen** speichert `corners = rectToCorners(plane, rect)`, `aspect = rect.w / rect.h`. + **Zurücksetzen** entfernt die Perspektive. +- Der Regler „Rand“ gilt nur noch im Modus mit vier Griffen; im Ausschnitt-Modus ersetzt das + Rechteck ihn (gespeichert wird dann kein `margin`). + +### Fehlerfälle + +- Nicht alle vier Marker: Meldung wie heute, Modus bleibt unverändert. +- Ebene nicht bestimmbar (entartete Marker): Meldung „Ebene konnte nicht bestimmt werden“; vier Griffe. +- Ausschnitt reicht über das Kamerabild hinaus: erlaubt, der Rand wird schwarz (wie heute bei einer + Perspektive mit Rand); der Dialog zeigt einen Hinweis. + +## Teil B — D-Spur mit Tischkamera + +### Datenmodell (`legacyEditor`, optional, keine Schema-Version) + +```ts +/** Index der Tischkamera (0 = Kamera 1). Fehlt = automatisch (siehe unten). */ +deskCamera?: number; + +/** Abschnitte, in denen die Tischkamera das Bild füllt. */ +deskRegions?: DeskRegion[]; + +interface DeskRegion { + id: string; + startMs: number; + endMs: number; + /** Nur `false` wird gespeichert: blendet das Label „Schreibtischmodus“ aus. */ + deskLabel?: false; +} +``` + +**Tischkamera auflösen** (`resolveDeskCamera`, eine Stelle für Editor und Szene): die festgelegte +Kamera, sofern im Projekt vorhanden; sonst die erste Kamera mit Perspektive außer Kamera 1; sonst +Kamera 2; sonst keine. Drehung, Spiegelung, Zuschnitt und Perspektive kommen aus deren +`cameraSettings`, nicht aus dem Abschnitt. + +### Timeline und Bedienung + +- **Taste D** (neue Aktion `addDeskSection` in `src/lib/shortcuts.ts`, Standard `d`, frei belegbar; + kollidiert nicht mit Strg+D) und ein Knopf in der Werkzeugleiste neben Full Camera, nur sichtbar + mit mindestens zwei Kameras. +- **Eigene Zeile** „Tisch“ unter der Kamera-Zeile, Pillen wie bei Full Camera, ziehen und + Länge ändern wie dort. +- **Ausschluss:** `addCameraFullscreen`, `addCameraLayout` und das neue `addDeskSection` lehnen + Überschneidungen mit allen drei Listen ab (`"occupied"`); Ziehen und Längenändern werden an den + Nachbarn begrenzt wie heute innerhalb der Kamera-Zeile. Berühren ist erlaubt. +- Ohne auflösbare Tischkamera: kein Abschnitt, Meldung „Keine Tischkamera — lege eine im Abschnitt + Kameras fest“ (über `showCameraSectionOutcome`). +- Ein Schreibvorgang = ein Undo-Schritt, wie bei allen Abschnitten. + +### Inspector + +- **D-Abschnitt:** Name der Tischkamera mit Verweis „ändern“ (springt zum Abschnitt „Kameras“), + Schalter „Label anzeigen“, Löschen. +- **Abschnitt „Kameras“:** pro Kamera ein Auswahlpunkt „Tischkamera“ (genau eine; erneutes Klicken + hebt die Festlegung auf → automatisch). Kamera 1 ist wählbar; dann zeigt D Kamera 1 im Vollbild mit + deren Drehung — der Ein-Kamera-Aufbau mit geschwenkter Kamera bleibt aber der C-Abschnitt mit + Desk-Ansicht. + +### Szene (`src/native/sceneDescription.ts`) + +`buildSceneDescription` übersetzt jeden D-Abschnitt vor der Projektion: + +- Tischkamera = Kamera *k* ≥ 2 → `CameraLayoutRegion { template: "camera-full", slots: [{ camera: k − 1 }] }` + in die Liste der Layout-Abschnitte. Der Compositor gleitet bzw. blendet dort bereits wie + gewünscht: eine Kamera, die nur in einem der beiden Zustände vorkommt, blendet über. +- Tischkamera = Kamera 1 → `CameraFullscreenRegion` mit Drehung/Spiegelung aus deren Einstellungen. +- Label: `deskLabelTextRegions` bekommt D-Abschnitte mit `deskLabel !== false` dazu (gleiche + Animation `deskCover` für das Label; die Kipp-Abdeckung selbst entfällt, weil sich nichts dreht). +- Ohne auflösbare Tischkamera werden D-Abschnitte übersprungen (nicht gezeichnet, nicht gelöscht). + +Vorschau, Export und CLI-Export laufen alle über die Szene und brauchen keine eigene Änderung. Die +DOM-Vorschau (`PreviewCanvas`) ist im Plan zu prüfen, ob sie Layout-Abschnitte schon zeigt. + +### Fehlerfälle + +- Festgelegte Tischkamera fehlt im Projekt (Datei verloren): automatisch auflösen; im Inspector + „Tischkamera nicht verfügbar“. +- Alte Projekte ohne `deskRegions`/`deskCamera`: unverändert. +- Kopieren/Einfügen von D-Abschnitten verhält sich wie bei Full-Camera-Abschnitten (gleicher Weg in + der Zwischenablage; können diese nicht kopiert werden, D auch nicht). + +## Tests und Abnahme + +- **Geometrie (Vitest, node):** gerenderte Testbilder mit frei verteilten Markern (kein Rechteck) → + `planeFromMarkers` → ein Rechteck auf der Ebene → `rectToCorners` → zurückprojiziert ist es + rechtwinklig (Winkel 90° ± 1°) und hat das gewählte Verhältnis; `planeFromPerspective` gibt für + gespeicherte Daten dieselbe Ebene zurück (Rundreise). +- **Dialog (jsdom):** Formate, Ziehen mit festem Verhältnis, Übernehmen speichert Ecken + Verhältnis, + Wiederöffnen stellt den Ausschnitt wieder her, Umschalten auf vier Griffe. +- **Store:** D anlegen, Ausschluss in allen Richtungen, Berühren erlaubt, Undo, Speichern/Laden, + `resolveDeskCamera` in allen Fällen. +- **Szene:** D → `camera-full` der Tischkamera bzw. `CameraFullscreenRegion` für Kamera 1; Label; + ohne Tischkamera übersprungen; alte Projekte erzeugen eine identische Szene. +- **Shortcuts/i18n:** D in der Liste, keine Kollision; alle 15 Sprachen (`npm run i18n:check`). +- **Echter Lauf** am Tisch des Nutzers: Marker frei verteilt → 16:9-Ausschnitt rechtwinklig; + D-Abschnitt nach C-Abschnitt blendet über; Export gegen Vorschau; Eintrag im Results-Log von + `technical-documentation/testing/manual-e2e-checklist.md`. diff --git a/docs/superpowers/specs/2026-10-07-main-camera-pip-toggles-hud-desk-design.md b/docs/superpowers/specs/2026-10-07-main-camera-pip-toggles-hud-desk-design.md new file mode 100644 index 0000000000..3cc3305a24 --- /dev/null +++ b/docs/superpowers/specs/2026-10-07-main-camera-pip-toggles-hud-desk-design.md @@ -0,0 +1,126 @@ +# Hauptkamera, Kamera-Schalter für „Bildschirm + Kamera“, Tischkamera im HUD — Design + +Stand: 2026-10-07 · Branch: `feat/multi-camera-layouts` (Worktree `C:\osc-cams`), danach im Squash von #1026 + +## Ziel + +Drei Erweiterungen der Mehrkamera-Bearbeitung, mit dem Nutzer abgestimmt: + +- **C — Hauptkamera:** Im Editor wählen, welche Kamera die Rolle von Kamera 1 übernimmt. Sie erscheint im + Preset (Bild-im-Bild, Dual Frame, Vertical Stack) und in Full-Camera-Abschnitten (C); die Kamera-Effekte + gelten für sie. +- **D — „Bildschirm + Kamera“ mit Schaltern:** Im Inspector eines solchen Abschnitts pro Kamera an/aus statt + einer Auswahlliste pro Platz. +- **E — Tischkamera im HUD:** In den Geräte-Einstellungen des HUD die Tischkamera als Vorgabe für neue + Aufnahmen wählen; das neue Projekt übernimmt sie. + +**Erfolg heißt:** Mit Gesicht auf Kamera 2 und Tisch auf Kamera 1 lässt sich im Editor Kamera 2 als +Hauptkamera wählen; Preset, C-Abschnitte, Effekte, Vorschau und Export zeigen Kamera 2, ohne dass sich +Layout-Abschnitte oder die Tischkamera verschieben. Ein „Bildschirm + Kamera“-Abschnitt bekommt Fenster per +Schalter. Eine neue Aufnahme mit im HUD gewählter Tischkamera öffnet mit dieser Tischkamera. Projekte ohne +die neuen Felder sehen unverändert aus. + +## Entscheidungen (mit dem Nutzer abgestimmt) + +- Die Preset-Auswahl ist eine **Hauptkamera-Rolle** (nicht „nur fürs Preset“): sie gilt für Preset, + C-Abschnitte und Effekte. +- Umsetzung der Rolle **nur im Szenenaufbau** durch Tauschen der Kameras; keine Änderung an Compositor, + Backends oder Shadern. +- „Bildschirm + Kamera“: Schalter pro Kamera, mindestens 1, höchstens 3, Fenster in Kamera-Reihenfolge. +- HUD-Tischkamera kommt **zusätzlich** zur Tischkamera im Editor; das Projekt übernimmt sie nur, wenn es + noch keine hat. +- Reihenfolge: D, dann C, dann E; jeder Teil mit eigenem Lauf in der echten App (Playwright). + +## Nicht in diesem Schritt + +- Effekte (Unschärfe, Freistellen, eigener Hintergrund) für mehr als eine Kamera gleichzeitig. +- Zuordnung baugleicher Kameras über die Seriennummer (bekannte Schwäche aus #1025 bleibt). +- Hauptkamera im HUD. +- Compositor-Befund „außerhalb des Kamerabilds Hintergrund statt Schwarz“ (eigener Folgepunkt). + +## Teil D — „Bildschirm + Kamera“ mit Schaltern + +**Bedienung:** Im Inspector eines Layout-Abschnitts mit Vorlage `screen-pip` ersetzt eine Liste „Kameras“ +die Auswahllisten pro Platz: pro Kamera des Projekts ein Schalter (Name wie im Abschnitt „Kameras“). +Eingeschaltet = die Kamera hat ein Fenster. Nicht verfügbare Kameras sind ausgegraut (außer sie sind +eingeschaltet — dann lassen sie sich ausschalten). Der letzte eingeschaltete Schalter ist gesperrt +(Hinweis „Mindestens eine Kamera“); bei drei eingeschalteten sind die übrigen gesperrt (Hinweis „Höchstens +drei Fenster“). Andere Vorlagen behalten ihre Auswahllisten. + +**Daten:** unverändert `slots: CameraLayoutSlot[]`. Ein Schalter ändert die Liste: einschalten fügt +`{ camera }` an der Stelle ein, die die Kamera-Reihenfolge verlangt; ausschalten entfernt den Platz dieser +Kamera. Ein Platz mit eigenem `rect` behält es, solange seine Kamera an bleibt. Ein Schreibvorgang = ein +Undo-Schritt. + +**Store:** neue Funktion `setLayoutSectionCameras(handle, cameras: number[])` (Reihenfolge = Kamera-Index; +übernimmt vorhandene `rect` der bleibenden Kameras; verweigert < min / > max der Vorlage). + +## Teil C — Hauptkamera + +**Daten:** `legacyEditor.mainCamera?: number` (Index, 0 = Kamera 1; fehlt = 0). Indizes in +Layout-Abschnitten, `cameraSettings` und `deskCamera` meinen weiterhin **die Kamera selbst** (das Gerät), +nicht die Rolle. + +**Auflösen:** `resolveMainCamera({ mainCamera, cameraCount, available })` → gewählte Kamera, wenn im Projekt +vorhanden und verfügbar (gleiche Verfügbarkeitsregel wie die Tischkamera); sonst 0. Editor und Szene +benutzen dieselbe Funktion. + +**Einstellungen:** +- Die Regler des Kamera-Layout-Bereichs (Preset, Form, Größe, Position, Rundung, Zuschnitt, Spiegeln, + Effekte) gelten für die **Hauptkamera**. +- Jede andere Kamera — auch Kamera 1, wenn sie nicht Hauptkamera ist — nimmt Drehung, Spiegeln und + Zuschnitt aus ihrem Eintrag in `cameraSettings` (Abschnitt „Kameras“). +- Die Perspektive bleibt immer bei ihrer Kamera (`cameraSettings[i].perspective`), auch als Hauptkamera. +- Der Abschnitt „Kameras“ zeigt bei der Hauptkamera den Hinweis, dass ihre Regler oben stehen (heute fest + bei Kamera 1). + +**Szene (`buildSceneDescription`):** Ist die Hauptkamera *m* ≠ 0, werden Kamera 0 und *m* vertauscht, +bevor die Szene entsteht: +- pro Clip: `webcamPath`/`webcamOffsetSec` ← Kamera *m*; in `additionalCameras` steht an *m*s Platz die + bisherige Kamera 1; +- `cameras[]`: der Eintrag für Index 0 trägt die Perspektive von *m*; der Eintrag für Index *m* trägt + Drehung/Spiegeln/Zuschnitt/Perspektive der echten Kamera 1 aus `cameraSettings[0]`; +- Quellmaße für das Preset (`webcamSourceSizeOf`) aus Kamera *m*; +- Layout-Abschnitte und Tischkamera-Zeilen: Kamera-Index 0 ↔ *m* umgeschrieben; +- Tischkamera = Hauptkamera → der D-Abschnitt wird zur Full-Camera-Zeile (heute: Tischkamera = Kamera 1). +Ohne `mainCamera` (oder = 0) ist die Szene unverändert. + +**Bedienung:** Im Kamera-Layout-Bereich neben „Preset“ eine Auswahl „Hauptkamera“ (ab zwei Kameras +sichtbar), Einträge mit Kameranamen. Ist die gewählte Kamera nicht verfügbar: Hinweis „nicht verfügbar — +verwendet wird Kamera 1“. Ein Schreibvorgang = ein Undo-Schritt, über die Schreib-Warteschlange der Shell. + +## Teil E — Tischkamera im HUD + +**Bedienung:** In den Geräte-Einstellungen des HUD unter „Zusätzliche Kameras“ eine Auswahl „Tischkamera“: +„Keine“ oder eine der Kameras, die aufgenommen werden (Kamera 1 und die angehakten zusätzlichen). Gleiche +Sperrregel wie die zusätzlichen Kameras (nur mit Kamera an und nativer Windows-Aufnahme, ab zwei Kameras). +Gespeichert in den Aufnahme-Einstellungen wie `camAdditionalDevices` (Gerät per id/Name); eine gewählte +Kamera, die nicht mehr aufgenommen wird, gilt als „Keine“. + +**Aufnahme:** Beim Aufnehmen wird aus der Wahl der Index in Aufnahme-Reihenfolge (0 = Kamera 1, *k* = +zusätzliche Kamera *k*, wie die Dateien `-webcam.mp4`, `-webcam-2.mp4` …) und als `deskCamera` in die +Sitzungsdatei geschrieben (neben `additionalWebcams`, optional). + +**Projekt:** Beim Verknüpfen der Aufnahme mit dem Projekt (`projectStore.addAsset`) wird +`legacyEditor.deskCamera` gesetzt — nur, wenn das Projekt noch keine Festlegung hat und der Index zu einer +verknüpften Kamera gehört. + +## Fehlerfälle + +- Hauptkamera/Tischkamera nicht (mehr) verfügbar: automatisch auf Kamera 1 bzw. die automatische + Tischkamera; Hinweis im Editor. +- „Bildschirm + Kamera“ mit nur einer verfügbaren Kamera: Schalter dieser Kamera gesperrt an. +- Ältere Aufnahmen ohne `deskCamera` in der Sitzungsdatei: keine Festlegung, automatische Wahl. + +## Tests und Abnahme + +- **D:** Store (`setLayoutSectionCameras`: einfügen in Reihenfolge, entfernen, `rect` behalten, min/max, + Undo); Inspector (Schalter, Sperren, Hinweise). +- **C:** `resolveMainCamera`; Szene (Tausch für *m* = 1, 2; Layout- und Tischkamera-Indizes umgeschrieben; + Tischkamera = Hauptkamera; ohne `mainCamera` identisch zu vorher mit wörtlichen Erwartungswerten); Editor + (Auswahl, Hinweis, Abschnitt „Kameras“ zeigt die Regler-Notiz bei der Hauptkamera). +- **E:** HUD-Liste (Auswahl, Sperre, „Keine“); Aufnahme-Anfrage/Sitzungsdatei (`deskCamera`-Index); + `addAsset` setzt `deskCamera` nur ohne vorhandene Festlegung. +- **Echte App (Playwright, Controller):** D und C auf einer Kopie des Zwei-Kamera-Projekts des Nutzers mit + Screenshots und Export gegen Vorschau; E so weit der HUD erreichbar ist (Auswahl speichern). Eine echte + Aufnahme mit Kameras macht der Nutzer oder computer-use. Zeilen im Results-Log. diff --git a/electron/ai-edition/agent-tools.test.ts b/electron/ai-edition/agent-tools.test.ts index fc8ba40857..758d5368f3 100644 --- a/electron/ai-edition/agent-tools.test.ts +++ b/electron/ai-edition/agent-tools.test.ts @@ -1211,6 +1211,161 @@ describe("a full-camera region needs a camera", () => { }); }); +describe("full-camera regions and layout sections share one lane", () => { + /** A side-by-side section of cameras 1 and 2 over 10–15 s of clip_1. */ + function withLayoutSection(document: AxcutDocument): AxcutDocument { + return documentSchema.parse({ + ...document, + legacyEditor: { + ...((document.legacyEditor as Record) ?? {}), + cameraLayoutRegions: [ + { + id: "camlayout_1", + startMs: 10_000, + endMs: 15_000, + clipId: "clip_1", + assetId: "asset_1", + sourceStartSec: 10, + sourceEndSec: 15, + template: "side-by-side", + slots: [{ camera: 0 }, { camera: 1 }], + }, + ], + }, + }); + } + + it("exposes layout sections read-only in the snapshot", () => { + const document = withLayoutSection(withCameraTrack(fixtureDocument())); + const snapshot = JSON.parse(executeAgentTool(document, "getCurrentDocument", "").resultJson); + expect(snapshot.cameraLayoutRegions).toEqual([ + { id: "camlayout_1", startSec: 10, endSec: 15, template: "side-by-side", cameras: [1, 2] }, + ]); + expect(snapshot.cameraLayoutNote).toMatch(/read-only/); + expect(snapshot.timeBaseNote).toMatch(/cameraLayoutRegions/); + }); + + it("an empty project reports no layout sections", () => { + const snapshot = JSON.parse( + executeAgentTool(fixtureDocument(), "getCurrentDocument", "").resultJson, + ); + expect(snapshot.cameraLayoutRegions).toEqual([]); + }); + + it("refuses to add a full-camera region over a layout section", () => { + const document = withLayoutSection(withCameraTrack(fixtureDocument())); + const result = executeAgentTool( + document, + "addCameraFullscreen", + JSON.stringify({ startSec: 12, endSec: 18 }), + ); + expect(result.ok).toBe(false); + expect(result.document).toBeUndefined(); + const error = JSON.parse(result.resultJson).error as string; + expect(error).toMatch(/camlayout_1/); + expect(error).toMatch(/side-by-side, 10–15 s/); + expect(error).toMatch(/may not overlap/); + }); + + it("allows a full-camera region that only touches a layout section", () => { + const document = withLayoutSection(withCameraTrack(fixtureDocument())); + const result = executeAgentTool( + document, + "addCameraFullscreen", + JSON.stringify({ startSec: 5, endSec: 10 }), + ); + expect(result.ok).toBe(true); + }); + + it("refuses to move a full-camera region onto a layout section", () => { + const document = withLayoutSection(withCameraTrack(fixtureDocument())); + const added = executeAgentTool( + document, + "addCameraFullscreen", + JSON.stringify({ startSec: 0, endSec: 5 }), + ); + expect(added.ok).toBe(true); + const regionId = ( + (added.document?.legacyEditor as Record).cameraFullscreenRegions as Array<{ + id: string; + }> + )[0].id; + const moved = executeAgentTool( + added.document as AxcutDocument, + "setCameraFullscreen", + JSON.stringify({ cameraFullscreenId: regionId, startSec: 8, endSec: 11 }), + ); + expect(moved.ok).toBe(false); + expect(moved.document).toBeUndefined(); + expect(JSON.parse(moved.resultJson).error).toMatch(/camlayout_1/); + }); +}); + +describe("the shared camera lane while another camera is the main one", () => { + /** Camera 2 is the main camera; a camera-full section of `camera` covers 10–15 s of clip_1. */ + function withCameraFullRow(document: AxcutDocument, camera: number): AxcutDocument { + return documentSchema.parse({ + ...document, + legacyEditor: { + ...((document.legacyEditor as Record) ?? {}), + mainCamera: 1, + cameraLayoutRegions: [ + { + id: "camlayout_1", + startMs: 10_000, + endMs: 15_000, + clipId: "clip_1", + assetId: "asset_1", + sourceStartSec: 10, + sourceEndSec: 15, + template: "camera-full", + slots: [{ camera }], + }, + ], + }, + }); + } + + it("refuses a full-camera region over camera 1's camera-full section", () => { + const document = withCameraFullRow(withCameraTrack(fixtureDocument()), 0); + const result = executeAgentTool( + document, + "addCameraFullscreen", + JSON.stringify({ startSec: 12, endSec: 18 }), + ); + expect(result.ok).toBe(false); + expect(result.document).toBeUndefined(); + expect(JSON.parse(result.resultJson).error).toMatch(/camlayout_1/); + }); + + it("refuses a full-camera region over a camera-full row stored before its camera became main", () => { + const document = withCameraFullRow(withCameraTrack(fixtureDocument()), 1); + const result = executeAgentTool( + document, + "addCameraFullscreen", + JSON.stringify({ startSec: 12, endSec: 18 }), + ); + expect(result.ok).toBe(false); + expect(JSON.parse(result.resultJson).error).toMatch(/camlayout_1/); + }); + + it("lists every stored layout row, as the editor lane shows it", () => { + for (const camera of [0, 1]) { + const document = withCameraFullRow(withCameraTrack(fixtureDocument()), camera); + const snapshot = JSON.parse(executeAgentTool(document, "getCurrentDocument", "").resultJson); + expect(snapshot.cameraLayoutRegions).toEqual([ + { + id: "camlayout_1", + startSec: 10, + endSec: 15, + template: "camera-full", + cameras: [camera + 1], + }, + ]); + } + }); +}); + // ── D-DESTRUCT ────────────────────────────────────────────────────────────── // // "Swap the two clips: put the demo first." There was no tool for it — while diff --git a/electron/ai-edition/agent-tools.ts b/electron/ai-edition/agent-tools.ts index 3ffc9b2153..7b99daf22e 100644 --- a/electron/ai-edition/agent-tools.ts +++ b/electron/ai-edition/agent-tools.ts @@ -57,6 +57,8 @@ import { effectiveZoomScale, ZOOM_DEPTH_LEGEND, } from "../../src/lib/ai-edition/timeline/zoom-scale"; +import { cameraSectionsOverlapping, storedCameraLayoutRows } from "../../src/lib/cameraLayouts"; +import { normalizeDeskRegions } from "../../src/lib/deskSections"; import { SETTING_BOUNDS } from "../../src/lib/projectDefaults"; export interface AgentToolExecution { @@ -97,6 +99,14 @@ function modifierIds(document: AxcutDocument, kind: ModifierKind): string[] { return ((legacy.cameraFullscreenRegions as Array<{ id: string }> | undefined) ?? []).map( (region) => region.id, ); + case "cameraLayout": + return ((legacy.cameraLayoutRegions as Array<{ id: string }> | undefined) ?? []).map( + (region) => region.id, + ); + case "desk": + return ((legacy.deskRegions as Array<{ id: string }> | undefined) ?? []).map( + (region) => region.id, + ); case "audio": // Fragments of one user-visible track share `trackId` and render as a single pill, // so the id that disappears is the group key once, not one per fragment. @@ -265,6 +275,63 @@ function noCameraUnderSpan( ); } +/** + * The editor's file the agent reads layout sections from, coalesced to the pills the ruler draws. + * Every valid stored row, with no role filter: the editor lane shows exactly these, and a + * camera-full row of the main camera still occupies the lane, so the overlap check must see it. + */ +function cameraLayoutPillsForAgent(document: AxcutDocument) { + const legacy = document.legacyEditor as Record | null; + return coalesceForAgent(storedCameraLayoutRows(legacy?.cameraLayoutRegions)); +} + +/** The editor's desk sections, coalesced to the pills the ruler draws. */ +function deskPillsForAgent(document: AxcutDocument) { + const legacy = document.legacyEditor as Record | null; + return coalesceForAgent(normalizeDeskRegions(legacy?.deskRegions)); +} + +/** + * Refuse a full-camera region that would land on a camera layout or desk section. Full-camera + * regions never overlap either (the editor refuses the same add as "occupied"); two + * overlapping sections would leave the scene to drop one of them. Layout and desk sections + * are not editable from here, so the message names the one in the way. + */ +function cameraSectionInTheWay( + document: AxcutDocument, + startSec: number, + endSec: number, +): AgentToolExecution | null { + const desk = cameraSectionsOverlapping( + deskPillsForAgent(document), + toMs(startSec), + toMs(endSec), + )[0]; + if (desk) { + return failure( + `The span ${startSec.toFixed(1)}–${endSec.toFixed(1)} s overlaps the desk section ` + + `${desk.id} (${roundSec(desk.startMs)}–${roundSec(desk.endMs)} s), so no full-camera ` + + "region was written. Full-camera regions and desk sections may not overlap. Pick a " + + "span outside every entry of deskRegions in getCurrentDocument, or ask the user to " + + "change the desk section in the editor.", + ); + } + const blocking = cameraSectionsOverlapping( + cameraLayoutPillsForAgent(document), + toMs(startSec), + toMs(endSec), + )[0]; + if (!blocking) return null; + return failure( + `The span ${startSec.toFixed(1)}–${endSec.toFixed(1)} s overlaps the camera layout ` + + `section ${blocking.id} (${blocking.template}, ${roundSec(blocking.startMs)}–` + + `${roundSec(blocking.endMs)} s), so no full-camera region was written. Full-camera ` + + "regions and layout sections share one camera lane and may not overlap. Pick a span " + + "outside every entry of cameraLayoutRegions in getCurrentDocument, or ask the user " + + "to change the layout section in the editor.", + ); +} + /** The span the edited timeline actually occupies, for an actionable refusal. */ function editedExtentSec(document: AxcutDocument): { startSec: number; endSec: number } { const clips = document.timeline.clips; @@ -335,18 +402,23 @@ function landingSuffix( return parts.length ? ` (${parts.join(", ")})` : ""; } -/** Ids of every modifier in the document, all four families at once — the basis +/** Ids of every modifier in the document, every family at once — the basis * for naming what a destructive edit took with it. */ function modifierIdsOf(document: AxcutDocument): string[] { const legacy = (document.legacyEditor as Record) ?? {}; const speedRegions = (legacy.speedRegions as Array<{ id: string }> | undefined) ?? []; const cameraFullscreenRegions = (legacy.cameraFullscreenRegions as Array<{ id: string }> | undefined) ?? []; + const cameraLayoutRegions = + (legacy.cameraLayoutRegions as Array<{ id: string }> | undefined) ?? []; + const deskRegions = (legacy.deskRegions as Array<{ id: string }> | undefined) ?? []; return [ ...document.zoomRanges.map((r) => r.id), ...document.annotations.map((r) => r.id), ...speedRegions.map((r) => r.id), ...cameraFullscreenRegions.map((r) => r.id), + ...cameraLayoutRegions.map((r) => r.id), + ...deskRegions.map((r) => r.id), ]; } @@ -758,7 +830,12 @@ export function documentSnapshotForModel( const autoFocusAll = legacy?.autoFocusAll === true; return { timeBaseNote: - "clips and trims are in source-time seconds; zooms, speedRegions, annotations, cameraFullscreenRegions and audioTracks are in virtual (edited-timeline) seconds.", + "clips and trims are in source-time seconds; zooms, speedRegions, annotations, cameraFullscreenRegions, cameraLayoutRegions, deskRegions and audioTracks are in virtual (edited-timeline) seconds.", + cameraLayoutNote: + "cameraLayoutRegions are multi-camera layout sections the user placed in the editor; they are read-only here. " + + "They share one camera lane with cameraFullscreenRegions and the two may never overlap, so addCameraFullscreen / setCameraFullscreen refuse a span that lands on one. " + + "deskRegions are desk sections (the project's desk camera fills the frame), also read-only here; no camera section overlaps them, so the same tools refuse a span that lands on one. " + + "cameras lists the cameras shown, in place order, numbered as the user sees them (1 = the recording's main webcam).", audioNote: "audioTracks are imported voiceover / music files laid over the recording. They are clip-anchored like every other region, so they travel with their clip through reorder and trim, and they play at 1x whatever a speed region does to the picture under them. addAudio places an EXISTING asset of kind 'audio'; nothing here can import a file from disk or record one, so if the project has no audio asset, say so rather than inventing an id.", zoomNote: @@ -854,6 +931,18 @@ export function documentSnapshotForModel( startSec: roundSec(c.startMs), endSec: roundSec(c.endMs), })), + cameraLayoutRegions: cameraLayoutPillsForAgent(document).map((l) => ({ + id: l.id, + startSec: roundSec(l.startMs), + endSec: roundSec(l.endMs), + template: l.template, + cameras: l.slots.map((slot) => slot.camera + 1), + })), + deskRegions: deskPillsForAgent(document).map((d) => ({ + id: d.id, + startSec: roundSec(d.startMs), + endSec: roundSec(d.endMs), + })), // Imported audio, collapsed to the pills the ruler draws — a track ventilated // across a clip boundary is several fragments the user sees as one thing, and // the model has to name what the user sees. @@ -2034,6 +2123,8 @@ export function executeAgentTool( } const blind = noCameraUnderSpan(document, landing.startSec, landing.endSec); if (blind) return blind; + const occupied = cameraSectionInTheWay(document, landing.startSec, landing.endSec); + if (occupied) return occupied; const next: AxcutDocument = { ...document, legacyEditor: { ...legacy, cameraFullscreenRegions: [...prev, ...placed] }, @@ -2078,6 +2169,8 @@ export function executeAgentTool( } const blindMove = noCameraUnderSpan(document, landing.startSec, landing.endSec); if (blindMove) return blindMove; + const occupiedMove = cameraSectionInTheWay(document, landing.startSec, landing.endSec); + if (occupiedMove) return occupiedMove; const next: AxcutDocument = { ...document, legacyEditor: { ...legacy, cameraFullscreenRegions: rebuiltCamera }, diff --git a/electron/ai-edition/deep-agent/service.ts b/electron/ai-edition/deep-agent/service.ts index 64bcef03bd..90a1b815f0 100644 --- a/electron/ai-edition/deep-agent/service.ts +++ b/electron/ai-edition/deep-agent/service.ts @@ -26,6 +26,7 @@ import type { AxcutDocument } from "../../../src/lib/ai-edition/schema"; // wrong at both ends of a table that actually runs 1.25× to 5.0×. import { ZOOM_DEPTH_LEGEND } from "../../../src/lib/ai-edition/timeline/zoom-scale"; import { + type AgentToolExecution, addAnnotationArgs, addAudioArgs, addCameraFullscreenArgs, @@ -176,9 +177,9 @@ export const TOOL_DESCRIPTIONS: Record = { setAnnotation: "Move, resize, or edit the text of an existing annotation by id (virtual-timeline seconds). Only the fields you pass are changed.", addCameraFullscreen: - "Add a camera-fullscreen region over a span of the edited timeline (virtual seconds): the webcam fills the frame for that span. This only does something when the footage under that span comes from an asset with a linked webcam — check assets[].hasCameraTrack (or hasAnyCamera) in getCurrentDocument first. On footage with no camera the call is refused rather than storing a region that would render nothing; say so instead of retrying.", + "Add a camera-fullscreen region over a span of the edited timeline (virtual seconds): the webcam fills the frame for that span. This only does something when the footage under that span comes from an asset with a linked webcam — check assets[].hasCameraTrack (or hasAnyCamera) in getCurrentDocument first. On footage with no camera the call is refused rather than storing a region that would render nothing; say so instead of retrying. It is also refused over a multi-camera layout section (cameraLayoutRegions): the two share one lane and may not overlap.", setCameraFullscreen: - "Move or resize an existing camera-fullscreen region by id (virtual-timeline seconds). Only the fields you pass are changed. Refused if the new span lands on footage with no linked webcam.", + "Move or resize an existing camera-fullscreen region by id (virtual-timeline seconds). Only the fields you pass are changed. Refused if the new span lands on footage with no linked webcam or on a layout section (cameraLayoutRegions).", addAudio: "Lay an ALREADY-IMPORTED audio file over the recording across a span of the edited timeline (virtual seconds): a voiceover, or a music bed. assetId must name an asset whose kind is 'audio' — getCurrentDocument lists them; nothing here can import a file from disk or record one, so if there is none, say so instead of guessing an id. Omit endSec to play the whole file from offsetSec. kind picks the lane ('voiceover' or 'music'). offsetSec is where in the FILE playback starts, gainDb its level (0 unchanged, negative ducks it). A voiceover-lane track is also what gets transcribed, so the lane is not only cosmetic.", setAudio: @@ -218,7 +219,7 @@ export interface CursorTelemetryReader { probe?(input: { assetId: string; originalPath: string | null }): Promise; } -interface ToolRuntime { +export interface ToolRuntime { cursor?: CursorTelemetryReader; availableByAssetId?: Record; } @@ -268,18 +269,7 @@ function documentTool( return tool( async (args: z.infer) => { sink.toolStart(name, args); - // ponytail: the ONE async step the pure executor cannot take. Reading a - // sidecar is IO; `executeAgentTool` is synchronous by design (it is the - // gate every mutation passes through, and it has to stay testable - // without a filesystem). So the load happens here and its verdict — - // including "I could not look" — goes in as data. - const load = TOOLS_READING_CURSOR.has(name) - ? await loadCursorTelemetry(holder.current, args, runtime) - : undefined; - const execution = executeAgentTool(holder.current, name, JSON.stringify(args), { - editsAllowed, - cursorTelemetry: { availableByAssetId: runtime.availableByAssetId, load }, - }); + const execution = await runDocumentTool(holder.current, name, args, editsAllowed, runtime); if (execution.document) holder.current = execution.document; sink.toolEnd(name, execution.ok, execution.summary); return execution.resultJson; @@ -288,6 +278,33 @@ function documentTool( ); } +/** + * Run one tool against a document: the cursor read it may need, then the shared + * executor. The in-app agent (`documentTool`) and the MCP server both call this, + * so a tool behaves the same whichever agent is driving it. + * + * ponytail: the ONE async step the pure executor cannot take. Reading a sidecar + * is IO; `executeAgentTool` is synchronous by design (it is the gate every + * mutation passes through, and it has to stay testable without a filesystem). + * So the load happens here and its verdict — including "I could not look" — goes + * in as data. + */ +export async function runDocumentTool( + document: AxcutDocument, + name: string, + args: unknown, + editsAllowed: boolean, + runtime: ToolRuntime, +): Promise { + const load = TOOLS_READING_CURSOR.has(name) + ? await loadCursorTelemetry(document, args, runtime) + : undefined; + return executeAgentTool(document, name, JSON.stringify(args ?? {}), { + editsAllowed, + cursorTelemetry: { availableByAssetId: runtime.availableByAssetId, load }, + }); +} + /** Reads the sidecar for whichever asset the call names, defaulting to the * primary one — the same resolution the executor will use to report it. */ async function loadCursorTelemetry( @@ -331,37 +348,44 @@ export function buildTools( editsAllowed = true, runtime: ToolRuntime = {}, ) { - const build = (name: string, schema: S) => - documentTool(holder, sink, name, schema, editsAllowed, runtime); - return [ - build("getCurrentDocument", z.object({})), - build("getTranscript", getTranscriptArgs), - build("getTranscriptWords", getTranscriptWordsArgs), - build("getCursorTrack", getCursorTrackArgs), - build("setWordText", setWordTextArgs), - build("addTrim", addTrimArgs), - build("addTrims", addTrimsArgs), - build("setTrim", setTrimArgs), - build("setClipRange", setClipRangeArgs), - build("moveClip", moveClipArgs), - build("replaceTimeline", replaceTimelineArgs), - build("addZoom", addZoomArgs), - build("addZooms", addZoomsArgs), - build("setZoom", setZoomArgs), - build("addSpeed", addSpeedArgs), - build("setSpeed", setSpeedArgs), - build("addAnnotation", addAnnotationArgs), - build("setAnnotation", setAnnotationArgs), - build("addCameraFullscreen", addCameraFullscreenArgs), - build("setCameraFullscreen", setCameraFullscreenArgs), - build("addAudio", addAudioArgs), - build("setAudio", setAudioArgs), - build("removeTrim", removeTrimArgs), - build("removeModifier", removeModifierArgs), - build("removeClip", removeClipArgs), - ]; + return TOOL_ARG_SCHEMAS.map(([name, schema]) => + documentTool(holder, sink, name, schema, editsAllowed, runtime), + ); } +/** + * Every tool's name and argument schema, in the order the model is handed them. + * `buildTools` wraps these for the in-app agent and `electron/mcp/` exposes the + * same list over MCP, so the two surfaces cannot drift apart. + */ +export const TOOL_ARG_SCHEMAS: ReadonlyArray = [ + ["getCurrentDocument", z.object({})], + ["getTranscript", getTranscriptArgs], + ["getTranscriptWords", getTranscriptWordsArgs], + ["getCursorTrack", getCursorTrackArgs], + ["setWordText", setWordTextArgs], + ["addTrim", addTrimArgs], + ["addTrims", addTrimsArgs], + ["setTrim", setTrimArgs], + ["setClipRange", setClipRangeArgs], + ["moveClip", moveClipArgs], + ["replaceTimeline", replaceTimelineArgs], + ["addZoom", addZoomArgs], + ["addZooms", addZoomsArgs], + ["setZoom", setZoomArgs], + ["addSpeed", addSpeedArgs], + ["setSpeed", setSpeedArgs], + ["addAnnotation", addAnnotationArgs], + ["setAnnotation", setAnnotationArgs], + ["addCameraFullscreen", addCameraFullscreenArgs], + ["setCameraFullscreen", setCameraFullscreenArgs], + ["addAudio", addAudioArgs], + ["setAudio", setAudioArgs], + ["removeTrim", removeTrimArgs], + ["removeModifier", removeModifierArgs], + ["removeClip", removeClipArgs], +]; + /** * Prompt caching for the Anthropic-wire providers, which `createDeepAgent` * used to add for us (`isAnthropicModel` → `anthropicPromptCachingMiddleware`). @@ -401,7 +425,7 @@ export interface InvokeArgs { /** One cheap probe per asset, run before the tools are built so the very first * `getCurrentDocument` can already say whether telemetry exists. */ -async function probeCursorTelemetry( +export async function probeCursorTelemetry( document: AxcutDocument, cursor: CursorTelemetryReader | undefined, ): Promise | undefined> { diff --git a/electron/app-settings.test.ts b/electron/app-settings.test.ts index c6c9e74f3c..533b873fad 100644 --- a/electron/app-settings.test.ts +++ b/electron/app-settings.test.ts @@ -129,4 +129,142 @@ describe("app settings store", () => { store.dismissStarPrompt(); expect(store.getSnapshot().recording).toMatchObject({ micEnabled: true, micDeviceId: "mic" }); }); + + it("stores a camera quality and rejects one it cannot capture at", () => { + const dir = temp(); + const store = new AppSettingsStore(dir); + + expect(store.setRecordingPreferences({ camQuality: "1080p" }).recording.camQuality).toBe( + "1080p", + ); + expect(() => store.setRecordingPreferences({ camQuality: "4320p" as never })).toThrow( + TypeError, + ); + expect(store.getSnapshot().recording.camQuality).toBe("1080p"); + }); + + it("leaves the camera quality alone when a patch does not mention it", () => { + // Every other window writes narrow patches; a camera toggle from the HUD + // must not quietly reset the resolution the user picked. + const dir = temp(); + const store = new AppSettingsStore(dir); + store.setRecordingPreferences({ camQuality: "1440p" }); + + store.setRecordingPreferences({ camEnabled: true }); + + expect(store.getSnapshot().recording.camQuality).toBe("1440p"); + }); + + it("reads a settings file written before the camera had a quality", () => { + const dir = temp(); + writeFileSync( + path.join(dir, "recording-settings.json"), + JSON.stringify({ micEnabled: true }), + "utf8", + ); + + expect(new AppSettingsStore(dir).getSnapshot().recording.camQuality).toBe("2160p"); + }); + + it("reads a missing additional camera list as empty", () => { + const dir = temp(); + writeFileSync( + path.join(dir, "recording-settings.json"), + JSON.stringify({ camEnabled: true }), + "utf8", + ); + + expect(new AppSettingsStore(dir).getSnapshot().recording.camAdditionalDevices).toEqual([]); + }); + + it("drops additional camera entries without a name and keeps at most three", () => { + const store = new AppSettingsStore(temp()); + store.setRecordingPreferences({ + camAdditionalDevices: [ + { id: "a", name: "A" }, + { id: null, name: "B" }, + { id: "x", name: "" }, + { id: "c", name: "C" }, + { id: "d", name: "D" }, + { id: "e", name: "E" }, + ], + }); + + expect(store.getSnapshot().recording.camAdditionalDevices).toEqual([ + { id: "a", name: "A" }, + { id: null, name: "B" }, + { id: "c", name: "C" }, + ]); + }); + + it("ignores junk in a stored additional camera list", () => { + const dir = temp(); + writeFileSync( + path.join(dir, "recording-settings.json"), + JSON.stringify({ + camAdditionalDevices: [null, 3, "x", { name: 5 }, { id: 7, name: "Kept" }, { name: "Ok" }], + }), + "utf8", + ); + + expect(new AppSettingsStore(dir).getSnapshot().recording.camAdditionalDevices).toEqual([ + { id: null, name: "Kept" }, + { id: null, name: "Ok" }, + ]); + }); + + it("rejects an additional camera list that is not a list", () => { + const store = new AppSettingsStore(temp()); + + expect(() => store.setRecordingPreferences({ camAdditionalDevices: "x" as never })).toThrow( + TypeError, + ); + }); +}); + +describe("desk camera preference", () => { + it("defaults to no desk camera", () => { + expect(new AppSettingsStore(temp()).getSnapshot().recording.camDeskDevice).toBeNull(); + }); + + it("stores and clears the desk camera", () => { + const dir = temp(); + const store = new AppSettingsStore(dir); + store.setRecordingPreferences({ camDeskDevice: { id: "b", name: "Cam B" } }); + expect(new AppSettingsStore(dir).getSnapshot().recording.camDeskDevice).toEqual({ + id: "b", + name: "Cam B", + }); + + store.setRecordingPreferences({ camDeskDevice: null }); + expect(new AppSettingsStore(dir).getSnapshot().recording.camDeskDevice).toBeNull(); + }); + + it("reads a stored desk camera without a name as none", () => { + const dir = temp(); + writeFileSync( + path.join(dir, "recording-settings.json"), + JSON.stringify({ camDeskDevice: { id: 7, name: "" } }), + "utf8", + ); + expect(new AppSettingsStore(dir).getSnapshot().recording.camDeskDevice).toBeNull(); + + writeFileSync( + path.join(dir, "recording-settings.json"), + JSON.stringify({ camDeskDevice: { id: 7, name: "Kept" } }), + "utf8", + ); + expect(new AppSettingsStore(dir).getSnapshot().recording.camDeskDevice).toEqual({ + id: null, + name: "Kept", + }); + }); + + it("rejects a desk camera that is not a camera", () => { + const store = new AppSettingsStore(temp()); + expect(() => store.setRecordingPreferences({ camDeskDevice: "x" as never })).toThrow(TypeError); + expect(() => store.setRecordingPreferences({ camDeskDevice: { id: "x" } as never })).toThrow( + TypeError, + ); + }); }); diff --git a/electron/app-settings.ts b/electron/app-settings.ts index 2cc1cee258..0e08162c21 100644 --- a/electron/app-settings.ts +++ b/electron/app-settings.ts @@ -1,5 +1,11 @@ import { readFileSync, renameSync, rmSync, writeFileSync } from "node:fs"; import path from "node:path"; +import { + DEFAULT_WEBCAM_QUALITY, + WEBCAM_QUALITY_IDS, + type WebcamQualityId, + webcamQualityFrom, +} from "../src/hooks/webcamCaptureTarget"; import type { CursorCaptureMode } from "../src/lib/recordingSession"; export interface RecordingPreferences { @@ -9,6 +15,12 @@ export interface RecordingPreferences { camEnabled: boolean; camDeviceId: string | null; camDeviceName: string | null; + /** Cameras 2-4 of a native Windows recording, in the order they were picked. At most three. */ + camAdditionalDevices: Array<{ id: string | null; name: string }>; + /** The camera filming the desk, matched like the additional cameras (id, else name). */ + camDeskDevice: { id: string | null; name: string } | null; + /** Capture resolution for the camera. See WEBCAM_QUALITY_PRESETS. */ + camQuality: WebcamQualityId; systemAudioEnabled: boolean; cursorCaptureMode: CursorCaptureMode; /** Display captures on macOS and Windows. Opt-in: on Windows the icons also leave the real desktop while recording. */ @@ -30,6 +42,9 @@ export const DEFAULT_RECORDING_PREFERENCES: RecordingPreferences = { camEnabled: false, camDeviceId: null, camDeviceName: null, + camAdditionalDevices: [], + camDeskDevice: null, + camQuality: DEFAULT_WEBCAM_QUALITY, systemAudioEnabled: false, cursorCaptureMode: "editable-overlay", hideDesktopIcons: false, @@ -92,6 +107,30 @@ const bool = (value: unknown, fallback: boolean) => (typeof value === "boolean" const nullableString = (value: unknown, fallback: string | null) => value === null || typeof value === "string" ? value : fallback; +const MAX_ADDITIONAL_CAMERAS = 3; + +/** Keeps the entries that name a camera, in order, up to the cap; everything else is junk. */ +function additionalCameras(value: unknown): RecordingPreferences["camAdditionalDevices"] { + if (!Array.isArray(value)) return []; + const kept: RecordingPreferences["camAdditionalDevices"] = []; + for (const entry of value) { + if (kept.length >= MAX_ADDITIONAL_CAMERAS) break; + if (!entry || typeof entry !== "object") continue; + const { id, name } = entry as Record; + if (typeof name !== "string" || name.length === 0) continue; + kept.push({ id: typeof id === "string" ? id : null, name }); + } + return kept; +} + +/** A camera with a name, or none: anything else is junk. */ +function deskCamera(value: unknown): RecordingPreferences["camDeskDevice"] { + if (!value || typeof value !== "object") return null; + const { id, name } = value as Record; + if (typeof name !== "string" || name.length === 0) return null; + return { id: typeof id === "string" ? id : null, name }; +} + function parseRecording(raw: RawSettings): RecordingPreferences { return { micEnabled: bool(raw.micEnabled, DEFAULT_RECORDING_PREFERENCES.micEnabled), @@ -100,6 +139,11 @@ function parseRecording(raw: RawSettings): RecordingPreferences { camEnabled: bool(raw.camEnabled, DEFAULT_RECORDING_PREFERENCES.camEnabled), camDeviceId: nullableString(raw.camDeviceId, DEFAULT_RECORDING_PREFERENCES.camDeviceId), camDeviceName: nullableString(raw.camDeviceName, DEFAULT_RECORDING_PREFERENCES.camDeviceName), + camAdditionalDevices: additionalCameras(raw.camAdditionalDevices), + camDeskDevice: deskCamera(raw.camDeskDevice), + // Unset in every settings file written before the camera had a quality + // setting, and `webcamQualityFrom` answers those with the default. + camQuality: webcamQualityFrom(raw.camQuality), systemAudioEnabled: bool( raw.systemAudioEnabled, DEFAULT_RECORDING_PREFERENCES.systemAudioEnabled, @@ -157,6 +201,16 @@ function validateRecordingPatch(patch: Partial): void { if ((key.endsWith("Enabled") || key === "hideDesktopIcons") && typeof value !== "boolean") { throw new TypeError(`${key} must be a boolean`); } + if (key === "camAdditionalDevices") { + if (!Array.isArray(value)) throw new TypeError("camAdditionalDevices must be a list"); + continue; + } + if (key === "camDeskDevice") { + if (value !== null && deskCamera(value) === null) { + throw new TypeError("camDeskDevice must be a camera or null"); + } + continue; + } if ( (key.endsWith("DeviceId") || key.endsWith("DeviceName")) && value !== null && @@ -167,6 +221,12 @@ function validateRecordingPatch(patch: Partial): void { if (key === "cursorCaptureMode" && value !== "system" && value !== "editable-overlay") { throw new TypeError("cursorCaptureMode is invalid"); } + // Rejected here rather than coerced on read, so a bad write is a visible + // error at its source instead of a resolution that silently is not the + // one the caller asked for. + if (key === "camQuality" && !WEBCAM_QUALITY_IDS.includes(value as WebcamQualityId)) { + throw new TypeError("camQuality is invalid"); + } } } @@ -189,6 +249,12 @@ export class AppSettingsStore { const next = Object.fromEntries( Object.entries(patch).filter(([, value]) => value !== undefined), ) as Partial; + if (next.camAdditionalDevices) { + next.camAdditionalDevices = additionalCameras(next.camAdditionalDevices); + } + if (next.camDeskDevice) { + next.camDeskDevice = deskCamera(next.camDeskDevice); + } atomicWrite(this.userData, { ...raw, ...current, ...next }); return this.getSnapshot(); } diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index 29d5f828b4..1c28c8b4fe 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -37,6 +37,15 @@ interface Window { * `compositor.export`/`compositor.exportMulti` runs. Distinct from `exportOnFrameAck`, * the OLD web/CPU pipeline's per-frame ack, not a progress signal. */ onNativeExportProgress?: (callback: (frames: number, exportId?: string) => void) => () => void; + /** Preview frames the main process hands over as shared GPU textures (Windows). The + * listener draws `frame` and closes it; one listener at a time. Returns the + * unsubscribe. Optional: shim/web contexts have no bridge. */ + onCompositorFrame?: ( + listener: ( + frame: VideoFrame, + meta: import("../src/native/contracts").CompositorSharedFrameMeta, + ) => void, + ) => () => void; getSources: (opts: Electron.SourcesOptions) => Promise; switchToEditor: () => Promise; switchToHud: () => Promise; @@ -47,6 +56,8 @@ interface Window { }>; /** Sources are picked in Apple's system picker (macOS 15.2+), not in an app list. */ usesSystemSourcePicker?: () => Promise; + /** Name of the last source picked in Apple's picker, for display only: it is not selected. */ + getLastPickedSource?: () => Promise; openNotes: () => Promise<{ opened: boolean; reason?: string; @@ -71,6 +82,12 @@ interface Window { onAiEditionChatEvent: ( callback: (event: import("../src/native/contracts").AiEditionChatEvent) => void, ) => () => void; + /** Optional: absent in the browser shim and in tests that stub electronAPI. */ + onAiEditionMcpRequest?: ( + callback: ( + request: import("../src/native/contracts").AiEditionMcpHostRequest, + ) => Promise, + ) => () => void; requestCameraAccess: () => Promise<{ success: boolean; granted: boolean; @@ -160,6 +177,17 @@ interface Window { * saved without it. Still a success — the screen video is intact. */ webcamDropped?: boolean; + /** + * Labels (device name, else `Camera `) of additional cameras (2-4) that + * produced nothing usable and were left out of the session. Camera 1 is + * `webcamDropped`. + */ + droppedWebcams?: string[]; + /** + * Labels of cameras (camera 1 included) that the helper disabled mid-take. + * Their partial files are kept in the session; the user is told which. + */ + webcamsStoppedEarly?: string[]; }>; pauseNativeWindowsRecording: () => Promise<{ success: boolean; @@ -172,6 +200,7 @@ interface Window { startNativeMacRecording: ( request: import("../src/lib/nativeMacRecording").NativeMacRecordingRequest, ) => Promise; + onNativeMacSystemAudioUnavailable: (callback: () => void) => () => void; pauseNativeMacRecording: () => Promise<{ success: boolean; error?: string; @@ -335,12 +364,9 @@ interface Window { /** Why this recording ended before it was stopped, when it did. */ warning?: string; }>; - findRecordingCamera: (videoPath: string) => Promise<{ - success: boolean; - webcamVideoPath?: string; - offsetMs?: number; - error?: string; - }>; + findRecordingCamera: ( + videoPath: string, + ) => Promise; readBinaryFile: (filePath: string) => Promise<{ success: boolean; data?: ArrayBuffer; diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index bd19f10596..b8cf2dce05 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -13,10 +13,12 @@ import { desktopCapturer, dialog, ipcMain, + safeStorage, screen, shell, systemPreferences, } from "electron"; +import { DEFAULT_WEBCAM_QUALITY, type WebcamQualityId } from "../../src/hooks/webcamCaptureTarget"; import { type AxcutDocument, isAxcutDocumentFile, @@ -32,7 +34,10 @@ import { } from "../../src/lib/nativeMacRecording"; import type { NativeWindowsRecordingRequest } from "../../src/lib/nativeWindowsRecording"; import { + type AdditionalWebcam, type CursorCaptureMode, + type FindRecordingCameraResult, + findRecordingCameraResult, normalizeCursorCaptureMode, normalizeProjectMedia, normalizeRecordingSession, @@ -68,6 +73,9 @@ import { isDiagnosticModeEnabled, mainLogBuffer } from "../diagnostics/main-log- import { mainT } from "../i18n"; import { getInstallChannel } from "../install-channel"; import { RECORDINGS_DIR } from "../main"; +import { EditorDocumentHost } from "../mcp/editor-document-host"; +import { McpController } from "../mcp/mcp-controller"; +import { McpSettingsStore } from "../mcp/mcp-settings-store"; import { type AudioPeaksResult, getAudioPeaks } from "../media/audioPeaks"; import { readCursorRecordingFile as readCursorRecordingFileFrom, @@ -116,17 +124,36 @@ import { NATIVE_WINDOWS_SALVAGEABLE_OUTPUT_BYTES, readMicrophoneDefaulted, readMicrophoneUnavailable, - readWebcamFormat, - readWebcamUnavailable, + readReportedWebcamPaths, + readSecondaryWindowsApplied, + readStoppedWebcamPaths, + readUnavailableWebcamIndices, + readWebcamFormatAt, terminateNativeWindowsCapture, waitForNativeWindowsCaptureStop, } from "../recording/nativeWindowsCaptureStop"; +import { + additionalWebcamLabels, + buildHelperWebcamConfig, + collectStoppedWebcams, + dedupeAdditionalWebcams, + deskWebcamPath, + isWebcamSidecarFile, + labelsOfUnavailableAdditionalWebcams, + labelsOfWebcamsStoppedEarly, + recordedDeskCamera, + remapDeskCamera, + stripWebcamSuffix, + webcamOutputPath, +} from "../recording/nativeWindowsWebcams"; import { patchWebmDurationOnDisk } from "../recording/webm-duration"; import { reindexRecordingOnDisk } from "../recording/webm-seek-index"; import { + describeMacPickerSource, describeRecordingSource, enumerationIncludesSourceKind, mergeEnumeratedSources, + rememberedPickerSourceName, resolveRecordingSource, restoreRecordingSourceAfterEnumeration, shouldEnumerateRecordingSources, @@ -590,9 +617,25 @@ async function getApprovedProjectSession( throw new Error("Project references an invalid or unsupported webcam video path"); } - return webcamVideoPath - ? { screenVideoPath, webcamVideoPath, createdAt: Date.now() } - : { screenVideoPath, createdAt: Date.now() }; + // Additional cameras go through the same approval as camera 1; one that no + // longer resolves is dropped rather than failing the whole project. + const additionalWebcams: AdditionalWebcam[] = []; + for (const extra of media.additionalWebcams ?? []) { + const approved = await approveReadableVideoPath( + await resolveWithSiblingFallback(extra.path), + trustedDirs, + ); + if (approved) { + additionalWebcams.push({ path: approved, label: extra.label }); + } + } + + return { + screenVideoPath, + ...(webcamVideoPath ? { webcamVideoPath } : {}), + ...(additionalWebcams.length > 0 ? { additionalWebcams } : {}), + createdAt: Date.now(), + }; } type SelectedSource = { @@ -649,6 +692,12 @@ export interface RecordingPrefs { camDeviceId: string | null; /** Camera label paired with the preferred id for restart-safe resolution. */ camDeviceName: string | null; + /** Cameras 2-4 of a native Windows recording, in pick order. At most three. */ + camAdditionalDevices: Array<{ id: string | null; name: string }>; + /** The camera filming the desk; a take records its index as the session's `deskCamera`. */ + camDeskDevice: { id: string | null; name: string } | null; + /** Capture resolution for the camera. See WEBCAM_QUALITY_PRESETS. */ + camQuality: WebcamQualityId; systemAudioEnabled: boolean; cursorCaptureMode: CursorCaptureMode; hideDesktopIcons: boolean; @@ -662,6 +711,9 @@ const defaultRecordingPrefs: RecordingPrefs = { camEnabled: false, camDeviceId: null, camDeviceName: null, + camAdditionalDevices: [], + camDeskDevice: null, + camQuality: DEFAULT_WEBCAM_QUALITY, systemAudioEnabled: false, cursorCaptureMode: "editable-overlay", hideDesktopIcons: false, @@ -715,6 +767,23 @@ let nativeWindowsCaptureProcess: ChildProcessWithoutNullStreams | null = null; let nativeWindowsCaptureOutput = ""; let nativeWindowsCaptureTargetPath: string | null = null; let nativeWindowsCaptureWebcamTargetPath: string | null = null; +/** + * Cameras 2-4 of the running take, in helper order after camera 1, each with + * the label it is reported under. Only paths generated here ever land in it. + */ +let nativeWindowsCaptureAdditionalWebcamTargets: Array<{ path: string; label: string }> = []; +/** + * The file the desk camera of the running take records into, or null. A path, not an index: + * cameras dropped at start or stop shift the indices, the file stays the desk camera's. + */ +let nativeWindowsCaptureDeskWebcamPath: string | null = null; +/** Camera 1's label for notices: its device name, else "Camera 1". */ +let nativeWindowsCaptureWebcamLabel = "Camera 1"; +/** + * Files of cameras the helper dropped at start. The helper deletes them, but a + * delete that failed leaves a 0-byte stub; stop and discard remove it if empty. + */ +let nativeWindowsCaptureDroppedWebcamPaths: string[] = []; let nativeWindowsCaptureRecordingId: number | null = null; let nativeWindowsCursorOffsetMs = 0; let nativeWindowsCursorCaptureMode: CursorCaptureMode = "editable-overlay"; @@ -741,6 +810,10 @@ function resetNativeWindowsCaptureState() { nativeWindowsCaptureProcess = null; nativeWindowsCaptureTargetPath = null; nativeWindowsCaptureWebcamTargetPath = null; + nativeWindowsCaptureAdditionalWebcamTargets = []; + nativeWindowsCaptureDeskWebcamPath = null; + nativeWindowsCaptureWebcamLabel = "Camera 1"; + nativeWindowsCaptureDroppedWebcamPaths = []; nativeWindowsCaptureRecordingId = null; nativeWindowsCursorOffsetMs = 0; nativeWindowsCursorCaptureMode = "editable-overlay"; @@ -768,12 +841,12 @@ async function salvageNativeWindowsFragmentedCapture(screenVideoPath: string | n */ async function removeNativeWindowsCaptureOutputs( screenVideoPath: string | null, - webcamVideoPath: string | null, + webcamVideoPaths: Array, options: { onlyIfUnusable?: boolean } = {}, ) { const targets = [ screenVideoPath, - webcamVideoPath, + ...webcamVideoPaths, screenVideoPath ? `${screenVideoPath}.cursor.json` : null, ]; @@ -799,6 +872,22 @@ async function removeNativeWindowsCaptureOutputs( } } } + +/** Removes the 0-byte stubs of cameras dropped at start; anything with data stays. */ +async function removeEmptyNativeWindowsWebcamFiles(paths: string[]) { + for (const target of paths) { + if (!isPathWithinDir(target, RECORDINGS_DIR)) { + continue; + } + const stats = await fs.stat(target).catch(() => null); + if (stats?.size !== 0) { + continue; + } + await fs.rm(target, { force: true }).catch((error) => { + console.warn("[native-wgc] could not remove an empty camera file:", target, error); + }); + } +} let nativeMacCaptureProcess: ChildProcessWithoutNullStreams | null = null; /** * Apple's system picker session (macOS 15.2+), started on first use and kept for the app's @@ -1054,7 +1143,9 @@ async function getMacPickerSession(): Promise { return session; } -function selectedSourceFromPick(pick: MacPickerSelection): SelectedSource { +function selectedSourceFromPick( + pick: MacPickerSelection, +): SelectedSource & { id: string; display_id: string } { const display = pick.displayId !== null ? screen.getAllDisplays().find((candidate) => candidate.id === pick.displayId) @@ -1328,6 +1419,7 @@ async function registerRecordingMediaLinks( options: { webcamVideoPath?: string; webcamOffsetMs?: number; + additionalWebcams?: AdditionalWebcam[]; cursorCaptureMode?: CursorCaptureMode; }, ) { @@ -1342,6 +1434,9 @@ async function registerRecordingMediaLinks( ...(options.webcamVideoPath && Number.isFinite(options.webcamOffsetMs) ? { webcamOffsetMs: options.webcamOffsetMs } : {}), + ...(options.additionalWebcams?.length + ? { additionalWebcams: options.additionalWebcams } + : {}), ...(hasCursorTelemetry ? { cursorTelemetryPath } : {}), ...(options.cursorCaptureMode ? { cursorCaptureMode: options.cursorCaptureMode } : {}), }); @@ -1605,6 +1700,7 @@ function inspectNativeMacCaptureOutput() { function attachNativeMacCaptureOutputDrain( proc: ChildProcessWithoutNullStreams, onTakeEnded: () => void, + onSystemAudioUnavailable: () => void, ) { let lineBuffer = ""; // Hooked here rather than on `nativeMacCaptureEvents`, which the start wait @@ -1612,6 +1708,7 @@ function attachNativeMacCaptureOutputDrain( const watchLiveTake = createNativeMacMidCaptureErrorWatch( () => nativeMacCaptureProcess === proc && !nativeMacStopInFlight, onTakeEnded, + onSystemAudioUnavailable, ); const drain = (chunk: Buffer) => { const text = chunk.toString(); @@ -1721,9 +1818,7 @@ function setCurrentRecordingSessionState(session: RecordingSession | null) { function getSessionManifestPathForVideo(videoPath: string) { const parsedPath = path.parse(videoPath); - const baseName = parsedPath.name.endsWith("-webcam") - ? parsedPath.name.slice(0, -"-webcam".length) - : parsedPath.name; + const baseName = stripWebcamSuffix(parsedPath.name); return path.join(parsedPath.dir, `${baseName}${RECORDING_SESSION_SUFFIX}`); } @@ -1777,10 +1872,48 @@ async function loadRecordedSessionForVideoPath( } } + const keptExtraIndices: number[] = []; + if (session.additionalWebcams) { + const approvedExtras: AdditionalWebcam[] = []; + for (const [index, extra] of session.additionalWebcams.entries()) { + let extraPath: string | null = extra.path; + if (!isPathAllowed(extraPath)) { + extraPath = await approveReadableVideoPath(extraPath, [ + path.dirname(manifestPath), + RECORDINGS_DIR, + ]); + } + if (extraPath) { + approvedExtras.push({ path: extraPath, label: extra.label }); + keptExtraIndices.push(index); + } + } + if (approvedExtras.length > 0) { + session.additionalWebcams = approvedExtras; + } else { + delete session.additionalWebcams; + } + } + + // Dropped cameras shift the indices after them; a dropped desk camera is no desk camera. + const deskCamera = remapDeskCamera( + session.deskCamera, + Boolean(session.webcamVideoPath), + keptExtraIndices, + ); + if (deskCamera !== undefined) { + session.deskCamera = deskCamera; + } else { + delete session.deskCamera; + } + approveFilePath(session.screenVideoPath); if (session.webcamVideoPath) { approveFilePath(session.webcamVideoPath); } + for (const extra of session.additionalWebcams ?? []) { + approveFilePath(extra.path); + } return session; } catch (error) { const nodeError = error as NodeJS.ErrnoException; @@ -1800,6 +1933,8 @@ async function loadRecordedSessionForVideoPath( async function resolveMediaLinksForVideo(videoPath: string): Promise<{ webcamVideoPath?: string; webcamOffsetMs?: number; + additionalWebcams?: AdditionalWebcam[]; + deskCamera?: number; cursorTelemetryPath?: string; resolvedVia: "sidecar" | "fingerprint" | "none"; }> { @@ -1810,6 +1945,7 @@ async function resolveMediaLinksForVideo(videoPath: string): Promise<{ .then(() => true) .catch(() => false); + const sessionAdditionalWebcams = session?.additionalWebcams ?? []; if (session?.webcamVideoPath || hasCursorTelemetry) { // Opportunistic backfill so the link survives a later move even if this // recording predates the registry, or if its sidecar doesn't travel with it. @@ -1818,6 +1954,9 @@ async function resolveMediaLinksForVideo(videoPath: string): Promise<{ ...(session?.webcamVideoPath && Number.isFinite(session.webcamOffsetMs) ? { webcamOffsetMs: session.webcamOffsetMs } : {}), + ...(sessionAdditionalWebcams.length > 0 + ? { additionalWebcams: sessionAdditionalWebcams } + : {}), ...(hasCursorTelemetry ? { cursorTelemetryPath } : {}), }).catch((error) => console.warn("[media-links] backfill failed:", error)); @@ -1828,6 +1967,10 @@ async function resolveMediaLinksForVideo(videoPath: string): Promise<{ webcamOffsetMs: session.webcamOffsetMs ?? 0, } : {}), + ...(sessionAdditionalWebcams.length > 0 + ? { additionalWebcams: sessionAdditionalWebcams } + : {}), + ...(session?.deskCamera !== undefined ? { deskCamera: session.deskCamera } : {}), ...(hasCursorTelemetry ? { cursorTelemetryPath } : {}), resolvedVia: "sidecar", }; @@ -1841,8 +1984,19 @@ async function resolveMediaLinksForVideo(videoPath: string): Promise<{ webcamVideoPath = (await approveReadableVideoPath(webcamVideoPath, [RECORDINGS_DIR])) ?? undefined; } + const additionalWebcams: AdditionalWebcam[] = []; + for (const extra of links.additionalWebcams ?? []) { + let extraPath: string | null = extra.path; + if (!isPathAllowed(extraPath)) { + extraPath = await approveReadableVideoPath(extraPath, [RECORDINGS_DIR]); + } + if (extraPath) { + additionalWebcams.push({ path: extraPath, label: extra.label }); + } + } return { ...(webcamVideoPath ? { webcamVideoPath, webcamOffsetMs: links.webcamOffsetMs ?? 0 } : {}), + ...(additionalWebcams.length > 0 ? { additionalWebcams } : {}), ...(links.cursorTelemetryPath ? { cursorTelemetryPath: links.cursorTelemetryPath } : {}), resolvedVia: "fingerprint", }; @@ -2081,13 +2235,19 @@ export function registerIpcHandlers( }, ); - async function presentMacSystemPicker(session: MacPickerSession) { - // The HUD and the notes window belong to this process, not to the helper, so a display - // pick would record them unless the picker is told to leave them out. + /** + * The HUD and the notes window belong to this process, not to the helper, so a display + * pick would record them unless the picker is told to leave them out. + */ + function macPickerExcludedWindowIds() { const appWindowSourceIds = [getMainWindow(), getNotesWindow()] .filter((window): window is BrowserWindow => !!window && !window.isDestroyed()) .map((window) => window.getMediaSourceId()); - const excludedWindowIds = collectMacCaptureExcludedWindowIds(appWindowSourceIds); + return collectMacCaptureExcludedWindowIds(appWindowSourceIds); + } + + async function presentMacSystemPicker(session: MacPickerSession) { + const excludedWindowIds = macPickerExcludedWindowIds(); // Out of the way while the picker is up. The HUD window is far larger than the bar // it draws (a transparent reserve above it), and Apple's picker targets windows by // their frame, not by where clicks land -- so that invisible rectangle hid every @@ -2119,8 +2279,15 @@ export function registerIpcHandlers( } return; } - selectedSource = selectedSourceFromPick(pick); + const picked = selectedSourceFromPick(pick); + selectedSource = picked; selectedDesktopSource = null; + // Written only so the next launch can name it: the pick itself dies with the helper. + try { + appSettings.setLastSource(describeMacPickerSource(picked, pick.kind)); + } catch (error) { + console.warn("Failed to persist the picked recording source:", error); + } broadcastSelectedSource(selectedSource); } @@ -2132,6 +2299,12 @@ export function registerIpcHandlers( // picker they must hand the choice to `open-source-selector` rather than enumerate. ipcMain.handle("uses-system-source-picker", () => macPickerOwnsSources()); + // Named in the HUD and Record mode after a relaunch, while `get-selected-source` still + // answers null: the pick cannot be restored, so it is never reported as selected. + ipcMain.handle("get-last-picked-source", () => + rememberedPickerSourceName(appSettings.getSnapshot().lastSource, macPickerOwnsSources()), + ); + ipcMain.handle("get-selected-source", async () => { const previousSelectedSource = selectedSource; if (process.platform === "linux" && findPipeWireCursorHelperPath()) { @@ -2151,6 +2324,9 @@ export function registerIpcHandlers( if (!isMacPickerSourceId(selectedSource?.id)) { return selectedDesktopSource ? selectedSource : null; } + // A screen pick that would record the current HUD or notes window is no pick at all: + // answering null makes the next Record present the picker again (#965). + macPickerSession?.forgetSelectionUnlessExcluding(macPickerExcludedWindowIds()); if (!macPickerSession?.getSelection()) { selectedSource = null; selectedDesktopSource = null; @@ -2287,12 +2463,11 @@ export function registerIpcHandlers( return access; } - // The helper that answered has just raised macOS' own Accessibility prompt on - // its way up, so the window must offer System Settings, not a second prompt. It - // explains what the grant is for and tracks it live, where a message box could - // only say "go to System Settings" in English. + // Accessibility improves cursor shape hints but is not required to record. + // Remember that the helper raised the system prompt so a later visit to the + // permissions window can direct the user to Settings, but don't reopen that + // window from every Record press. getMacPermissions().noteRequested("accessibility"); - showPermissionsWindow(); } return access; @@ -2755,10 +2930,7 @@ export function registerIpcHandlers( ? request.recordingId : Date.now(); const outputPath = path.join(RECORDINGS_DIR, `${RECORDING_FILE_PREFIX}${recordingId}.mp4`); - const webcamOutputPath = path.join( - RECORDINGS_DIR, - `${RECORDING_FILE_PREFIX}${recordingId}-webcam.mp4`, - ); + const webcamPath = webcamOutputPath(RECORDINGS_DIR, RECORDING_FILE_PREFIX, recordingId, 1); const sourceDisplay = request.source.type === "display" && typeof request.source.displayId === "number" ? (screen.getAllDisplays().find((display) => display.id === request.source.displayId) ?? @@ -2777,6 +2949,24 @@ export function registerIpcHandlers( const webcamDirectShowClsid = request.webcam.enabled ? await resolveDirectShowWebcamClsid(request.webcam.deviceName) : null; + // Cameras 2-4 only while camera 1 is on: its toggle governs every + // camera. Files are numbered from 2 in the order they are sent. + const requestedExtras = Array.isArray(request.additionalWebcams) + ? request.additionalWebcams + : []; + const keptExtras = request.webcam.enabled + ? dedupeAdditionalWebcams(request.webcam, requestedExtras) + : []; + const extraLabels = additionalWebcamLabels(request.webcam.deviceName, keptExtras); + const additionalWebcams = await Promise.all( + keptExtras.map(async (extra, i) => ({ + deviceId: extra.deviceId, + deviceName: extra.deviceName, + label: extraLabels[i], + clsid: await resolveDirectShowWebcamClsid(extra.deviceName), + path: webcamOutputPath(RECORDINGS_DIR, RECORDING_FILE_PREFIX, recordingId, i + 2), + })), + ); const cursorCaptureMode = normalizeCursorCaptureMode(request.cursor?.mode) ?? "editable-overlay"; const envPreferSoftwareEncoder = (process.env.OPENSCREEN_WGC_PREFER_SOFTWARE_ENCODER ?? "") @@ -2808,13 +2998,12 @@ export function registerIpcHandlers( microphoneDeviceId: request.audio.microphone.deviceId ?? null, microphoneDeviceName: request.audio.microphone.deviceName ?? null, microphoneGain: request.audio.microphone.gain, - webcamEnabled: request.webcam.enabled, - webcamDeviceId: request.webcam.deviceId ?? null, - webcamDeviceName: request.webcam.deviceName ?? null, - webcamDirectShowClsid, - webcamWidth: request.webcam.width, - webcamHeight: request.webcam.height, - webcamFps: request.webcam.fps, + ...buildHelperWebcamConfig({ + camera1: request.webcam, + camera1Clsid: webcamDirectShowClsid, + camera1Path: webcamPath, + extras: additionalWebcams, + }), captureCursor: cursorCaptureMode === "system", cursorCaptureMode, hideDesktopIcons: @@ -2822,7 +3011,7 @@ export function registerIpcHandlers( appSettings.getSnapshot().recording.hideDesktopIcons, outputs: { screenPath: outputPath, - webcamPath: webcamOutputPath, + webcamPath, }, source: { type: request.source.type, @@ -2844,6 +3033,10 @@ export function registerIpcHandlers( source: request.source, audio: request.audio, webcam: request.webcam, + additionalWebcams: additionalWebcams.map(({ label, path: cameraPath }) => ({ + label, + path: cameraPath, + })), encoder: { preferSoftwareEncoder }, cursor: { mode: cursorCaptureMode }, // Both spaces, deliberately: the helper's own errors quote the physical @@ -2858,7 +3051,19 @@ export function registerIpcHandlers( await fs.mkdir(RECORDINGS_DIR, { recursive: true }); nativeWindowsCaptureOutput = ""; nativeWindowsCaptureTargetPath = outputPath; - nativeWindowsCaptureWebcamTargetPath = request.webcam.enabled ? webcamOutputPath : null; + nativeWindowsCaptureWebcamTargetPath = request.webcam.enabled ? webcamPath : null; + nativeWindowsCaptureAdditionalWebcamTargets = additionalWebcams.map( + ({ label, path: cameraPath }) => ({ label, path: cameraPath }), + ); + nativeWindowsCaptureDeskWebcamPath = deskWebcamPath({ + deskCamera: request.deskCamera, + camera1Path: request.webcam.enabled ? webcamPath : null, + requested: requestedExtras, + kept: keptExtras, + keptPaths: additionalWebcams.map((extra) => extra.path), + }); + nativeWindowsCaptureWebcamLabel = request.webcam.deviceName?.trim() || "Camera 1"; + nativeWindowsCaptureDroppedWebcamPaths = []; nativeWindowsCaptureRecordingId = recordingId; nativeWindowsCursorOffsetMs = 0; nativeWindowsCursorCaptureMode = cursorCaptureMode; @@ -2894,7 +3099,13 @@ export function registerIpcHandlers( cursorCaptureMode === "editable-overlay" ? Math.max(0, captureStartedAtMs - cursorStartTimeMs) : 0; - const webcamFormat = readWebcamFormat(nativeWindowsCaptureOutput); + // Index-aware readers: with several cameras the helper prints one + // format line and possibly one unavailable warning per camera, and + // only index 0 (or no index, from an old helper) is camera 1. + const webcamFormat = readWebcamFormatAt(nativeWindowsCaptureOutput, 0); + const unavailableWebcamIndices = new Set( + readUnavailableWebcamIndices(nativeWindowsCaptureOutput), + ); const encoderSelection = readNativeWindowsEncoderSelection(nativeWindowsCaptureOutput); // Captured now because stop may have no helper left to ask. A helper // killed mid-recording is exactly the case where this matters most. @@ -2903,7 +3114,13 @@ export function registerIpcHandlers( captureStartedAtMs, cursorOffsetMs: nativeWindowsCursorOffsetMs, webcamFormat, + additionalWebcamFormats: nativeWindowsCaptureAdditionalWebcamTargets.map((_, i) => + readWebcamFormatAt(nativeWindowsCaptureOutput, i + 1), + ), encoderSelection, + // Logged only: menus missing from a window take on Windows before 11 + // 24H2 are a platform limit, not something to put in front of the user. + secondaryWindowsApplied: readSecondaryWindowsApplied(nativeWindowsCaptureOutput), }); const source = selectedSource || { name: "Screen" }; @@ -2919,8 +3136,7 @@ export function registerIpcHandlers( // missing `webcamFormat`: absence of the format line also means "the // line could not be parsed", which would put a red toast on a recording // whose camera is working perfectly. - const webcamUnavailable = - request.webcam.enabled && readWebcamUnavailable(nativeWindowsCaptureOutput); + const webcamUnavailable = request.webcam.enabled && unavailableWebcamIndices.has(0); // Same shape as the camera notice: the helper records the Windows // default input rather than failing, so this take is usable but is // almost certainly the wrong microphone. @@ -2938,6 +3154,35 @@ export function registerIpcHandlers( deviceName: request.webcam.deviceName, }); } + // Helper indices follow the `webcams` list: camera 1 at 0, extras after. + const startedWebcams = [ + { path: webcamPath, label: request.webcam.deviceName ?? "" }, + ...nativeWindowsCaptureAdditionalWebcamTargets, + ]; + const unavailableWebcams = request.webcam.enabled + ? labelsOfUnavailableAdditionalWebcams(startedWebcams, [...unavailableWebcamIndices]) + : []; + // The helper deletes a dropped camera's file but may fail to; a stub + // left behind is removed at stop or discard if it is still empty. + nativeWindowsCaptureDroppedWebcamPaths = request.webcam.enabled + ? startedWebcams + .filter((_, i) => unavailableWebcamIndices.has(i)) + .map((camera) => camera.path) + : []; + if (unavailableWebcams.length > 0) { + console.warn( + "[native-wgc] recording without additional cameras the helper could not open", + { + unavailableWebcams, + }, + ); + // Already reported now; the helper deleted their files, so leaving + // them in the targets would report them a second time at stop. + nativeWindowsCaptureAdditionalWebcamTargets = + nativeWindowsCaptureAdditionalWebcamTargets.filter( + (_, i) => !unavailableWebcamIndices.has(i + 1), + ); + } return { success: true, @@ -2947,6 +3192,7 @@ export function registerIpcHandlers( videoEncoderSelection: encoderSelection?.video ?? null, videoEncoderRuntime: encoderSelection?.videoEncoderRuntime ?? null, webcamUnavailable, + ...(unavailableWebcams.length > 0 ? { unavailableWebcams } : {}), microphoneDefaulted, }; } catch (error) { @@ -2988,6 +3234,8 @@ export function registerIpcHandlers( // A source from Apple's picker records through the session that holds the pick, // and needs no Screen Recording grant -- so nothing here may go near one. const pickerSession = isMacPickerSourceId(request.source.sourceId) ? macPickerSession : null; + // Normally caught by `get-selected-source` first; this is the last word (#965). + pickerSession?.forgetSelectionUnlessExcluding(macPickerExcludedWindowIds()); const pick = pickerSession?.getSelection() ?? null; if (isMacPickerSourceId(request.source.sourceId) && !pick) { selectedSource = null; @@ -3108,12 +3356,21 @@ export function registerIpcHandlers( // When the take ends without the user — the helper reported an error or // exited — this drives the renderer's own stop, the same one the tray's Stop // Recording sends: it clears the HUD and surfaces the result. - attachNativeMacCaptureOutputDrain(proc, () => { - const hudWindow = getMainWindow(); - if (hudWindow && !hudWindow.isDestroyed()) { - hudWindow.webContents.send("stop-recording-from-tray"); - } - }); + attachNativeMacCaptureOutputDrain( + proc, + () => { + const hudWindow = getMainWindow(); + if (hudWindow && !hudWindow.isDestroyed()) { + hudWindow.webContents.send("stop-recording-from-tray"); + } + }, + () => { + const hudWindow = getMainWindow(); + if (hudWindow && !hudWindow.isDestroyed()) { + hudWindow.webContents.send("native-mac-system-audio-unavailable"); + } + }, + ); await waitForNativeMacCaptureStart(proc); const captureStartedAtMs = Date.now(); @@ -3271,6 +3528,17 @@ export function registerIpcHandlers( const proc = nativeWindowsCaptureProcess; const preferredPath = nativeWindowsCaptureTargetPath; const preferredWebcamPath = nativeWindowsCaptureWebcamTargetPath; + const additionalWebcamTargets = nativeWindowsCaptureAdditionalWebcamTargets; + const deskPath = nativeWindowsCaptureDeskWebcamPath; + const camera1Label = nativeWindowsCaptureWebcamLabel; + const droppedWebcamPaths = nativeWindowsCaptureDroppedWebcamPaths; + // Start-dropped cameras ride along so a discard or a failed stop also + // removes a stub the helper could not delete (both are empty). + const allWebcamPaths = [ + preferredWebcamPath, + ...additionalWebcamTargets.map((target) => target.path), + ...droppedWebcamPaths, + ]; const recordingId = nativeWindowsCaptureRecordingId ?? Date.now(); const cursorCaptureMode = nativeWindowsCursorCaptureMode; @@ -3293,7 +3561,7 @@ export function registerIpcHandlers( if (!exited) { detachNativeWindowsCaptureOutputDrain(); } - await removeNativeWindowsCaptureOutputs(preferredPath, preferredWebcamPath); + await removeNativeWindowsCaptureOutputs(preferredPath, allWebcamPaths); return { success: true, discarded: true }; } finally { // Unconditional. Killing a wedged helper can itself throw, and @@ -3367,7 +3635,7 @@ export function registerIpcHandlers( // explain. Size-gate it anyway: throwing away a recording to tidy // up after a failed stop is the worse mistake of the two, and the // gate is the same one the salvage check above uses. - await removeNativeWindowsCaptureOutputs(preferredPath, preferredWebcamPath, { + await removeNativeWindowsCaptureOutputs(preferredPath, allWebcamPaths, { onlyIfUnusable: true, }); // The helper log goes to console/diagnostics above, not into this @@ -3403,31 +3671,72 @@ export function registerIpcHandlers( shiftPendingCursorTelemetry(nativeWindowsCursorOffsetMs); await writePendingCursorTelemetry(screenVideoPath); } - let webcamVideoPath: string | undefined; - if (preferredWebcamPath) { - try { - // Size, not just existence. A camera that opened but delivered no - // frame still gets a file created for it, and its `Finalize()` then - // fails, leaving nought bytes on disk. Admitting that file put a - // camera track in the document pointing at something no demuxer can - // read, and the preview compositor answers an unreadable camera by - // drawing the SCREEN recording inside the little camera rectangle — - // which is how a webcam that never recorded showed up as the desktop - // duplicated into its own corner (getopenscreen/openscreen#387). - const webcamStat = await fs.stat(preferredWebcamPath); - webcamVideoPath = webcamStat.size > 0 ? preferredWebcamPath : undefined; - if (!webcamVideoPath) { - console.warn("[native-wgc] the webcam file is empty; saving without a camera", { - path: preferredWebcamPath, - }); - } - } catch { - webcamVideoPath = undefined; + // Size, not just existence. A camera that opened but delivered no frame + // still gets a file created for it, and its `Finalize()` then fails, + // leaving nought bytes on disk. Admitting that file put a camera track in + // the document pointing at something no demuxer can read, and the preview + // compositor answers an unreadable camera by drawing the SCREEN recording + // inside the little camera rectangle — which is how a webcam that never + // recorded showed up as the desktop duplicated into its own corner + // (getopenscreen/openscreen#387). Every camera is judged by its own file, + // not by the helper's list at stop (see `collectStoppedWebcams`). + const requestedWebcams = [ + ...(preferredWebcamPath ? [{ path: preferredWebcamPath, label: camera1Label }] : []), + ...additionalWebcamTargets, + ]; + const webcamSizes = new Map(); + for (const camera of requestedWebcams) { + const stat = await fs.stat(camera.path).catch(() => null); + if (stat) { + webcamSizes.set(camera.path, stat.size); } } - const session: RecordingSession = webcamVideoPath - ? { screenVideoPath, webcamVideoPath, createdAt: recordingId, cursorCaptureMode } - : { screenVideoPath, createdAt: recordingId, cursorCaptureMode }; + const stoppedWebcams = collectStoppedWebcams({ + camera1Enabled: Boolean(preferredWebcamPath), + requested: requestedWebcams, + sizes: webcamSizes, + }); + const webcamVideoPath = stoppedWebcams.camera1; + const additionalWebcams = stoppedWebcams.additional; + if (preferredWebcamPath && !webcamVideoPath && webcamSizes.has(preferredWebcamPath)) { + console.warn("[native-wgc] the webcam file is empty; saving without a camera", { + path: preferredWebcamPath, + }); + } + if (stoppedWebcams.dropped.length > 0) { + console.warn("[native-wgc] additional cameras produced nothing usable", { + dropped: stoppedWebcams.dropped, + helperWebcamPaths: readStoppedWebcamPaths(nativeWindowsCaptureOutput), + }); + } + // A camera the helper disabled mid-take keeps its partial file (it is + // in the take) but is named, so the user knows why it ends early. Only + // a helper that sends `webcamPaths` can tell; otherwise nothing is said. + const webcamsStoppedEarly = labelsOfWebcamsStoppedEarly({ + requested: requestedWebcams, + sizes: webcamSizes, + helperWebcamPaths: readReportedWebcamPaths(nativeWindowsCaptureOutput), + }); + if (webcamsStoppedEarly.length > 0) { + console.warn("[native-wgc] cameras stopped before the end of the take", { + webcamsStoppedEarly, + }); + } + await removeEmptyNativeWindowsWebcamFiles(droppedWebcamPaths); + // Generated by the start handler, never taken from the renderer or the + // helper; approved like the session's other media. + for (const extra of additionalWebcams) { + approveFilePath(extra.path); + } + const deskCamera = recordedDeskCamera(deskPath, webcamVideoPath, additionalWebcams); + const session: RecordingSession = { + screenVideoPath, + ...(webcamVideoPath ? { webcamVideoPath } : {}), + ...(additionalWebcams.length > 0 ? { additionalWebcams } : {}), + createdAt: recordingId, + cursorCaptureMode, + ...(deskCamera !== undefined ? { deskCamera } : {}), + }; setCurrentRecordingSessionState(session); currentProjectPath = null; @@ -3436,7 +3745,11 @@ export function registerIpcHandlers( `${path.parse(screenVideoPath).name}${RECORDING_SESSION_SUFFIX}`, ); await fs.writeFile(sessionManifestPath, JSON.stringify(session, null, 2), "utf-8"); - await registerRecordingMediaLinks(screenVideoPath, { webcamVideoPath, cursorCaptureMode }); + await registerRecordingMediaLinks(screenVideoPath, { + webcamVideoPath, + ...(additionalWebcams.length > 0 ? { additionalWebcams } : {}), + cursorCaptureMode, + }); return { success: true, @@ -3451,6 +3764,8 @@ export function registerIpcHandlers( // unreported, the user would find out in the editor — which is exactly // the silence this change exists to end. webcamDropped: Boolean(preferredWebcamPath) && !webcamVideoPath, + ...(stoppedWebcams.dropped.length > 0 ? { droppedWebcams: stoppedWebcams.dropped } : {}), + ...(webcamsStoppedEarly.length > 0 ? { webcamsStoppedEarly } : {}), message: recovered ? "Native Windows recording recovered from a failed stop" : "Native Windows recording session stored successfully", @@ -3919,7 +4234,7 @@ export function registerIpcHandlers( const files = await fs.readdir(RECORDINGS_DIR); const videoFiles = files.filter( - (file) => file.endsWith(".webm") && !file.endsWith("-webcam.webm"), + (file) => file.endsWith(".webm") && !isWebcamSidecarFile(file), ); if (videoFiles.length === 0) { @@ -4677,29 +4992,13 @@ export function registerIpcHandlers( // `addAsset` in the new editor's project store. ipcMain.handle( "find-recording-camera", - async ( - _event, - videoPath: string, - ): Promise<{ - success: boolean; - webcamVideoPath?: string; - offsetMs?: number; - error?: string; - }> => { + async (_event, videoPath: string): Promise => { try { const normalized = normalizeVideoSourcePath(videoPath); if (!normalized || !isPathAllowed(normalized)) { return { success: false, error: "Video path has not been approved" }; } - const resolution = await resolveMediaLinksForVideo(normalized); - if (!resolution.webcamVideoPath) { - return { success: false, error: "No camera attached to this recording" }; - } - return { - success: true, - webcamVideoPath: resolution.webcamVideoPath, - offsetMs: resolution.webcamOffsetMs ?? 0, - }; + return findRecordingCameraResult(await resolveMediaLinksForVideo(normalized)); } catch (err) { return { success: false, @@ -4822,6 +5121,21 @@ export function registerIpcHandlers( return aiEditionLlmConfigInstance; }; + // The local MCP server offers the agent's tools to MCP clients the user runs + // (Claude Code, Codex…). Built here because this is where the agent's own + // dependencies live, but NOT started here: the headless CLI shares this + // function and must never bind the port a running app is listening on. + // `main.ts` starts it. Its writes have their own switch, off by default and + // read on every call — separate from the in-app agent's "Project edits", so + // turning the server on grants a client read access and nothing more. + const mcpSettings = new McpSettingsStore(app.getPath("userData"), safeStorage); + const mcpController = new McpController(mcpSettings, { + host: new EditorDocumentHost(ipcMain), + editsAllowed: () => mcpSettings.getSettings().allowEdits, + cursor: agentCursorTelemetryReader, + version: app.getVersion(), + }); + registerNativeBridgeHandlers({ getPlatform: () => process.platform, getCurrentProjectPath: () => currentProjectPath, @@ -4877,5 +5191,8 @@ export function registerIpcHandlers( renameAiEditionChatSession: (projectId, sessionId, title) => renameSession(projectId, sessionId, title), deleteAiEditionChatSession: (projectId, sessionId) => deleteSession(projectId, sessionId), + getMcpController: () => mcpController, }); + + return { mcpController }; } diff --git a/electron/ipc/nativeBridge.mcp.test.ts b/electron/ipc/nativeBridge.mcp.test.ts new file mode 100644 index 0000000000..59b477c934 --- /dev/null +++ b/electron/ipc/nativeBridge.mcp.test.ts @@ -0,0 +1,82 @@ +// The MCP settings actions take renderer input, so the bridge checks its types +// before anything reaches the settings store: a bad value is an INVALID_REQUEST, +// not an INTERNAL_ERROR thrown further in. + +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { NativeBridgeResponse } from "../../src/native/contracts"; +import { type NativeBridgeContext, registerNativeBridgeHandlers } from "./nativeBridge"; + +const electron = vi.hoisted(() => ({ handle: vi.fn() })); +vi.mock("electron", () => ({ + app: { getAppPath: () => "", isPackaged: false }, + ipcMain: { handle: electron.handle, removeHandler: vi.fn() }, + shell: {}, +})); +vi.mock("../native-bridge/services/compositorViewService", () => ({ + CompositorViewService: class {}, +})); + +const service = vi.hoisted(() => ({ + mcpSetEnabled: vi.fn(async () => ({ ok: "enabled" })), + mcpSetPort: vi.fn(async () => ({ ok: "port" })), + mcpSetAllowEdits: vi.fn(async () => ({ ok: "allowEdits" })), +})); +vi.mock("../native-bridge/services/aiEditionService", () => ({ + AiEditionService: class { + mcpSetEnabled = service.mcpSetEnabled; + mcpSetPort = service.mcpSetPort; + mcpSetAllowEdits = service.mcpSetAllowEdits; + }, +})); + +let invoke: (action: string, payload: unknown) => Promise; + +beforeEach(() => { + electron.handle.mockClear(); + for (const fn of Object.values(service)) fn.mockClear(); + registerNativeBridgeHandlers({ + getPlatform: () => "linux", + getAiEditionDocuments: () => ({}), + getAiEditionLlmConfig: () => ({}), + } as unknown as NativeBridgeContext); + const handler = electron.handle.mock.calls[0]?.[1] as ( + event: unknown, + request: unknown, + ) => Promise; + invoke = (action, payload) => + handler({ sender: {} }, { domain: "aiEdition", action, payload, requestId: "r" }); +}); + +function errorCode(response: NativeBridgeResponse): string | undefined { + return response.ok ? undefined : response.error?.code; +} + +describe("native bridge MCP settings", () => { + it.each([80, 70000, 4.5, "47821", null])("rejects port %s as a bad request", async (port) => { + expect(errorCode(await invoke("mcp.setPort", { port }))).toBe("INVALID_REQUEST"); + expect(service.mcpSetPort).not.toHaveBeenCalled(); + }); + + it("passes a valid port on", async () => { + const response = await invoke("mcp.setPort", { port: 47821 }); + expect(response.ok).toBe(true); + expect(service.mcpSetPort).toHaveBeenCalledWith(47821); + }); + + it.each(["true", 1, undefined])("rejects enabled=%s as a bad request", async (enabled) => { + expect(errorCode(await invoke("mcp.setEnabled", { enabled }))).toBe("INVALID_REQUEST"); + expect(service.mcpSetEnabled).not.toHaveBeenCalled(); + }); + + it.each(["yes", 0, undefined])("rejects allowEdits=%s as a bad request", async (allowEdits) => { + expect(errorCode(await invoke("mcp.setAllowEdits", { allowEdits }))).toBe("INVALID_REQUEST"); + expect(service.mcpSetAllowEdits).not.toHaveBeenCalled(); + }); + + it("passes valid flags on", async () => { + await invoke("mcp.setEnabled", { enabled: true }); + await invoke("mcp.setAllowEdits", { allowEdits: false }); + expect(service.mcpSetEnabled).toHaveBeenCalledWith(true); + expect(service.mcpSetAllowEdits).toHaveBeenCalledWith(false); + }); +}); diff --git a/electron/ipc/nativeBridge.ts b/electron/ipc/nativeBridge.ts index e25ea048be..82e7c15a1b 100644 --- a/electron/ipc/nativeBridge.ts +++ b/electron/ipc/nativeBridge.ts @@ -13,6 +13,7 @@ import { import type { ChatEventSink } from "../ai-edition/chat-service"; import type { DocumentService } from "../ai-edition/document-service"; import { StylePresetError, type StylePresetService } from "../ai-edition/style-preset-service"; +import { isValidMcpPort } from "../mcp/mcp-settings-store"; import { type CursorTelemetryLoadResult, TelemetryCursorAdapter, @@ -62,6 +63,8 @@ export interface NativeBridgeContext { /** The one shared style preset service — it serialises writes per instance. */ getStylePresets: () => StylePresetService; getAiEditionLlmConfig: () => import("../ai-edition/llm-config-store").LlmConfigStore; + /** The local MCP server's controller. Absent in the headless CLI. */ + getMcpController?: () => import("../mcp/mcp-controller").McpController; runAiEditionChat: ( projectId: string, sessionId: string, @@ -234,6 +237,7 @@ export function registerNativeBridgeHandlers(context: NativeBridgeContext) { // Passed uncalled on purpose — invoking it here would build the store (and // hit the macOS Keychain) while wiring the bridge at startup. llmConfig: context.getAiEditionLlmConfig, + mcp: context.getMcpController?.(), runChat: context.runAiEditionChat, undoLastToolBatch: context.undoAiEditionToolBatch, rewindToMessage: context.rewindToMessage, @@ -380,18 +384,23 @@ export function registerNativeBridgeHandlers(context: NativeBridgeContext) { compositorViewService.setRect(request.payload.id, request.payload.rect); return createSuccessResponse(requestId, { ok: true }); case "readFrame": { - // The renderer polls this every rAF tick (~30fps). It passes the - // generation it last painted as `sinceGen`; native returns `null` when - // nothing newer exists (idle path — no buffer copy). On a new frame it - // returns `{ gen, width, height, data }`. The response wrapper does NOT - // JSON-stringify — `ipcMain.handle` round-trips via structured clone, - // which preserves the nested `Buffer` in `.data` as binary. - const frame = compositorViewService.readFrame( + // The renderer polls this every rAF tick. It passes the generation it + // last painted as `sinceGen`; native returns `null` when nothing newer + // exists (idle path — no buffer copy). On a new frame it returns + // `{ gen, width, height, data }`, or — for a view on shared textures — + // sends the texture to the asking frame and returns its receipt. The + // response wrapper does NOT JSON-stringify — `ipcMain.handle` round-trips + // via structured clone, which preserves the nested `Buffer` as binary. + const frame = await compositorViewService.readFrame( request.payload.id, request.payload.sinceGen, + event.senderFrame, ); return createSuccessResponse(requestId, frame); } + case "stopSharedFrames": + compositorViewService.stopSharedFrames(request.payload.id); + return createSuccessResponse(requestId, { ok: true }); case "setParam": compositorViewService.setParam( request.payload.id, @@ -416,6 +425,7 @@ export function registerNativeBridgeHandlers(context: NativeBridgeContext) { request.payload.webcamOffsetSec, request.payload.clipIndex, request.payload.sourceTimeSec, + request.payload.additionalCameras ?? [], ); return createSuccessResponse(requestId, { ok: true }); case "destroyView": @@ -573,6 +583,48 @@ export function registerNativeBridgeHandlers(context: NativeBridgeContext) { requestId, await aiEditionService.llmListProviderModels(request.payload.providerId), ); + case "mcp.getStatus": + return createSuccessResponse(requestId, await aiEditionService.mcpGetStatus()); + // Renderer input: check the types here, so a bad value is reported as + // the bad request it is rather than failing later as an internal error. + case "mcp.setEnabled": + if (typeof request.payload?.enabled !== "boolean") { + return createErrorResponse( + requestId, + "INVALID_REQUEST", + "Invalid MCP enabled flag.", + ); + } + return createSuccessResponse( + requestId, + await aiEditionService.mcpSetEnabled(request.payload.enabled), + ); + case "mcp.setPort": + if (!isValidMcpPort(request.payload?.port)) { + return createErrorResponse( + requestId, + "INVALID_REQUEST", + "MCP port must be a whole number between 1024 and 65535.", + ); + } + return createSuccessResponse( + requestId, + await aiEditionService.mcpSetPort(request.payload.port), + ); + case "mcp.setAllowEdits": + if (typeof request.payload?.allowEdits !== "boolean") { + return createErrorResponse( + requestId, + "INVALID_REQUEST", + "Invalid MCP edit permission.", + ); + } + return createSuccessResponse( + requestId, + await aiEditionService.mcpSetAllowEdits(request.payload.allowEdits), + ); + case "mcp.regenerateToken": + return createSuccessResponse(requestId, await aiEditionService.mcpRegenerateToken()); case "chat.run": { const sessionId = request.payload.sessionId; const sink = buildChatEventSink(event.sender, sessionId); diff --git a/electron/ipc/nativeMacMidCaptureErrorWatch.test.ts b/electron/ipc/nativeMacMidCaptureErrorWatch.test.ts index c6958cd4b1..6eaec1ef30 100644 --- a/electron/ipc/nativeMacMidCaptureErrorWatch.test.ts +++ b/electron/ipc/nativeMacMidCaptureErrorWatch.test.ts @@ -6,7 +6,7 @@ describe("createNativeMacMidCaptureErrorWatch", () => { it("fires on an error raised after recording started", () => { const onError = vi.fn(); - const watch = createNativeMacMidCaptureErrorWatch(() => true, onError); + const watch = createNativeMacMidCaptureErrorWatch(() => true, onError, vi.fn()); watch({ event: "ready" }); watch({ event: "recording-started" }); @@ -17,7 +17,7 @@ describe("createNativeMacMidCaptureErrorWatch", () => { it("leaves an error raised before recording started to the start wait", () => { const onError = vi.fn(); - const watch = createNativeMacMidCaptureErrorWatch(() => true, onError); + const watch = createNativeMacMidCaptureErrorWatch(() => true, onError, vi.fn()); watch(error); @@ -26,7 +26,7 @@ describe("createNativeMacMidCaptureErrorWatch", () => { it("ignores events that are not errors", () => { const onError = vi.fn(); - const watch = createNativeMacMidCaptureErrorWatch(() => true, onError); + const watch = createNativeMacMidCaptureErrorWatch(() => true, onError, vi.fn()); watch({ event: "recording-started" }); watch({ event: "warning", code: "stop-capture-failed" }); @@ -38,7 +38,7 @@ describe("createNativeMacMidCaptureErrorWatch", () => { it("ignores a helper that is no longer the current process", () => { const onError = vi.fn(); let current = true; - const watch = createNativeMacMidCaptureErrorWatch(() => current, onError); + const watch = createNativeMacMidCaptureErrorWatch(() => current, onError, vi.fn()); watch({ event: "recording-started" }); current = false; @@ -50,7 +50,7 @@ describe("createNativeMacMidCaptureErrorWatch", () => { /** Killed or crashed: no error line ever comes, only the process closing. */ it("fires when the helper exits in the middle of a take", () => { const onTakeEnded = vi.fn(); - const watch = createNativeMacMidCaptureErrorWatch(() => true, onTakeEnded); + const watch = createNativeMacMidCaptureErrorWatch(() => true, onTakeEnded, vi.fn()); watch({ event: "recording-started" }); watch.exited(); @@ -60,7 +60,7 @@ describe("createNativeMacMidCaptureErrorWatch", () => { it("leaves an exit before recording started to the start wait", () => { const onTakeEnded = vi.fn(); - const watch = createNativeMacMidCaptureErrorWatch(() => true, onTakeEnded); + const watch = createNativeMacMidCaptureErrorWatch(() => true, onTakeEnded, vi.fn()); watch.exited(); @@ -71,7 +71,7 @@ describe("createNativeMacMidCaptureErrorWatch", () => { it("ignores the exit of a take that is already being stopped", () => { const onTakeEnded = vi.fn(); let live = true; - const watch = createNativeMacMidCaptureErrorWatch(() => live, onTakeEnded); + const watch = createNativeMacMidCaptureErrorWatch(() => live, onTakeEnded, vi.fn()); watch({ event: "recording-started" }); live = false; @@ -79,4 +79,34 @@ describe("createNativeMacMidCaptureErrorWatch", () => { expect(onTakeEnded).not.toHaveBeenCalled(); }); + + it("surfaces a system-audio warning once without ending the take", () => { + const onTakeEnded = vi.fn(); + const onSystemAudioUnavailable = vi.fn(); + const watch = createNativeMacMidCaptureErrorWatch( + () => true, + onTakeEnded, + onSystemAudioUnavailable, + ); + + watch({ event: "recording-started" }); + watch({ event: "warning", code: "system-audio-unavailable" }); + watch({ event: "warning", code: "system-audio-unavailable" }); + + expect(onSystemAudioUnavailable).toHaveBeenCalledOnce(); + expect(onTakeEnded).not.toHaveBeenCalled(); + }); + + it("does not surface the system-audio warning for a take that is no longer live", () => { + const onSystemAudioUnavailable = vi.fn(); + const watch = createNativeMacMidCaptureErrorWatch( + () => false, + vi.fn(), + onSystemAudioUnavailable, + ); + + watch({ event: "warning", code: "system-audio-unavailable" }); + + expect(onSystemAudioUnavailable).not.toHaveBeenCalled(); + }); }); diff --git a/electron/ipc/nativeMacMidCaptureErrorWatch.ts b/electron/ipc/nativeMacMidCaptureErrorWatch.ts index 6c786d3f37..48f8628886 100644 --- a/electron/ipc/nativeMacMidCaptureErrorWatch.ts +++ b/electron/ipc/nativeMacMidCaptureErrorWatch.ts @@ -15,12 +15,23 @@ export function createNativeMacMidCaptureErrorWatch( isLiveTake: () => boolean, onTakeEnded: () => void, + onSystemAudioUnavailable: () => void, ) { let recordingStarted = false; + let reportedSystemAudioUnavailable = false; const watch = (event: Record) => { if (event.event === "recording-started") { recordingStarted = true; } + if ( + event.event === "warning" && + event.code === "system-audio-unavailable" && + !reportedSystemAudioUnavailable && + isLiveTake() + ) { + reportedSystemAudioUnavailable = true; + onSystemAudioUnavailable(); + } if (event.event === "error" && recordingStarted && isLiveTake()) { onTakeEnded(); } diff --git a/electron/ipc/recordingPrefs.test.ts b/electron/ipc/recordingPrefs.test.ts index fe8cdce9dc..19860f13ad 100644 --- a/electron/ipc/recordingPrefs.test.ts +++ b/electron/ipc/recordingPrefs.test.ts @@ -19,6 +19,9 @@ const defaults: RecordingPrefs = { camEnabled: false, camDeviceId: null, camDeviceName: null, + camAdditionalDevices: [], + camDeskDevice: null, + camQuality: "2160p", systemAudioEnabled: false, cursorCaptureMode: "editable-overlay", hideDesktopIcons: false, diff --git a/electron/lingeringOverlay.test.ts b/electron/lingeringOverlay.test.ts new file mode 100644 index 0000000000..37700033ff --- /dev/null +++ b/electron/lingeringOverlay.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from "vitest"; +import { isOnlyLingeringOverlay } from "./lingeringOverlay"; + +const window = (destroyed = false) => ({ isDestroyed: () => destroyed }); + +describe("isOnlyLingeringOverlay", () => { + it("is true when the hidden overlay is the last window open", () => { + const overlay = window(); + expect(isOnlyLingeringOverlay([overlay], overlay)).toBe(true); + expect(isOnlyLingeringOverlay([overlay, window(true)], overlay)).toBe(true); + }); + + it("is false while the HUD or the editor is still open", () => { + const overlay = window(); + expect(isOnlyLingeringOverlay([window(), overlay], overlay)).toBe(false); + }); + + it("is false without an overlay to close", () => { + expect(isOnlyLingeringOverlay([], null)).toBe(false); + const gone = window(true); + expect(isOnlyLingeringOverlay([gone], gone)).toBe(false); + }); +}); diff --git a/electron/lingeringOverlay.ts b/electron/lingeringOverlay.ts new file mode 100644 index 0000000000..35d64e64d6 --- /dev/null +++ b/electron/lingeringOverlay.ts @@ -0,0 +1,18 @@ +// The countdown overlay is created on the first take and only hidden between takes, so it is +// ready to paint the next countdown at once. A hidden window still counts for Electron, so +// once the HUD and the editor were gone `window-all-closed` never fired: OpenScreen stayed +// up with no window and only the tray icon left (issue #961). `main.ts` closes the overlay +// when it is the last window standing. + +/** The slice of `BrowserWindow` this needs, so the rule can be tested without Electron. */ +export type LingeringWindow = { isDestroyed(): boolean }; + +/** True when `overlay` is the only window still open, so it alone holds the app up. */ +export function isOnlyLingeringOverlay( + open: readonly LingeringWindow[], + overlay: LingeringWindow | null, +): boolean { + if (!overlay || overlay.isDestroyed()) return false; + const alive = open.filter((window) => !window.isDestroyed()); + return alive.length === 1 && alive[0] === overlay; +} diff --git a/electron/main.ts b/electron/main.ts index de3e3bbd75..e525aa0648 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -58,6 +58,7 @@ import { getSelectedDesktopSource, registerIpcHandlers, } from "./ipc/handlers"; +import { isOnlyLingeringOverlay } from "./lingeringOverlay"; import { installMainProcessErrorGuards } from "./main-process-errors"; import { showMessageBoxOver } from "./messageBox"; import { @@ -65,6 +66,7 @@ import { showPermissionsWindow, showPermissionsWindowIfNeeded, } from "./permissions"; +import { setDisplaySleepBlocked } from "./recording/displaySleepBlocker"; import { offersStarPrompt, REPO_URL, storeReviewUrl } from "./star-prompt"; import { registerSttIpc, shutdownStt } from "./stt"; import { checkLatestRelease } from "./update-checker"; @@ -1088,6 +1090,17 @@ if (!cliCommand) { app.on("window-all-closed", () => { app.quit(); }); + // The countdown overlay hides between takes instead of closing, so it would keep + // `window-all-closed` from ever firing (#961). Close it with the last other window. Every + // HUD/editor switch opens the next window before the previous one's `closed` arrives, so + // this only fires when the user really closed the last one. + app.on("browser-window-created", (_, win) => { + win.once("closed", () => { + if (isOnlyLingeringOverlay(BrowserWindow.getAllWindows(), countdownOverlayWindow)) { + countdownOverlayWindow?.close(); + } + }); + }); } app.on("activate", () => { @@ -1360,7 +1373,7 @@ appReady?.then(async () => { showMainWindow(); } - registerIpcHandlers( + const { mcpController } = registerIpcHandlers( createEditorWindowWrapper, createSourceSelectorWindowWrapper, createCountdownOverlayWindowWrapper, @@ -1372,6 +1385,7 @@ appReady?.then(async () => { (recording: boolean, sourceName: string) => { selectedSourceName = sourceName; isRecording = recording; + setDisplaySleepBlocked(recording); if (!tray) createTray(); updateTrayMenu(recording); // `canOfferUpdateCheck()` now answers "not mid-take" too, and the app/Help menus are @@ -1404,6 +1418,9 @@ appReady?.then(async () => { } createWindow(); + // Off unless the user turned it on in Settings → AI. Started here rather than + // in registerIpcHandlers so neither the headless CLI nor a bench run binds it. + void mcpController.startIfEnabled(); void showPermissionsWindowIfNeeded().catch((error) => console.warn("[permissions] could not read the permissions at launch:", error), ); diff --git a/electron/mcp/editor-document-host.test.ts b/electron/mcp/editor-document-host.test.ts new file mode 100644 index 0000000000..5b11cd19a0 --- /dev/null +++ b/electron/mcp/editor-document-host.test.ts @@ -0,0 +1,126 @@ +import { EventEmitter } from "node:events"; +import type { IpcMain, WebContents } from "electron"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createEmptyDocument } from "../../src/lib/ai-edition/schema"; +import { + AI_EDITION_MCP_HOST_CHANNEL, + AI_EDITION_MCP_REQUEST_CHANNEL, + AI_EDITION_MCP_RESPONSE_CHANNEL, + type AiEditionMcpHostRequest, +} from "../../src/native/contracts"; +import { EditorDocumentHost } from "./editor-document-host"; + +/** A webContents that records what main sends it and can be destroyed. */ +class FakeWebContents extends EventEmitter { + sent: AiEditionMcpHostRequest[] = []; + destroyed = false; + send(channel: string, payload: AiEditionMcpHostRequest) { + if (channel === AI_EDITION_MCP_REQUEST_CHANNEL) this.sent.push(payload); + } + isDestroyed() { + return this.destroyed; + } + destroy() { + this.destroyed = true; + this.emit("destroyed"); + } +} + +function setup() { + const ipc = new EventEmitter(); + const host = new EditorDocumentHost(ipc as unknown as IpcMain); + const fromRenderer = (sender: FakeWebContents, channel: string, payload: unknown) => + ipc.emit(channel, { sender: sender as unknown as WebContents }, payload); + return { host, fromRenderer }; +} + +/** Answers the next request the editor receives. */ +async function answer( + editor: FakeWebContents, + fromRenderer: ReturnType["fromRenderer"], + result: unknown, + sender = editor, +) { + await vi.waitFor(() => expect(editor.sent.length).toBeGreaterThan(0)); + const request = editor.sent.shift(); + fromRenderer(sender, AI_EDITION_MCP_RESPONSE_CHANNEL, { requestId: request?.requestId, result }); +} + +afterEach(() => { + vi.useRealTimers(); +}); + +describe("EditorDocumentHost", () => { + it("answers null while no editor has registered", async () => { + const { host } = setup(); + await expect(host.snapshot()).resolves.toBeNull(); + }); + + it("asks the registered editor for its snapshot", async () => { + const { host, fromRenderer } = setup(); + const editor = new FakeWebContents(); + fromRenderer(editor, AI_EDITION_MCP_HOST_CHANNEL, true); + const pending = host.snapshot(); + await answer(editor, fromRenderer, { document: { id: "d" }, revision: 3 }); + await expect(pending).resolves.toEqual({ document: { id: "d" }, revision: 3 }); + }); + + it("treats a snapshot without a numeric revision as no snapshot", async () => { + const { host, fromRenderer } = setup(); + const editor = new FakeWebContents(); + fromRenderer(editor, AI_EDITION_MCP_HOST_CHANNEL, true); + // A missing revision would reach the apply guard as `undefined` and skip it. + for (const revision of [undefined, "3", Number.NaN]) { + const pending = host.snapshot(); + await answer(editor, fromRenderer, { document: { id: "d" }, revision }); + await expect(pending).resolves.toBeNull(); + } + }); + + it("passes the apply verdict through", async () => { + const { host, fromRenderer } = setup(); + const editor = new FakeWebContents(); + fromRenderer(editor, AI_EDITION_MCP_HOST_CHANNEL, true); + const document = createEmptyDocument({ projectId: "p", title: "t" }); + const pending = host.apply(document, 7); + await vi.waitFor(() => expect(editor.sent).toHaveLength(1)); + expect(editor.sent[0]).toMatchObject({ op: "apply", expectedRevision: 7 }); + await answer(editor, fromRenderer, "conflict"); + await expect(pending).resolves.toBe("conflict"); + }); + + it("ignores a reply from any other window", async () => { + vi.useFakeTimers(); + const { host, fromRenderer } = setup(); + const editor = new FakeWebContents(); + fromRenderer(editor, AI_EDITION_MCP_HOST_CHANNEL, true); + const pending = host.snapshot(); + await answer(editor, fromRenderer, { document: {}, revision: 1 }, new FakeWebContents()); + await vi.advanceTimersByTimeAsync(30_000); + await expect(pending).resolves.toBeNull(); + }); + + it("forgets an editor that unregisters or is destroyed", async () => { + const { host, fromRenderer } = setup(); + const first = new FakeWebContents(); + fromRenderer(first, AI_EDITION_MCP_HOST_CHANNEL, true); + fromRenderer(first, AI_EDITION_MCP_HOST_CHANNEL, false); + await expect(host.snapshot()).resolves.toBeNull(); + + const second = new FakeWebContents(); + fromRenderer(second, AI_EDITION_MCP_HOST_CHANNEL, true); + second.destroy(); + await expect(host.snapshot()).resolves.toBeNull(); + expect(second.sent).toHaveLength(0); + }); + + it("reports an unanswered apply as a timeout, not as a failure", async () => { + vi.useFakeTimers(); + const { host, fromRenderer } = setup(); + const editor = new FakeWebContents(); + fromRenderer(editor, AI_EDITION_MCP_HOST_CHANNEL, true); + const pending = host.apply(createEmptyDocument({ projectId: "p", title: "t" }), 1); + await vi.advanceTimersByTimeAsync(30_000); + await expect(pending).resolves.toBe("timeout"); + }); +}); diff --git a/electron/mcp/editor-document-host.ts b/electron/mcp/editor-document-host.ts new file mode 100644 index 0000000000..55d6d61569 --- /dev/null +++ b/electron/mcp/editor-document-host.ts @@ -0,0 +1,107 @@ +// The MCP server's door onto the live document, which the editor window owns. +// +// The open project lives in the renderer's store (with the revision that guards +// agent writes), not in the main process, so each MCP call asks the editor for +// a snapshot and hands the result back to the editor's own revision-guarded +// apply — the same `applyAgentDocumentIfCurrent` an in-app chat turn uses. +// +// The editor announces itself on AI_EDITION_MCP_HOST_CHANNEL when it starts +// answering, and again with `false` when it stops; a destroyed window counts as +// stopped. Only that one webContents is ever asked, and only its replies count. + +import { randomUUID } from "node:crypto"; +import type { IpcMain, IpcMainEvent, WebContents } from "electron"; +import type { AxcutDocument } from "../../src/lib/ai-edition/schema"; +import { + AI_EDITION_MCP_HOST_CHANNEL, + AI_EDITION_MCP_REQUEST_CHANNEL, + AI_EDITION_MCP_RESPONSE_CHANNEL, + type AiEditionMcpHostRequest, + type AiEditionMcpHostResponse, + type AiEditionMcpHostSnapshot, +} from "../../src/native/contracts"; +import type { McpApplyResult, McpDocumentHost } from "./openscreen-mcp-server"; + +// Long enough for a save of a large project to land, short enough that a hung +// renderer does not hold the client's call open indefinitely. +const REQUEST_TIMEOUT_MS = 30_000; + +const TIMED_OUT = Symbol("timed-out"); + +interface Pending { + resolve: (value: unknown) => void; + timer: ReturnType; +} + +type RequestBody = + | { op: "snapshot" } + | { op: "apply"; document: unknown; expectedRevision: number }; + +export class EditorDocumentHost implements McpDocumentHost { + private editor: WebContents | null = null; + private readonly pending = new Map(); + + constructor(ipcMain: IpcMain) { + ipcMain.on(AI_EDITION_MCP_HOST_CHANNEL, (event: IpcMainEvent, active: unknown) => { + if (active === true) { + this.editor = event.sender; + event.sender.once("destroyed", () => { + if (this.editor === event.sender) this.editor = null; + }); + } else if (this.editor === event.sender) { + this.editor = null; + } + }); + ipcMain.on(AI_EDITION_MCP_RESPONSE_CHANNEL, (event: IpcMainEvent, message: unknown) => { + if (event.sender !== this.editor) return; + const response = message as AiEditionMcpHostResponse | null; + if (!response || typeof response.requestId !== "string") return; + const pending = this.pending.get(response.requestId); + if (!pending) return; + clearTimeout(pending.timer); + this.pending.delete(response.requestId); + pending.resolve(response.result); + }); + } + + private request(body: RequestBody): Promise { + const editor = this.editor; + if (!editor || editor.isDestroyed()) return Promise.resolve(undefined); + const requestId = randomUUID(); + return new Promise((resolve) => { + const timer = setTimeout(() => { + this.pending.delete(requestId); + resolve(TIMED_OUT); + }, REQUEST_TIMEOUT_MS); + this.pending.set(requestId, { resolve, timer }); + try { + editor.send(AI_EDITION_MCP_REQUEST_CHANNEL, { + requestId, + ...body, + } satisfies AiEditionMcpHostRequest); + } catch { + clearTimeout(timer); + this.pending.delete(requestId); + resolve(undefined); + } + }); + } + + async snapshot(): Promise { + const result = await this.request({ op: "snapshot" }); + if (!result || result === TIMED_OUT || typeof result !== "object") return null; + // The revision is the apply guard: one that is missing or not a number would + // reach `applyAgentDocumentIfCurrent` as `undefined`, which skips the + // stale-edit check entirely. Treat such a reply as no snapshot at all. + const snapshot = result as Partial; + if (typeof snapshot.revision !== "number" || !Number.isFinite(snapshot.revision)) return null; + return snapshot as AiEditionMcpHostSnapshot; + } + + async apply(document: AxcutDocument, expectedRevision: number): Promise { + const result = await this.request({ op: "apply", document, expectedRevision }); + if (result === TIMED_OUT) return "timeout"; + if (typeof result !== "string") return "no-editor"; + return result as McpApplyResult; + } +} diff --git a/electron/mcp/mcp-controller.test.ts b/electron/mcp/mcp-controller.test.ts new file mode 100644 index 0000000000..3a23bae090 --- /dev/null +++ b/electron/mcp/mcp-controller.test.ts @@ -0,0 +1,165 @@ +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { createServer, type Server } from "node:net"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { McpController } from "./mcp-controller"; +import { DEFAULT_MCP_PORT, McpSettingsStore, type McpTokenCrypto } from "./mcp-settings-store"; +import type { McpToolDeps } from "./openscreen-mcp-server"; + +// Reversible and visibly not plaintext, so a test can tell the token was encrypted. +const fakeCrypto: McpTokenCrypto = { + isEncryptionAvailable: () => true, + encryptString: (plain) => Buffer.from(`enc:${plain}`), + decryptString: (encrypted) => encrypted.toString().replace(/^enc:/, ""), +}; + +const deps: McpToolDeps = { + host: { snapshot: async () => null, apply: async () => "no-editor" }, + editsAllowed: () => true, + version: "0.0.0", +}; + +let dir: string; +let controller: McpController | null = null; +let blocker: Server | null = null; + +beforeEach(() => { + dir = mkdtempSync(path.join(os.tmpdir(), "openscreen-mcp-")); +}); + +afterEach(async () => { + await controller?.stop(); + controller = null; + await new Promise((resolve) => (blocker ? blocker.close(() => resolve()) : resolve())); + blocker = null; + rmSync(dir, { recursive: true, force: true }); +}); + +/** A free port, found by binding one and letting it go. */ +async function freePort(): Promise { + const server = createServer(); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const { port } = server.address() as { port: number }; + await new Promise((resolve) => server.close(() => resolve())); + return port; +} + +async function initializeStatus(url: string, token: string) { + const response = await fetch(url, { + method: "POST", + headers: { + Authorization: `Bearer ${token}`, + "Content-Type": "application/json", + Accept: "application/json, text/event-stream", + }, + body: JSON.stringify({ + jsonrpc: "2.0", + id: 1, + method: "initialize", + params: { + protocolVersion: "2025-06-18", + capabilities: {}, + clientInfo: { name: "t", version: "0" }, + }, + }), + }); + return response.status; +} + +describe("McpSettingsStore", () => { + it("is off by default, on the default port", () => { + const store = new McpSettingsStore(dir, fakeCrypto); + expect(store.getSettings()).toEqual({ + enabled: false, + port: DEFAULT_MCP_PORT, + allowEdits: false, + }); + }); + + it("keeps MCP edits off unless they were turned on explicitly", async () => { + expect(new McpSettingsStore(dir, fakeCrypto).getSettings().allowEdits).toBe(false); + await new McpSettingsStore(dir, fakeCrypto).setSettings({ enabled: true }); + expect(new McpSettingsStore(dir, fakeCrypto).getSettings().allowEdits).toBe(false); + await new McpSettingsStore(dir, fakeCrypto).setSettings({ allowEdits: true }); + expect(new McpSettingsStore(dir, fakeCrypto).getSettings().allowEdits).toBe(true); + }); + + it("keeps the token encrypted on disk and stable across instances", async () => { + const token = await new McpSettingsStore(dir, fakeCrypto).getToken(); + expect(readFileSync(path.join(dir, "mcp-token.enc"), "utf8")).toBe(`enc:${token}`); + expect(await new McpSettingsStore(dir, fakeCrypto).getToken()).toBe(token); + }); + + it("refuses a port outside the unprivileged range", async () => { + const store = new McpSettingsStore(dir, fakeCrypto); + await expect(store.setSettings({ port: 80 })).rejects.toThrow(/1024/); + }); + + it("will not mint a token without OS encryption", async () => { + const store = new McpSettingsStore(dir, { ...fakeCrypto, isEncryptionAvailable: () => false }); + await expect(store.getToken()).rejects.toThrow(/safeStorage/); + }); +}); + +describe("McpController", () => { + it("does not listen until enabled", async () => { + controller = new McpController(new McpSettingsStore(dir, fakeCrypto), deps); + await controller.startIfEnabled(); + const status = await controller.getStatus(); + expect(status.running).toBe(false); + expect(status.token).toBeNull(); + }); + + it("starts on enable and stops on disable", async () => { + const store = new McpSettingsStore(dir, fakeCrypto); + await store.setSettings({ port: await freePort() }); + controller = new McpController(store, deps); + + const on = await controller.setEnabled(true); + expect(on.running).toBe(true); + expect(await initializeStatus(on.url, on.token ?? "")).toBe(200); + + const off = await controller.setEnabled(false); + expect(off.running).toBe(false); + await expect(initializeStatus(on.url, on.token ?? "")).rejects.toThrow(); + }); + + it("explains a port that is already taken", async () => { + const port = await freePort(); + blocker = createServer(); + await new Promise((resolve) => blocker?.listen(port, "127.0.0.1", resolve)); + const store = new McpSettingsStore(dir, fakeCrypto); + await store.setSettings({ port }); + controller = new McpController(store, deps); + + const status = await controller.setEnabled(true); + expect(status.running).toBe(false); + expect(status.error).toContain(String(port)); + }); + + it("saves the edit permission without restarting the server", async () => { + const store = new McpSettingsStore(dir, fakeCrypto); + await store.setSettings({ port: await freePort() }); + controller = new McpController(store, deps); + const on = await controller.setEnabled(true); + expect(on.allowEdits).toBe(false); + + const allowed = await controller.setAllowEdits(true); + expect(allowed.allowEdits).toBe(true); + expect(allowed.token).toBe(on.token); + expect(await initializeStatus(allowed.url, allowed.token ?? "")).toBe(200); + }); + + it("locks out the old token once it is regenerated", async () => { + const store = new McpSettingsStore(dir, fakeCrypto); + await store.setSettings({ port: await freePort() }); + controller = new McpController(store, deps); + const before = await controller.setEnabled(true); + const after = await controller.regenerateToken(); + + expect(after.token).not.toBe(before.token); + expect(await initializeStatus(after.url, before.token ?? "")).toBe(401); + expect(await initializeStatus(after.url, after.token ?? "")).toBe(200); + }); +}); diff --git a/electron/mcp/mcp-controller.ts b/electron/mcp/mcp-controller.ts new file mode 100644 index 0000000000..49eb1e5386 --- /dev/null +++ b/electron/mcp/mcp-controller.ts @@ -0,0 +1,119 @@ +// Owns the MCP server's lifecycle: start it when enabled, restart it when its +// port or token changes, stop it when disabled, and report status to settings. +// +// Off by default. While it is off nothing listens and the token is never read, +// so a user who never turns it on never meets a Keychain prompt for it. + +import type { AiEditionMcpStatus } from "../../src/native/contracts"; +import type { McpSettingsStore } from "./mcp-settings-store"; +import { + MCP_ENDPOINT_PATH, + type McpToolDeps, + type RunningMcpServer, + startMcpHttpServer, +} from "./openscreen-mcp-server"; + +export class McpController { + private running: RunningMcpServer | null = null; + private error: string | null = null; + /** Serialises start/stop so a quick toggle cannot leave two servers bound. */ + private transition: Promise = Promise.resolve(); + + constructor( + private readonly store: McpSettingsStore, + private readonly deps: McpToolDeps, + ) {} + + private serially(task: () => Promise): Promise { + const next = this.transition.then(task, task); + this.transition = next.catch(() => undefined); + return next; + } + + private async stopNow(): Promise { + const running = this.running; + this.running = null; + await running?.close(); + } + + /** (Re)start from the stored settings — or stop, if they say disabled. */ + private async applySettings(): Promise { + await this.stopNow(); + this.error = null; + const { enabled, port } = this.store.getSettings(); + if (!enabled) return; + try { + const token = await this.store.getToken(); + this.running = await startMcpHttpServer({ port, token, deps: this.deps }); + } catch (error) { + const code = (error as NodeJS.ErrnoException | null)?.code; + this.error = + code === "EADDRINUSE" + ? `Port ${port} is already in use. Choose another port.` + : error instanceof Error + ? error.message + : String(error); + } + } + + startIfEnabled(): Promise { + return this.serially(() => this.applySettings()); + } + + stop(): Promise { + return this.serially(() => this.stopNow()); + } + + async getStatus(): Promise { + await this.transition; + const { enabled, port, allowEdits } = this.store.getSettings(); + let token: string | null = null; + if (enabled) { + try { + token = await this.store.getToken(); + } catch { + // Surfaced through `error` already, from the failed start. + } + } + return { + enabled, + port, + allowEdits, + running: this.running !== null, + url: `http://127.0.0.1:${port}${MCP_ENDPOINT_PATH}`, + token, + error: this.error, + }; + } + + async setEnabled(enabled: boolean): Promise { + await this.serially(async () => { + await this.store.setSettings({ enabled }); + await this.applySettings(); + }); + return this.getStatus(); + } + + async setPort(port: number): Promise { + await this.serially(async () => { + await this.store.setSettings({ port }); + await this.applySettings(); + }); + return this.getStatus(); + } + + /** No restart needed: the server reads it on every call. */ + async setAllowEdits(allowEdits: boolean): Promise { + await this.serially(() => this.store.setSettings({ allowEdits })); + return this.getStatus(); + } + + /** New token; the running server is restarted so the old one stops working at once. */ + async regenerateToken(): Promise { + await this.serially(async () => { + await this.store.regenerateToken(); + await this.applySettings(); + }); + return this.getStatus(); + } +} diff --git a/electron/mcp/mcp-settings-store.ts b/electron/mcp/mcp-settings-store.ts new file mode 100644 index 0000000000..4d8aaaa475 --- /dev/null +++ b/electron/mcp/mcp-settings-store.ts @@ -0,0 +1,109 @@ +// Settings for the local MCP server: whether it runs, on which port, and the +// bearer token a client must present. +// +// Same split as LlmConfigStore: the non-secret part is plain JSON, the token is +// a credential and goes through the OS keychain (`safeStorage`), never plain +// JSON. The encryption is injected rather than imported so this file stays +// testable without Electron. + +import { randomBytes } from "node:crypto"; +import { readFileSync } from "node:fs"; +import fs from "node:fs/promises"; +import path from "node:path"; + +export const DEFAULT_MCP_PORT = 47821; + +export interface McpSettings { + enabled: boolean; + port: number; + /** + * Whether MCP clients may run the tools that change the project. Off by + * default and separate from the in-app agent's "Project edits": turning the + * server on gives a client read access, and writing is a second decision. + */ + allowEdits: boolean; +} + +export interface McpTokenCrypto { + isEncryptionAvailable(): boolean; + encryptString(plain: string): Buffer; + decryptString(encrypted: Buffer): string; +} + +export function isValidMcpPort(port: unknown): port is number { + return typeof port === "number" && Number.isInteger(port) && port >= 1024 && port <= 65535; +} + +export class McpSettingsStore { + private readonly settingsPath: string; + private readonly tokenPath: string; + private settings: McpSettings; + private token: string | null = null; + + constructor( + userDataPath: string, + private readonly crypto: McpTokenCrypto, + ) { + this.settingsPath = path.join(userDataPath, "mcp-server.json"); + this.tokenPath = path.join(userDataPath, "mcp-token.enc"); + this.settings = this.readSettings(); + } + + private readSettings(): McpSettings { + try { + const raw = JSON.parse(readFileSync(this.settingsPath, "utf8")) as Partial; + return { + enabled: raw.enabled === true, + port: isValidMcpPort(raw.port) ? raw.port : DEFAULT_MCP_PORT, + allowEdits: raw.allowEdits === true, + }; + } catch { + return { enabled: false, port: DEFAULT_MCP_PORT, allowEdits: false }; + } + } + + getSettings(): McpSettings { + return { ...this.settings }; + } + + async setSettings(patch: Partial): Promise { + if (patch.port !== undefined && !isValidMcpPort(patch.port)) { + throw new Error("Port must be a whole number between 1024 and 65535."); + } + this.settings = { ...this.settings, ...patch }; + await fs.writeFile(this.settingsPath, JSON.stringify(this.settings, null, 2), "utf8"); + return this.getSettings(); + } + + /** + * The token, created on first use. Read lazily: on macOS the decrypt is a + * Keychain access, which must not happen for users who never turn MCP on. + */ + async getToken(): Promise { + if (this.token) return this.token; + try { + const encrypted = readFileSync(this.tokenPath); + if (this.crypto.isEncryptionAvailable()) { + const token = this.crypto.decryptString(encrypted); + if (token) { + this.token = token; + return token; + } + } + } catch { + // No token yet, or unreadable — mint a fresh one below. + } + return this.regenerateToken(); + } + + /** Replace the token. Every client configured with the old one stops working. */ + async regenerateToken(): Promise { + if (!this.crypto.isEncryptionAvailable()) { + throw new Error("safeStorage is not available on this platform."); + } + const token = randomBytes(32).toString("base64url"); + await fs.writeFile(this.tokenPath, this.crypto.encryptString(token)); + this.token = token; + return token; + } +} diff --git a/electron/mcp/openscreen-mcp-server.test.ts b/electron/mcp/openscreen-mcp-server.test.ts new file mode 100644 index 0000000000..f57b22c98a --- /dev/null +++ b/electron/mcp/openscreen-mcp-server.test.ts @@ -0,0 +1,252 @@ +// End-to-end over real HTTP: the SDK's own client against the server on an +// ephemeral port, with an in-memory stand-in for the editor window. + +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js"; +import { afterEach, describe, expect, it } from "vitest"; +import { + type AxcutDocument, + createEmptyDocument, + documentSchema, +} from "../../src/lib/ai-edition/schema"; +import { OPENSCREEN_TOOL_NAMES } from "../ai-edition/agent-tools"; +import { TOOL_DESCRIPTIONS } from "../ai-edition/deep-agent/service"; +import { + MCP_ENDPOINT_PATH, + type McpApplyResult, + type McpDocumentHost, + type RunningMcpServer, + startMcpHttpServer, +} from "./openscreen-mcp-server"; + +const TOKEN = "test-token-0123456789"; + +function fixtureDocument(): AxcutDocument { + const base = createEmptyDocument({ + title: "Test", + projectId: "proj_1", + createdAt: "2026-01-01T00:00:00.000Z", + }); + return documentSchema.parse({ + ...base, + project: { ...base.project, primaryAssetId: "asset_1" }, + assets: [ + { + id: "asset_1", + kind: "video", + label: "Recording", + originalPath: "/tmp/rec.mp4", + durationSec: 30, + }, + ], + timeline: { + ...base.timeline, + clips: [ + { + id: "clip_1", + assetId: "asset_1", + sourceStartSec: 0, + sourceEndSec: 30, + timelineStartSec: 0, + timelineEndSec: 30, + wordRefs: [], + origin: "user", + reason: "", + }, + ], + }, + }); +} + +/** The editor window, reduced to a document and the revision guarding it. */ +class FakeEditor implements McpDocumentHost { + document: AxcutDocument | null = fixtureDocument(); + revision = 1; + applied: AxcutDocument[] = []; + /** Runs between the snapshot and the apply — where a user edit would land. */ + onSnapshot?: () => void; + + async snapshot() { + if (!this.document) return null; + const snapshot = { document: structuredClone(this.document), revision: this.revision }; + this.onSnapshot?.(); + return snapshot; + } + + async apply(document: AxcutDocument, expectedRevision: number): Promise { + if (!this.document) return "no-live-document"; + if (expectedRevision !== this.revision) return "conflict"; + this.document = document; + this.revision += 1; + this.applied.push(document); + return "applied"; + } +} + +let running: RunningMcpServer | null = null; +let client: Client | null = null; + +afterEach(async () => { + await client?.close(); + await running?.close(); + client = null; + running = null; +}); + +async function connect( + editor: FakeEditor, + options: { editsAllowed?: boolean; headers?: Record } = {}, +) { + running = await startMcpHttpServer({ + port: 0, + token: TOKEN, + deps: { host: editor, editsAllowed: () => options.editsAllowed ?? true, version: "0.0.0" }, + }); + client = new Client({ name: "test", version: "0.0.0" }); + const url = new URL(`http://127.0.0.1:${running.port}${MCP_ENDPOINT_PATH}`); + await client.connect( + new StreamableHTTPClientTransport(url, { + requestInit: { headers: options.headers ?? { Authorization: `Bearer ${TOKEN}` } }, + }), + ); + return client; +} + +function resultText(result: Awaited>): string { + const content = result.content as Array<{ type: string; text?: string }>; + return content.map((part) => part.text ?? "").join(""); +} + +describe("the MCP tool surface", () => { + it("is exactly the in-app agent's tools, with its descriptions", async () => { + const mcp = await connect(new FakeEditor()); + const { tools } = await mcp.listTools(); + expect(tools.map((t) => t.name)).toEqual([...OPENSCREEN_TOOL_NAMES]); + for (const tool of tools) { + expect(tool.description).toBe(TOOL_DESCRIPTIONS[tool.name]); + expect(tool.inputSchema.type).toBe("object"); + } + // The zod schemas survive the trip to JSON Schema with their fields intact. + const addTrim = tools.find((t) => t.name === "addTrim"); + expect(Object.keys(addTrim?.inputSchema.properties ?? {})).toEqual( + expect.arrayContaining(["startSec", "endSec"]), + ); + }); + + it("marks reads read-only and deletions destructive", async () => { + const mcp = await connect(new FakeEditor()); + const { tools } = await mcp.listTools(); + const byName = new Map(tools.map((t) => [t.name, t.annotations])); + expect(byName.get("getCurrentDocument")?.readOnlyHint).toBe(true); + expect(byName.get("addTrim")?.readOnlyHint).toBe(false); + expect(byName.get("removeClip")?.destructiveHint).toBe(true); + }); + + it("hands the client the in-app agent's guidance as server instructions", async () => { + const mcp = await connect(new FakeEditor()); + expect(mcp.getInstructions()).toContain("Time-bases (do not mix them up)"); + }); +}); + +describe("calling a tool", () => { + it("reads the live editor document", async () => { + const mcp = await connect(new FakeEditor()); + const result = await mcp.callTool({ name: "getCurrentDocument", arguments: {} }); + expect(result.isError).toBeFalsy(); + expect(resultText(result)).toContain("clip_1"); + }); + + it("applies an edit to the editor, against the revision it read", async () => { + const editor = new FakeEditor(); + const mcp = await connect(editor); + const result = await mcp.callTool({ + name: "addTrim", + arguments: { assetId: "asset_1", startSec: 5, endSec: 6 }, + }); + expect(result.isError).toBeFalsy(); + expect(editor.applied).toHaveLength(1); + expect(editor.document?.timeline.trimRanges).toHaveLength(1); + expect(editor.revision).toBe(2); + }); + + it("does not touch the editor for a read", async () => { + const editor = new FakeEditor(); + const mcp = await connect(editor); + await mcp.callTool({ name: "getTranscript", arguments: {} }); + expect(editor.applied).toHaveLength(0); + }); + + it("refuses a write when the user has turned project edits off", async () => { + const editor = new FakeEditor(); + const mcp = await connect(editor, { editsAllowed: false }); + const result = await mcp.callTool({ + name: "addTrim", + arguments: { assetId: "asset_1", startSec: 5, endSec: 6 }, + }); + expect(result.isError).toBe(true); + expect(editor.applied).toHaveLength(0); + }); + + it("reports a conflict instead of overwriting a user edit made mid-call", async () => { + const editor = new FakeEditor(); + editor.onSnapshot = () => { + editor.revision += 1; + }; + const mcp = await connect(editor); + const result = await mcp.callTool({ + name: "addTrim", + arguments: { assetId: "asset_1", startSec: 5, endSec: 6 }, + }); + expect(result.isError).toBe(true); + expect(resultText(result)).toContain("NOT applied"); + expect(editor.applied).toHaveLength(0); + }); + + it("says so when no project is open", async () => { + const editor = new FakeEditor(); + editor.document = null; + const mcp = await connect(editor); + const result = await mcp.callTool({ name: "getCurrentDocument", arguments: {} }); + expect(result.isError).toBe(true); + expect(resultText(result)).toContain("No project is open"); + }); +}); + +describe("the HTTP guard", () => { + async function post(headers: Record, path = MCP_ENDPOINT_PATH) { + running = await startMcpHttpServer({ + port: 0, + token: TOKEN, + deps: { host: new FakeEditor(), editsAllowed: () => true, version: "0.0.0" }, + }); + return fetch(`http://127.0.0.1:${running.port}${path}`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json, text/event-stream", + ...headers, + }, + body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/list" }), + }); + } + + it("rejects a request without the token", async () => { + expect((await post({})).status).toBe(401); + }); + + it("rejects a wrong token", async () => { + expect((await post({ Authorization: "Bearer nope" })).status).toBe(401); + }); + + it("rejects a browser origin", async () => { + const response = await post({ + Authorization: `Bearer ${TOKEN}`, + Origin: "https://evil.example", + }); + expect(response.status).toBe(403); + }); + + it("rejects any path but the endpoint", async () => { + expect((await post({ Authorization: `Bearer ${TOKEN}` }, "/other")).status).toBe(404); + }); +}); diff --git a/electron/mcp/openscreen-mcp-server.ts b/electron/mcp/openscreen-mcp-server.ts new file mode 100644 index 0000000000..1e24700b97 --- /dev/null +++ b/electron/mcp/openscreen-mcp-server.ts @@ -0,0 +1,255 @@ +// The OpenScreen MCP server: the in-app agent's tools, offered to any MCP client +// (Claude Code, Codex, Cursor…) that the user runs themselves. +// +// Nothing here is a second implementation. The tool list, argument schemas, +// descriptions and guidance are the in-app agent's own (`TOOL_ARG_SCHEMAS`, +// `TOOL_DESCRIPTIONS`, `buildSystemPrompt`), and every call runs through +// `runDocumentTool` — the same executor, consent gate and cursor read the +// in-app agent uses. The one difference is where the document comes from: the +// in-app agent is handed a snapshot per chat turn, while here each call reads +// the live document from the editor window and writes the result back through +// the same revision-guarded apply, so a user edit landing mid-call is never +// overwritten and every edit is one undo step. +// +// The HTTP layer is local-only: bound to 127.0.0.1, a bearer token on every +// request, and a Host/Origin check so a web page cannot reach it by DNS +// rebinding. + +import { timingSafeEqual } from "node:crypto"; +import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http"; +import type { AddressInfo } from "node:net"; +import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js"; +import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; +import { type AxcutDocument, documentSchema } from "../../src/lib/ai-edition/schema"; +import { isMutatingTool } from "../ai-edition/agent-tools"; +import { + buildSystemPrompt, + type CursorTelemetryReader, + probeCursorTelemetry, + runDocumentTool, + TOOL_ARG_SCHEMAS, + TOOL_DESCRIPTIONS, +} from "../ai-edition/deep-agent/service"; + +export const MCP_SERVER_NAME = "openscreen"; +export const MCP_ENDPOINT_PATH = "/mcp"; + +export interface McpDocumentSnapshot { + document: unknown; + /** The editor's revision when the snapshot was taken — the apply guard. */ + revision: number; +} + +/** Mirrors the renderer's `AgentDocumentApplyResult`, plus the editor having gone + * away or not answering in time. */ +export type McpApplyResult = + | "applied" + | "conflict" + | "save-failed" + | "no-live-document" + | "no-editor" + | "timeout"; + +/** The live document, as held by the editor window. */ +export interface McpDocumentHost { + /** `null` when no editor window is open or it has no project loaded. */ + snapshot(): Promise; + apply(document: AxcutDocument, expectedRevision: number): Promise; +} + +export interface McpToolDeps { + host: McpDocumentHost; + /** The "Project edits" setting — the same one the in-app agent obeys. */ + editsAllowed(): boolean; + cursor?: CursorTelemetryReader; + version: string; +} + +const NO_PROJECT_MESSAGE = + "No project is open in the OpenScreen editor. Ask the user to open one in OpenScreen, then retry."; + +const APPLY_FAILURE_MESSAGES: Record, string> = { + conflict: + "The edit was NOT applied: the project changed in the editor while this call ran. Call getCurrentDocument to re-read it, then retry.", + "save-failed": "The edit was NOT applied: OpenScreen could not save the project.", + "no-live-document": NO_PROJECT_MESSAGE, + "no-editor": NO_PROJECT_MESSAGE, + // Not "NOT applied": the editor may have saved it and only the answer was lost. + timeout: + "OpenScreen did not confirm this edit in time. Call getCurrentDocument to see whether it landed before retrying.", +}; + +const DESTRUCTIVE_TOOLS: ReadonlySet = new Set([ + "replaceTimeline", + "removeTrim", + "removeModifier", + "removeClip", +]); + +const MCP_PREAMBLE = [ + "These tools act on the project currently open in the OpenScreen editor. Every edit is saved straight away and appears in the editor, where the user can undo it with Ctrl/Cmd+Z. Nothing here records, exports or imports media.", + "", +].join("\n"); + +function textResult(text: string, isError: boolean): CallToolResult { + return { content: [{ type: "text", text }], ...(isError ? { isError: true } : {}) }; +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +/** + * Runs tool calls one at a time. Each call is snapshot → execute → apply, and + * two of those interleaving would make the second apply against a revision the + * first just moved — a spurious conflict at best. + */ +export function createToolRunner(deps: McpToolDeps) { + let queue: Promise = Promise.resolve(); + + async function run(name: string, args: unknown): Promise { + const snapshot = await deps.host.snapshot(); + if (!snapshot) return textResult(NO_PROJECT_MESSAGE, true); + const parsed = documentSchema.safeParse(snapshot.document); + if (!parsed.success) { + return textResult("The project open in the editor could not be read.", true); + } + const document = parsed.data; + const availableByAssetId = await probeCursorTelemetry(document, deps.cursor); + const execution = await runDocumentTool(document, name, args, deps.editsAllowed(), { + cursor: deps.cursor, + availableByAssetId, + }); + if (execution.document) { + const applied = await deps.host.apply(execution.document, snapshot.revision); + if (applied !== "applied") return textResult(APPLY_FAILURE_MESSAGES[applied], true); + } + return textResult(execution.resultJson, !execution.ok); + } + + return (name: string, args: unknown): Promise => { + const next = queue.then( + () => run(name, args), + () => run(name, args), + ); + queue = next.catch(() => undefined); + return next.catch((error) => textResult(`Tool failed: ${errorMessage(error)}`, true)); + }; +} + +export function createOpenScreenMcpServer( + deps: McpToolDeps, + runTool: (name: string, args: unknown) => Promise, +): McpServer { + const server = new McpServer( + { name: MCP_SERVER_NAME, version: deps.version }, + { instructions: MCP_PREAMBLE + buildSystemPrompt({ editsAllowed: deps.editsAllowed() }) }, + ); + for (const [name, schema] of TOOL_ARG_SCHEMAS) { + const mutating = isMutatingTool(name); + server.registerTool( + name, + { + description: TOOL_DESCRIPTIONS[name], + inputSchema: schema, + annotations: { + readOnlyHint: !mutating, + destructiveHint: DESTRUCTIVE_TOOLS.has(name), + openWorldHint: false, + }, + }, + (args: unknown) => runTool(name, args), + ); + } + return server; +} + +function isAllowedHost(hostHeader: string | undefined, port: number): boolean { + return hostHeader === `127.0.0.1:${port}` || hostHeader === `localhost:${port}`; +} + +function isAllowedOrigin(origin: string | undefined, port: number): boolean { + // MCP clients are not browsers and send no Origin; a browser always does. + if (origin === undefined) return true; + return origin === `http://127.0.0.1:${port}` || origin === `http://localhost:${port}`; +} + +function hasValidToken(authorization: string | undefined, token: string): boolean { + const presented = authorization?.startsWith("Bearer ") ? authorization.slice(7).trim() : ""; + const a = Buffer.from(presented); + const b = Buffer.from(token); + return a.length === b.length && timingSafeEqual(a, b); +} + +function reject(res: ServerResponse, status: number, message: string, headers = {}): void { + res.writeHead(status, { "Content-Type": "application/json", ...headers }); + res.end(JSON.stringify({ jsonrpc: "2.0", error: { code: -32000, message }, id: null })); +} + +export interface RunningMcpServer { + port: number; + close(): Promise; +} + +export async function startMcpHttpServer(options: { + port: number; + token: string; + deps: McpToolDeps; +}): Promise { + const { token, deps } = options; + const runTool = createToolRunner(deps); + let boundPort = options.port; + + const handle = async (req: IncomingMessage, res: ServerResponse) => { + const pathname = new URL(req.url ?? "/", "http://127.0.0.1").pathname; + if (pathname !== MCP_ENDPOINT_PATH) return reject(res, 404, "Not found"); + if (!isAllowedHost(req.headers.host, boundPort)) return reject(res, 403, "Forbidden host"); + if (!isAllowedOrigin(req.headers.origin, boundPort)) { + return reject(res, 403, "Forbidden origin"); + } + if (!hasValidToken(req.headers.authorization, token)) { + return reject(res, 401, "Unauthorized", { "WWW-Authenticate": "Bearer" }); + } + + // Stateless: one server + transport per request, the SDK's documented shape + // for a server that keeps no per-session state. Tools read the live editor + // on every call, so there is nothing a session would need to remember. + const server = createOpenScreenMcpServer(deps, runTool); + const transport = new StreamableHTTPServerTransport({ + sessionIdGenerator: undefined, + enableJsonResponse: true, + }); + res.on("close", () => { + void transport.close(); + void server.close(); + }); + try { + await server.connect(transport); + await transport.handleRequest(req, res); + } catch (error) { + if (!res.headersSent) reject(res, 500, errorMessage(error)); + } + }; + + const httpServer: Server = createServer((req, res) => { + void handle(req, res); + }); + await new Promise((resolve, reject) => { + httpServer.once("error", reject); + httpServer.listen(options.port, "127.0.0.1", () => { + httpServer.off("error", reject); + resolve(); + }); + }); + boundPort = (httpServer.address() as AddressInfo).port; + + return { + port: boundPort, + close: () => + new Promise((resolve) => { + httpServer.closeAllConnections(); + httpServer.close(() => resolve()); + }), + }; +} diff --git a/electron/media/mediaLinksRegistry.test.ts b/electron/media/mediaLinksRegistry.test.ts index 7c76a75803..0aae452691 100644 --- a/electron/media/mediaLinksRegistry.test.ts +++ b/electron/media/mediaLinksRegistry.test.ts @@ -187,6 +187,29 @@ describe("mediaLinksRegistry", () => { } }); + it("round-trips additional cameras and resolves old records without them", async () => { + const screenPath = path.join(tempDir, "multi.webm"); + const oldScreenPath = path.join(tempDir, "old.webm"); + await writeFileOfSize(screenPath, 5000, "m"); + await writeFileOfSize(oldScreenPath, 5000, "o"); + const additionalWebcams = [ + { path: path.join(tempDir, "multi-webcam-2.mp4"), label: "Desk" }, + { path: path.join(tempDir, "multi-webcam-3.mp4"), label: "" }, + ]; + await registerMediaLinks(tempDir, screenPath, { + webcamVideoPath: path.join(tempDir, "multi-webcam.mp4"), + additionalWebcams, + }); + await registerMediaLinks(tempDir, oldScreenPath, { + webcamVideoPath: path.join(tempDir, "old-webcam.mp4"), + }); + + const resolved = await findMediaLinksByFingerprint(tempDir, screenPath); + expect(resolved?.additionalWebcams).toEqual(additionalWebcams); + const old = await findMediaLinksByFingerprint(tempDir, oldScreenPath); + expect(old).not.toHaveProperty("additionalWebcams"); + }); + it("returns null when there is no matching fingerprint", async () => { const unknownPath = path.join(tempDir, "unknown.webm"); await writeFileOfSize(unknownPath, 1000, "z"); diff --git a/electron/media/mediaLinksRegistry.ts b/electron/media/mediaLinksRegistry.ts index ec0ad76de5..56bd7d57f0 100644 --- a/electron/media/mediaLinksRegistry.ts +++ b/electron/media/mediaLinksRegistry.ts @@ -19,7 +19,11 @@ import fs from "node:fs/promises"; import path from "node:path"; -import type { CursorCaptureMode } from "../../src/lib/recordingSession"; +import { + type AdditionalWebcam, + type CursorCaptureMode, + normalizeAdditionalWebcams, +} from "../../src/lib/recordingSession"; // ponytail: `baseDir` is passed in by every caller (RECORDINGS_DIR in // electron/ipc/handlers.ts) rather than imported here, so this module has no @@ -40,6 +44,7 @@ export interface MediaLinkEntry { fingerprint: MediaFingerprint; webcamVideoPath?: string; webcamOffsetMs?: number; + additionalWebcams?: AdditionalWebcam[]; cursorTelemetryPath?: string; cursorCaptureMode?: CursorCaptureMode; updatedAt: string; @@ -98,6 +103,7 @@ function normalizeEntry(candidate: unknown): MediaLinkEntry | null { if (!candidate || typeof candidate !== "object") return null; const raw = candidate as Partial; const fp = raw.fingerprint; + const additionalWebcams = normalizeAdditionalWebcams(raw.additionalWebcams); if ( typeof raw.lastKnownPath !== "string" || !fp || @@ -116,6 +122,7 @@ function normalizeEntry(candidate: unknown): MediaLinkEntry | null { }, ...(typeof raw.webcamVideoPath === "string" ? { webcamVideoPath: raw.webcamVideoPath } : {}), ...(typeof raw.webcamOffsetMs === "number" ? { webcamOffsetMs: raw.webcamOffsetMs } : {}), + ...(additionalWebcams.length > 0 ? { additionalWebcams } : {}), ...(typeof raw.cursorTelemetryPath === "string" ? { cursorTelemetryPath: raw.cursorTelemetryPath } : {}), @@ -220,6 +227,7 @@ async function updateRegistry( export interface MediaLinksToRegister { webcamVideoPath?: string; webcamOffsetMs?: number; + additionalWebcams?: AdditionalWebcam[]; cursorTelemetryPath?: string; cursorCaptureMode?: CursorCaptureMode; } @@ -235,7 +243,12 @@ export async function registerMediaLinks( videoPath: string, links: MediaLinksToRegister, ): Promise { + // Extras alone register nothing. That relies on R6: additional cameras are + // recorded only while camera 1 is on, so a take with extras but no camera 1 + // is one whose camera 1 file came out empty — a rare loss accepted here. if (!links.webcamVideoPath && !links.cursorTelemetryPath) return; + const { additionalWebcams: rawAdditionalWebcams, ...linksWithoutAdditional } = links; + const additionalWebcams = normalizeAdditionalWebcams(rawAdditionalWebcams); const fingerprint = await computeFingerprint(videoPath); await updateRegistry(baseDir, (file) => { const existingIndex = file.entries.findIndex((e) => @@ -244,7 +257,8 @@ export async function registerMediaLinks( const entry: MediaLinkEntry = { lastKnownPath: videoPath, fingerprint, - ...links, + ...linksWithoutAdditional, + ...(additionalWebcams.length > 0 ? { additionalWebcams } : {}), updatedAt: new Date().toISOString(), }; const entries = @@ -258,6 +272,7 @@ export async function registerMediaLinks( export interface MediaLinksLookup { webcamVideoPath?: string; webcamOffsetMs?: number; + additionalWebcams?: AdditionalWebcam[]; cursorTelemetryPath?: string; cursorCaptureMode?: CursorCaptureMode; } @@ -314,6 +329,7 @@ export async function findRelocatedMediaByStoredPath( screenVideoPath: match.lastKnownPath, ...(match.webcamVideoPath ? { webcamVideoPath: match.webcamVideoPath } : {}), ...(typeof match.webcamOffsetMs === "number" ? { webcamOffsetMs: match.webcamOffsetMs } : {}), + ...(match.additionalWebcams?.length ? { additionalWebcams: match.additionalWebcams } : {}), ...(match.cursorTelemetryPath ? { cursorTelemetryPath: match.cursorTelemetryPath } : {}), ...(match.cursorCaptureMode ? { cursorCaptureMode: match.cursorCaptureMode } : {}), }; @@ -358,6 +374,7 @@ export async function findMediaLinksByFingerprint( return { ...(match.webcamVideoPath ? { webcamVideoPath: match.webcamVideoPath } : {}), ...(typeof match.webcamOffsetMs === "number" ? { webcamOffsetMs: match.webcamOffsetMs } : {}), + ...(match.additionalWebcams?.length ? { additionalWebcams: match.additionalWebcams } : {}), ...(match.cursorTelemetryPath ? { cursorTelemetryPath: match.cursorTelemetryPath } : {}), ...(match.cursorCaptureMode ? { cursorCaptureMode: match.cursorCaptureMode } : {}), }; diff --git a/electron/media/projectMediaRelinker.test.ts b/electron/media/projectMediaRelinker.test.ts index 2b52d32b51..505a430f43 100644 --- a/electron/media/projectMediaRelinker.test.ts +++ b/electron/media/projectMediaRelinker.test.ts @@ -62,6 +62,54 @@ describe("relinkProjectMedia", () => { expect(logged.join("\n")).toContain(currentWebcamPath); }); + it("relinks additional cameras by index and leaves absent ones alone", async () => { + const currentScreenPath = path.join(tempDir, "recording-43.mp4"); + const currentWebcamPath = path.join(tempDir, "recording-43-webcam.mp4"); + const currentExtra2 = path.join(tempDir, "recording-43-webcam-2.mp4"); + const currentExtra3 = path.join(tempDir, "recording-43-webcam-3.mp4"); + await fs.writeFile(currentScreenPath, "screen bytes"); + await fs.writeFile(currentWebcamPath, "webcam bytes"); + await fs.writeFile(currentExtra2, "extra 2"); + await fs.writeFile(currentExtra3, "extra 3"); + await registerMediaLinks(tempDir, currentScreenPath, { + webcamVideoPath: currentWebcamPath, + additionalWebcams: [ + { path: currentExtra2, label: "Desk" }, + { path: currentExtra3, label: "Wide" }, + ], + }); + + const project = { + assets: [ + { + id: "asset-1", + originalPath: "C:\\Users\\demo\\recording-43.mp4", + sizeBytes: Buffer.byteLength("screen bytes"), + cameraTrack: { + sourcePath: "C:\\Users\\demo\\recording-43-webcam.mp4", + startMs: 0, + offsetMs: 0, + visible: true, + }, + additionalCameraTracks: [ + { sourcePath: "C:\\Users\\demo\\recording-43-webcam-2.mp4", label: "Desk" }, + { sourcePath: "C:\\Users\\demo\\recording-43-webcam-3.mp4", label: "Wide" }, + ], + }, + ], + }; + + const relinked = (await relinkProjectMedia(project, tempDir)) as typeof project; + + expect(relinked.assets[0].cameraTrack.sourcePath).toBe(currentWebcamPath); + expect(relinked.assets[0].additionalCameraTracks.map((t) => t.sourcePath)).toEqual([ + currentExtra2, + currentExtra3, + ]); + expect(relinked.assets[0].additionalCameraTracks.map((t) => t.label)).toEqual(["Desk", "Wide"]); + expect(project.assets[0].additionalCameraTracks[0].sourcePath).toContain("demo"); + }); + it("refuses to relink an asset the document recorded no size for", async () => { // A same-named recording exists and is registered with its webcam, so a // basename match would resolve — that is exactly what must not happen. The diff --git a/electron/media/projectMediaRelinker.ts b/electron/media/projectMediaRelinker.ts index 0c0b8c259f..bef7dad02d 100644 --- a/electron/media/projectMediaRelinker.ts +++ b/electron/media/projectMediaRelinker.ts @@ -41,11 +41,23 @@ async function resolveAssetMedia( isRecord(cameraTrack) && typeof cameraTrack.sourcePath === "string" && cameraTrack.sourcePath ? cameraTrack.sourcePath : null; + const additionalTracks = Array.isArray(asset.additionalCameraTracks) + ? asset.additionalCameraTracks + : []; + const additionalMissing = await Promise.all( + additionalTracks.map( + async (track) => + isRecord(track) && + typeof track.sourcePath === "string" && + track.sourcePath !== "" && + !(await fileExists(track.sourcePath)), + ), + ); const screenExists = await fileExists(originalPath); const cameraMissing = cameraPath !== null && !(await fileExists(cameraPath)); // Nothing to repair, and this runs on every project open — don't fingerprint // (i.e. open and read) every asset just to confirm what the stats already say. - if (screenExists && !cameraMissing) return asset; + if (screenExists && !cameraMissing && !additionalMissing.some(Boolean)) return asset; let links: RelocatedMediaLookup | null = null; if (screenExists) { @@ -81,10 +93,25 @@ async function resolveAssetMedia( nextCameraTrack = { ...cameraTrack, sourcePath: links.webcamVideoPath }; } + // Extras are matched by index, the same order the recording registered them in. + let additionalChanged = false; + const nextAdditionalTracks = await Promise.all( + additionalTracks.map(async (track, index) => { + const replacement = links.additionalWebcams?.[index]?.path; + if (!additionalMissing[index] || !replacement || !(await fileExists(replacement))) { + return track; + } + console.log(`[media-relink] additional webcam ${index + 2} -> ${replacement}`); + additionalChanged = true; + return { ...track, sourcePath: replacement }; + }), + ); + return { ...asset, originalPath: links.screenVideoPath, ...(nextCameraTrack === cameraTrack ? {} : { cameraTrack: nextCameraTrack }), + ...(additionalChanged ? { additionalCameraTracks: nextAdditionalTracks } : {}), }; } diff --git a/electron/native-bridge/screen/macPickerSession.test.ts b/electron/native-bridge/screen/macPickerSession.test.ts index c5945bedf4..adac0495b7 100644 --- a/electron/native-bridge/screen/macPickerSession.test.ts +++ b/electron/native-bridge/screen/macPickerSession.test.ts @@ -199,6 +199,57 @@ describe("MacPickerSession", () => { expect(helper.commands.at(-1)).toBe("stop"); }); + it("forgets a screen pick once an app window it does not leave out must be (#965)", async () => { + const { session } = await pickDisplay(); + + // The windows the picker was told about, in any order, or fewer of them: still good. + session.forgetSelectionUnlessExcluding([8, 7]); + session.forgetSelectionUnlessExcluding([7]); + expect(session.getSelection()).not.toBeNull(); + + // The HUD came back from the editor as a new window: the pick would record it. + session.forgetSelectionUnlessExcluding([9]); + expect(session.getSelection()).toBeNull(); + }); + + it("keeps the exclusions of the pick, not of a picker that was cancelled", async () => { + const { helper, session } = await pickDisplay(); + const cancelled = session.present([9]); + await flush(); + helper.say({ event: "picker-cancelled" }); + expect(await cancelled).toBeNull(); + + session.forgetSelectionUnlessExcluding([7, 8]); + expect(session.getSelection()).not.toBeNull(); + session.forgetSelectionUnlessExcluding([9]); + expect(session.getSelection()).toBeNull(); + }); + + it("keeps a window pick whatever the app's windows are", async () => { + const { helper, session } = await readySession(); + const pick = session.present([7]); + await flush(); + helper.say({ ...DISPLAY_PICK, kind: "window", windowId: 42 }); + expect(await pick).not.toBeNull(); + + session.forgetSelectionUnlessExcluding([9]); + expect(session.getSelection()).toMatchObject({ kind: "window" }); + }); + + it("keeps the pick while a take is running from it", async () => { + const { helper, session } = await pickDisplay(); + session.startTake({}); + + // The notes window opened mid-take: the running take still reads the pick's frame. + session.forgetSelectionUnlessExcluding([7, 8, 9]); + expect(session.getSelection()).not.toBeNull(); + + helper.say({ event: "take-ended" }); + await flush(); + session.forgetSelectionUnlessExcluding([7, 8, 9]); + expect(session.getSelection()).toBeNull(); + }); + it("ends a running take and forgets the pick when the session dies", async () => { const { helper, session } = await pickDisplay(); const take = session.startTake({}); diff --git a/electron/native-bridge/screen/macPickerSession.ts b/electron/native-bridge/screen/macPickerSession.ts index 0f0e5b1861..7b7aea7584 100644 --- a/electron/native-bridge/screen/macPickerSession.ts +++ b/electron/native-bridge/screen/macPickerSession.ts @@ -186,6 +186,10 @@ export class MacPickerSession { private proc: ChildProcessWithoutNullStreams | null = null; private starting: Promise | null = null; private selection: MacPickerSelection | null = null; + /** The windows the picker was told to leave out, as of the last `present`. */ + private presentedExclusions: readonly number[] = []; + /** The windows the retained pick leaves out: baked into its filter, fixed until the next pick. */ + private selectionExclusions: readonly number[] = []; private take: TakeProcess | null = null; private pendingPick: ((selection: MacPickerSelection | null) => void) | null = null; private lineBuffer = ""; @@ -199,6 +203,26 @@ export class MacPickerSession { return this.selection; } + /** + * Forgets a screen pick that does not leave out every one of `windowIds`. + * + * The windows a screen pick leaves out are the ones the picker was told about when it + * was shown, baked into the pick's filter. A take cannot add one: a fresh filter needs + * the Screen Recording grant this picker exists to avoid. So once the HUD or the notes + * window is not one the pick knows -- the HUD is a new window after every trip through + * the editor -- the pick would record it, and only picking again leaves it out (#965). + * A window pick records that one window and is never affected, and neither is a take + * already running: its filter is set, and its cursor telemetry still reads the pick. + */ + forgetSelectionUnlessExcluding(windowIds: readonly number[]) { + if (this.selection?.kind !== "display" || (this.take && !this.take.ended)) { + return; + } + if (!windowIds.every((windowId) => this.selectionExclusions.includes(windowId))) { + this.selection = null; + } + } + /** Starts the helper if it is not running. Resolves false when it cannot. */ start(): Promise { if (this.proc) { @@ -286,6 +310,7 @@ export class MacPickerSession { resolve(selection); }; }); + this.presentedExclusions = [...excludedWindowIds]; this.send({ command: "present", excludedWindowIds, @@ -335,6 +360,7 @@ export class MacPickerSession { const selection = parsePickerSelection(event); if (selection) { this.selection = selection; + this.selectionExclusions = this.presentedExclusions; } this.resolvePick(selection); return; diff --git a/electron/native-bridge/services/aiEditionService.ts b/electron/native-bridge/services/aiEditionService.ts index 90088781a4..579ba59a30 100644 --- a/electron/native-bridge/services/aiEditionService.ts +++ b/electron/native-bridge/services/aiEditionService.ts @@ -13,6 +13,7 @@ import type { AiEditionLlmConfig, AiEditionLlmDisconnectResult, AiEditionLlmSnapshot, + AiEditionMcpStatus, AiEditionProjectSummary, } from "../../../src/native/contracts"; import { @@ -31,6 +32,7 @@ import { probeMiniMaxModels, } from "../../ai-edition/llm-provider-auth"; import { PROVIDER_DEFINITIONS } from "../../ai-edition/provider-registry"; +import type { McpController } from "../../mcp/mcp-controller"; export interface AiEditionServiceOptions { documents: DocumentService; @@ -80,6 +82,8 @@ export interface AiEditionServiceOptions { title: string, ) => AiEditionChatSessionSummary | null; deleteSession: (projectId: string, sessionId: string) => boolean; + /** The local MCP server. Absent where no server is wired (tests, headless CLI). */ + mcp?: McpController; } export class AiEditionService { @@ -271,6 +275,31 @@ export class AiEditionService { } } + private get mcp(): McpController { + if (!this.options.mcp) throw new Error("The MCP server is not available in this build."); + return this.options.mcp; + } + + mcpGetStatus(): Promise { + return this.mcp.getStatus(); + } + + mcpSetEnabled(enabled: boolean): Promise { + return this.mcp.setEnabled(enabled); + } + + mcpSetPort(port: number): Promise { + return this.mcp.setPort(port); + } + + mcpSetAllowEdits(allowEdits: boolean): Promise { + return this.mcp.setAllowEdits(allowEdits); + } + + mcpRegenerateToken(): Promise { + return this.mcp.regenerateToken(); + } + async chatRun( projectId: string, sessionId: string, diff --git a/electron/native-bridge/services/compositorViewService.test.ts b/electron/native-bridge/services/compositorViewService.test.ts index eee8fe3052..4a099a7f9b 100644 --- a/electron/native-bridge/services/compositorViewService.test.ts +++ b/electron/native-bridge/services/compositorViewService.test.ts @@ -1,6 +1,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import type { WebFrameMain } from "electron"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { CURSOR_THEMES, DEFAULT_CURSOR_SPRITES } from "../../../src/lib/cursor/cursorThemes"; import type { CompositorViewAddon, GifExportStats } from "../../native/compositor-view/addon"; @@ -58,6 +59,166 @@ describe("native GIF cancellation capability", () => { }); }); +describe("CompositorViewService frames handed over as shared GPU textures", () => { + const target = { frameTreeNodeId: 1 } as unknown as WebFrameMain; + const sharedFrame = { + gen: 3, + slot: 1, + handle: Buffer.from([1, 2, 3, 4, 5, 6, 7, 8]), + width: 4, + height: 2, + footage: null, + footageProjective: false, + }; + + function setup(options: { addon?: Partial; gpuCompositing?: boolean } = {}) { + const addon = { + createView: vi.fn(() => 7), + setSharedFrames: vi.fn(() => true), + readSharedFrame: vi.fn(() => sharedFrame), + readFrame: vi.fn(() => null), + releaseSharedFrame: vi.fn(), + destroyView: vi.fn(), + ...options.addon, + }; + const imported = { release: vi.fn() }; + let allReferencesReleased: (() => void) | undefined; + const sharedTexture = { + importSharedTexture: vi.fn((importOptions: Electron.ImportSharedTextureOptions) => { + allReferencesReleased = importOptions.allReferencesReleased as () => void; + return imported as unknown as Electron.SharedTextureImported; + }), + sendSharedTexture: vi.fn(async () => undefined), + }; + const service = new CompositorViewService({ + addon: addon as unknown as CompositorViewAddon, + sharedTexture, + gpuCompositing: () => options.gpuCompositing ?? true, + }); + const id = service.createView({ x: 0, y: 0, width: 4, height: 2 }); + return { + addon, + id, + imported, + service, + sharedTexture, + releaseEverywhere: () => allReferencesReleased?.(), + }; + } + + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it("sends the texture to the frame that asked, and answers with its receipt", async () => { + const { addon, id, imported, service, sharedTexture } = setup(); + expect(addon.setSharedFrames).toHaveBeenCalledWith(id, true); + + const receipt = await service.readFrame(id, 2, target); + + expect(addon.readSharedFrame).toHaveBeenCalledWith(id, 2); + expect(sharedTexture.importSharedTexture).toHaveBeenCalledWith( + expect.objectContaining({ + textureInfo: { + pixelFormat: "rgba", + codedSize: { width: 4, height: 2 }, + handle: { ntHandle: sharedFrame.handle }, + }, + }), + ); + const meta = { + viewId: id, + gen: 3, + width: 4, + height: 2, + footage: null, + footageProjective: false, + }; + expect(sharedTexture.sendSharedTexture).toHaveBeenCalledWith( + { frame: target, importedSharedTexture: imported }, + meta, + ); + expect(receipt).toEqual({ ...meta, shared: true }); + // This process's reference goes at once: the renderer holds its own. + expect(imported.release).toHaveBeenCalled(); + expect(addon.releaseSharedFrame).not.toHaveBeenCalled(); + }); + + it("hands the slot back once Chromium has let go of the texture everywhere", async () => { + const { addon, id, releaseEverywhere, service } = setup(); + await service.readFrame(id, 0, target); + + releaseEverywhere(); + + expect(addon.releaseSharedFrame).toHaveBeenCalledWith(id, 1, 3); + }); + + it("goes back to read-back when a delivery fails", async () => { + const { addon, id, service, sharedTexture } = setup(); + sharedTexture.sendSharedTexture.mockRejectedValueOnce(new Error("timed out after 1000ms")); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + try { + expect(await service.readFrame(id, 0, target)).toBeNull(); + } finally { + warn.mockRestore(); + } + expect(addon.setSharedFrames).toHaveBeenLastCalledWith(id, false); + + vi.mocked(addon.readSharedFrame).mockClear(); + await service.readFrame(id, 3, target); + expect(addon.readSharedFrame).not.toHaveBeenCalled(); + expect(addon.readFrame).toHaveBeenCalledWith(id, 3); + }); + + it("frees the slot itself when the texture never got imported", async () => { + const { addon, id, service, sharedTexture } = setup(); + sharedTexture.importSharedTexture.mockImplementationOnce(() => { + throw new TypeError("Invalid ntHandle value"); + }); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + try { + await service.readFrame(id, 0, target); + } finally { + warn.mockRestore(); + } + expect(addon.releaseSharedFrame).toHaveBeenCalledWith(id, 1, 3); + }); + + it("reads pixels for a view whose render thread went back to read-back on its own", async () => { + const packet = { gen: 4, width: 4, height: 2, data: Buffer.alloc(32) }; + const { id, service, sharedTexture } = setup({ + addon: { readSharedFrame: vi.fn(() => null), readFrame: vi.fn(() => packet) }, + }); + + expect(await service.readFrame(id, 3, target)).toBe(packet); + expect(sharedTexture.sendSharedTexture).not.toHaveBeenCalled(); + }); + + it("keeps a view on read-back when Chromium does not composite on the GPU", async () => { + const { addon, id, service } = setup({ gpuCompositing: false }); + expect(addon.setSharedFrames).not.toHaveBeenCalled(); + + await service.readFrame(id, 0, target); + expect(addon.readSharedFrame).not.toHaveBeenCalled(); + expect(addon.readFrame).toHaveBeenCalledWith(id, 0); + }); + + it("keeps a view on read-back when OPENSCREEN_PREVIEW_READBACK=1", () => { + vi.stubEnv("OPENSCREEN_PREVIEW_READBACK", "1"); + const { addon } = setup(); + expect(addon.setSharedFrames).not.toHaveBeenCalled(); + }); + + it("keeps a view on read-back when the addon cannot share (not Windows, software backend)", async () => { + const { id, service, sharedTexture } = setup({ + addon: { setSharedFrames: vi.fn(() => false) }, + }); + + await service.readFrame(id, 0, target); + expect(sharedTexture.importSharedTexture).not.toHaveBeenCalled(); + }); +}); + /** A source checkout's `crates/.cargo/config.toml`, with `FFMPEG_DIR` written as `body`. */ function writeCargoConfig(root: string, body: string): void { const cargoDir = path.join(root, "crates", ".cargo"); diff --git a/electron/native-bridge/services/compositorViewService.ts b/electron/native-bridge/services/compositorViewService.ts index 4a765db2bd..550d67dcb4 100644 --- a/electron/native-bridge/services/compositorViewService.ts +++ b/electron/native-bridge/services/compositorViewService.ts @@ -2,14 +2,19 @@ import fs from "node:fs"; import { createRequire } from "node:module"; import path from "node:path"; import { fileURLToPath } from "node:url"; -import { app } from "electron"; +import { app, sharedTexture, type WebFrameMain } from "electron"; import { type CursorKind, readCursorAsArrow, resolveCursorSprites, } from "../../../src/lib/cursor/cursorThemes"; +import type { + CompositorSharedFrameMeta, + CompositorSharedFrameReceipt, +} from "../../../src/native/contracts"; import type { GifExportJob } from "../../ipc/gifExportJobs"; import type { + ClipCameraInput, ClipInput, CompositorBackend, CompositorParamValue, @@ -20,6 +25,7 @@ import type { GifExportStats, GifParamsInput, NativeFramePacket, + NativeSharedFramePacket, RemuxStats, SegmentationSupport, } from "../../native/compositor-view/addon"; @@ -215,6 +221,26 @@ export interface CompositorViewServiceOptions { */ appRoot?: string; isPackaged?: boolean; + /** Electron's `sharedTexture` module, injectable for tests. `null` keeps every view on + * read-back. */ + sharedTexture?: SharedTextureApi | null; + /** Whether Chromium composites on the GPU, where a shared texture is imported. Injectable + * for tests; defaults to `app.getGPUFeatureStatus()`. */ + gpuCompositing?: () => boolean; +} + +/** The two calls a shared preview frame needs from Electron's `sharedTexture` module. */ +export type SharedTextureApi = Pick< + Electron.SharedTexture, + "importSharedTexture" | "sendSharedTexture" +>; + +function defaultGpuCompositing(): boolean { + try { + return String(app.getGPUFeatureStatus().gpu_compositing).startsWith("enabled"); + } catch { + return false; + } } function defaultAppRoot(): string { @@ -467,6 +493,8 @@ function tryLoadAddon(candidates: string[]): CompositorViewAddon | null { export class CompositorViewService { private readonly options: CompositorViewServiceOptions; private readonly rects = new Map(); + /** Views whose frames go out as shared GPU textures rather than RAM pixels. */ + private readonly sharedViews = new Set(); private addon: CompositorViewAddon | null = null; private loadAttempted = false; private syntheticIdCounter = 0; @@ -593,9 +621,41 @@ export class CompositorViewService { } const id = addon.createView(rect, paths?.screenPath, paths?.webcamPath, paths?.cursorPath); this.rects.set(id, rect); + this.shareFrames(addon, id); return id; } + /** Hands the view's frames over as shared GPU textures when this host can, instead of + * copying them through IPC: measured, that transport alone kept 37 to 55 % of the + * renderer's main thread busy at 30 fps (rendering-performance.md). Anything missing — + * the Electron API, GPU compositing, Windows' hardware backend native-side — leaves the + * view on read-back. `OPENSCREEN_PREVIEW_READBACK=1` forces read-back, to compare. */ + private shareFrames(addon: CompositorViewAddon, id: number): void { + if (process.env.OPENSCREEN_PREVIEW_READBACK === "1" || !this.sharedTextureApi()) { + return; + } + if (!(this.options.gpuCompositing ?? defaultGpuCompositing)()) { + return; + } + if (addon.setSharedFrames?.(id, true)) { + this.sharedViews.add(id); + } + } + + private sharedTextureApi(): SharedTextureApi | null { + const api = + this.options.sharedTexture === undefined ? sharedTexture : this.options.sharedTexture; + return typeof api?.importSharedTexture === "function" ? api : null; + } + + /** Takes the view back to read-back frames: a delivery failed, or the renderer saw a shared + * frame land as nothing (Chromium could not open the texture). The render thread + * republishes the current frame, so the canvas is not left empty. */ + stopSharedFrames(id: number): void { + this.sharedViews.delete(id); + this.ensureAddon()?.setSharedFrames?.(id, false); + } + setRect(id: number, rect: CompositorViewRect): void { const addon = this.ensureAddon(); this.rects.set(id, rect); @@ -605,19 +665,78 @@ export class CompositorViewService { addon.setRect(id, rect); } - /** Reads the most recently rendered frame for `id` as a self-describing packet - * (`{ gen, width, height, data }`), but only if its generation is newer than - * `sinceGen`. Returns `null` when the addon is absent, no frame is ready yet, - * OR the caller already holds the current generation — the idle path, where - * `null` comes back without any buffer copy. Byte order is RGBA. */ - readFrame(id: number, sinceGen: number): NativeFramePacket | null { + /** Reads the most recently rendered frame for `id`, but only if its generation is newer + * than `sinceGen`. Returns `null` when the addon is absent, no frame is ready yet, OR the + * caller already holds the current generation — the idle path, where nothing is copied. + * + * A view on shared textures sends the frame to `target` as a GPU texture, and answers with + * its receipt (`shared: true`, no pixels): the texture reaches the renderer before this + * reply does. Any other view answers with the RGBA pixels themselves. */ + async readFrame( + id: number, + sinceGen: number, + target?: WebFrameMain | null, + ): Promise { const addon = this.ensureAddon(); if (!addon) { return null; } + if (target && this.sharedViews.has(id)) { + const frame = addon.readSharedFrame?.(id, sinceGen); + if (frame) { + return this.sendSharedFrame(addon, id, frame, target); + } + } + // Also the answer for a view whose render thread went back to read-back on its own. return addon.readFrame(id, sinceGen); } + private async sendSharedFrame( + addon: CompositorViewAddon, + id: number, + frame: NativeSharedFramePacket, + target: WebFrameMain, + ): Promise { + const meta: CompositorSharedFrameMeta = { + viewId: id, + gen: frame.gen, + width: frame.width, + height: frame.height, + footage: frame.footage ?? null, + footageProjective: frame.footageProjective ?? false, + clipIndex: frame.clipIndex, + sourceTimeSec: frame.sourceTimeSec, + }; + const api = this.sharedTextureApi(); + let imported: Electron.SharedTextureImported | undefined; + try { + if (!api) { + throw new Error("the sharedTexture API is gone"); + } + imported = api.importSharedTexture({ + textureInfo: { + pixelFormat: "rgba", + codedSize: { width: frame.width, height: frame.height }, + handle: { ntHandle: frame.handle }, + }, + // Chromium is done with the texture in every process: the slot may be written again. + allReferencesReleased: () => addon.releaseSharedFrame?.(id, frame.slot, frame.gen), + }); + await api.sendSharedTexture({ frame: target, importedSharedTexture: imported }, meta); + return { ...meta, shared: true }; + } catch (error) { + console.warn("[compositor-view] shared frame not delivered; reading frames back:", error); + this.stopSharedFrames(id); + if (!imported) { + addon.releaseSharedFrame?.(id, frame.slot, frame.gen); + } + return null; + } finally { + // This process's reference only: the renderer holds its own until it has drawn. + imported?.release(); + } + } + setParam(id: number, key: string, value: CompositorParamValue): void { const addon = this.ensureAddon(); if (!addon) { @@ -657,17 +776,27 @@ export class CompositorViewService { webcamOffsetSec: number, clipIndex: number, sourceTimeSec: number, + additionalCameras: ClipCameraInput[] = [], ): void { const addon = this.ensureAddon(); if (!addon) { return; } - addon.setActiveClip(id, screenPath, webcamPath, webcamOffsetSec, clipIndex, sourceTimeSec); + addon.setActiveClip( + id, + screenPath, + webcamPath, + webcamOffsetSec, + clipIndex, + sourceTimeSec, + additionalCameras, + ); } destroyView(id: number): void { const addon = this.ensureAddon(); this.rects.delete(id); + this.sharedViews.delete(id); if (!addon) { return; } diff --git a/electron/native/README.md b/electron/native/README.md index 16013c4bd4..a9c6f8b83e 100644 --- a/electron/native/README.md +++ b/electron/native/README.md @@ -85,6 +85,17 @@ Current V2 JSON shape: The current helper implementation supports display/window video capture, system audio loopback, selected-microphone capture, Media Foundation webcam capture, and a DirectShow webcam fallback for virtual cameras that are not exposed through Media Foundation. Webcam frames are currently composed into the primary MP4 as a bottom-right picture-in-picture overlay. Browser `deviceId` values do not always map to Media Foundation symbolic links or WASAPI endpoint IDs, so the renderer passes both browser IDs and user-visible device names. For microphones, the helper tries the requested WASAPI endpoint ID first, then resolves an active capture endpoint by `microphoneDeviceName`, then falls back to the default endpoint. For webcams, Electron resolves a matching DirectShow filter CLSID for the selected label; the helper uses Media Foundation first, then that exact DirectShow filter when the requested camera is absent from Media Foundation. +Several cameras: a `webcams` list records up to four cameras, each into its own MP4, all on the same T0 as the screen and audio. Each entry takes `camDeviceId`, `camDeviceName`, `camClsid` (the DirectShow filter CLSID), `camWidth`, `camHeight`, `camFps` and `camPath`; an entry without `camPath` is skipped. When the list is present and non-empty it replaces the legacy `webcam*` fields; without it those fields still describe one camera, which writes to `webcamPath` when one is given and is drawn into the screen as the inline picture-in-picture otherwise. + +```json +"webcams": [ + { "camDeviceId": "…", "camDeviceName": "Camera A", "camClsid": "{…}", "camWidth": 1920, "camHeight": 1080, "camFps": 30, "camPath": "C:\\path\\recording-123-webcam.mp4" }, + { "camDeviceId": "…", "camDeviceName": "Camera B", "camClsid": "", "camWidth": 1280, "camHeight": 720, "camFps": 30, "camPath": "C:\\path\\recording-123-webcam-2.mp4" } +] +``` + +Every per-camera event carries the camera's `index`, its position in the list after entries without `camPath` have been skipped: `webcam-format` (`{"event":"webcam-format","schemaVersion":2,"index":0,"width":…,"height":…,"fps":…,"deviceName":"…"}`) for each camera that opened, and `{"event":"warning","code":"webcam-unavailable","index":1,"deviceName":"…","message":"…"}` for each that did not (`code` comes before `index` so older substring readers still match). A camera that cannot be opened, or whose encoder or capture will not start (the message says which), is dropped and the rest of the take goes on; a camera whose encoder rejects a sample mid-take is disabled on its own, without stopping the screen or the other cameras. A camera unplugged mid-take (or whose reads keep failing for a second) is disabled the same way: its file ends at the loss, is still finalized at stop, and is left out of `webcamPaths`. `recording-stopped` keeps `webcamPath` (camera 0, when it recorded) and adds `webcamPaths`, the files of every camera still recording at stop, in index order — an empty list when every camera that wrote a file of its own was disabled mid-take, so a reader can tell "stopped early" from an older helper that never sends the key. Both are printed before the camera files are finalized (see the stop sequence), so a camera whose finalize fails is reported on stderr and by a non-zero exit, not removed from the list. Each camera finalizes in its own `[stop-timing]` step, `webcam-encoder-finalize-`. + Container: recordings are written as fragmented MP4 (`MFCreateFMPEG4MediaSink` + `MFCreateSinkWriterFromMediaSink`, `MF_MPEG4SINK_MIN_FRAGMENT_DURATION` = 1s) rather than plain MP4. A plain MP4 has no index until `IMFSinkWriter::Finalize()` writes `moov` at the very end, so when the shutdown watchdog force-exits a wedged helper the file on disk holds every frame and no way to read them — that is why issues #252 / #292 / #327 cost the whole recording rather than the frozen tail of it. A fragmented MP4 writes its index up front and its samples in self-describing `moof`+`mdat` pairs, so the same kill leaves a file that plays up to the last complete fragment. This does not fix the freeze; it removes the data loss the freeze causes. Because the fragmented sink needs both output media types at construction, the sink writer is built from a media sink instead of from a URL, and the helper reads the video/audio stream positions back off the sink rather than assuming them. If any of that is unavailable on a machine, the helper retries with the plain container and says so — `container` in the `encoder-selection` event is `fragmented-mp4` or `mp4`, and it reports what was used, not what was asked for. Encoder selection: by default the helper keeps the existing sink-writer path first. If that path fails while setting up H.264, it retries with the Microsoft software H.264 encoder (`mfh264enc.dll`). The key of this retry is registering that encoder locally in the helper process via `MFTRegisterLocalByCLSID`, which makes a software H.264 encoder available even when the machine's hardware encoders are missing or broken; hardware transforms are disabled for the retry only as a secondary guard so the sink writer prefers the locally registered software encoder, not as the fallback mechanism itself. Set `preferSoftwareEncoder: true` in the helper JSON, or set `OPENSCREEN_WGC_PREFER_SOFTWARE_ENCODER=true` before launching Electron, to force the software path from the first attempt. @@ -97,6 +108,13 @@ At startup the helper also emits `capture-adapter`, naming the GPU its D3D devic Desktop icons: `"hideDesktopIcons": true` on a display capture puts a borderless window, painted with the wallpaper by `PaintDesktop`, on the captured monitor directly above the window that hosts the icons (`SHELLDLL_DefView`'s top-level parent) and below every app window (`src/desktop_icon_cover.cpp`). WGC cannot exclude another process's window, so covering is the only route that changes no setting. The window belongs to the helper, so it goes away with the recording or with a crashed helper; there is nothing to restore. Windows notifications cannot be kept out: toasts are drawn above every app window, and the only switch is Do Not Disturb, a user setting the helper does not touch. +Menus and popups in window recordings: a window source is captured per HWND, and a context menu or dropdown is a window of its own, so it used to be missing from the video (#894). On Windows 11 24H2 (build 26100) and later the helper sets `GraphicsCaptureSession.IncludeSecondaryWindows` (`IGraphicsCaptureSession6`) on window sources, and DWM draws the popups belonging to the captured window into the same texture: the frame size, DPI handling, occlusion immunity and the fixed frame pool are unchanged. Monitor sources never set it, since they already contain everything on the screen. Four things to know: + +- **Popups are clipped at the window's edge.** The texture is the window's size, so the part of a menu that overhangs it is not recorded (measured: a 276 px menu opened 138 px inside the right edge shows 138 px, cut at the last column). +- **It needs 24H2.** Older runtimes keep capturing the window alone, as before. The helper reports the outcome as `{"event":"secondary-windows","applied":true|false}` (absent for monitor sources and when the option is disabled); the app logs it and shows nothing. Set `OPENSCREEN_WGC_DISABLE_SECONDARY_WINDOWS=1` to turn the option off and get the window-only capture back. +- **What DWM lets in was measured, because Microsoft's rule is style and z-order, not process.** It draws windows with `WS_POPUP` or `WS_EX_TOOLWINDOW` that overlap the window and have only other such windows between them and it. On build 26200, three unrelated TopMost popups from other processes (`WS_POPUP`, `WS_POPUP` plus `WS_EX_TOOLWINDOW`, and `WS_EX_TOOLWINDOW` alone), all overlapping the window with nothing else above it, stayed out of the recording, as did Explorer's own Win+X menu; the window's own owned menu was in. `npm run test:wgc-window-popup:win` reproduces the measurement (menu present, strangers absent, overhang clipped, kill switch removes the menu) and refuses to pass if a non-qualifying window sits above its target, since a stranger missing for that reason would prove nothing. +- **The menu shows up only when no ordinary window sits above the recorded window.** If another window (neither a popup nor a tool window) overlaps it, the menu is missing from the first frame to the last, which is expected from how DWM selects secondary windows (#910). Measured on a second machine (Windows 11 build 26200, AMD, 46 runs of the fixture): the menu was captured in all 25 runs with nothing ordinary above the window, and absent in all 21 runs with one above it (Claude Desktop, Notepad); in the 20 valid runs at 2000 ms and 5000 ms no unrelated popup leaked (0 px). That is 0 leaks in 20 runs, which does not exclude a failure rate up to about 15 percent, and the fixture is synthetic WinForms, not a real application's menus. + Encoder diagnostic on final sink-writer failure: when the final sink-writer attempt fails (`MFCreateSinkWriterFromMediaSink` on the fragmented container, `MFCreateSinkWriterFromURL` on the plain one; the message names which), the helper logs the registered H.264 video encoder MFT count (via `MFTEnumEx`), the registered AAC encoder count when audio was requested, and the hex HRESULT. If no H.264 encoder is registered, it additionally emits the four-bullet actionable error (missing Media Feature Pack / GPU driver registration / empty `HKLM:\SOFTWARE\Microsoft\Windows Media Foundation\Transforms` / reboot). If an H.264 encoder IS registered but the sink writer still failed, it logs a hint pointing at invalid output path, missing MP4 mux, or GPU driver incompatibility. There is still no fail-fast pre-flight gate because `MFTEnumEx` and the sink writer can disagree about which H.264 encoders are available in non-interactive / Session 0 contexts. Smoke-test the helper with: @@ -104,6 +122,7 @@ Smoke-test the helper with: ```powershell npm run test:wgc-helper:win npm run test:wgc-window:win +npm run test:wgc-window-popup:win npm run test:wgc-audio:win npm run test:wgc-mic:win npm run test:wgc-mixed-audio:win @@ -118,6 +137,12 @@ npm run test:wgc-webcam:win Remove-Item Env:OPENSCREEN_WGC_TEST_WEBCAM_DEVICE_NAME ``` +To check that a camera which cannot be opened costs only itself, record the real camera next to a nonexistent second one listed through `webcams`: + +```powershell +npm run test:wgc-helper:win -- --webcam --missing-second-webcam +``` + To validate a specific native microphone manually: ```powershell diff --git a/electron/native/compositor-view/addon.d.ts b/electron/native/compositor-view/addon.d.ts index 1365846c72..cbc1b8d798 100644 --- a/electron/native/compositor-view/addon.d.ts +++ b/electron/native/compositor-view/addon.d.ts @@ -34,6 +34,33 @@ export interface NativeFramePacket { width: number; height: number; data: Buffer; + /** The footage in this frame: its TL, TR, BR, BL corners (x, y as fractions of the frame, + * eight numbers), where a privacy blur's gimbal goes. Absent before anything is composed. */ + footage?: number[] | null; + /** The footage maps from those corners by their homography (real camera), not bilinearly. */ + footageProjective?: boolean; + /** Where the view was when it composed this frame: the active clip in `scene.clips` and the + * screen frame's time in its source file. Absent from an older `.node`. */ + clipIndex?: number; + sourceTimeSec?: number; +} + +/** A preview frame left in a shared GPU texture instead of copied into RAM (Windows, hardware + * backend — see `setSharedFrames`). `handle` is the texture's NT handle the way Electron's + * `sharedTexture.importSharedTexture` takes it: 8 bytes, little-endian, valid in this process + * only. Its `slot` stays reserved until `releaseSharedFrame(id, slot, gen)`. */ +export interface NativeSharedFramePacket { + gen: number; + slot: number; + handle: Buffer; + width: number; + height: number; + footage?: number[] | null; + footageProjective?: boolean; + /** Where the view was when it composed this frame: the active clip in `scene.clips` and the + * screen frame's time in its source file. Absent from an older `.node`. */ + clipIndex?: number; + sourceTimeSec?: number; } export interface ExportStats { @@ -91,6 +118,14 @@ export interface ExportParamsInput { bitrate?: number; } +/** An additional camera (2-4) of a clip, for `setActiveClip` and the export clip list. + * = napi `ClipCameraInput`. */ +export interface ClipCameraInput { + path: string; + /** Camera source time = screen source time - this. */ + offsetSec: number; +} + /** One timeline clip for the native multiclip export (screen + webcam files + source trim). */ export interface ClipInput { screenPath: string; @@ -99,6 +134,8 @@ export interface ClipInput { sourceEndSec: number; /** webcam source time = screen source time − this. */ webcamOffsetSec: number; + /** Cameras 2-4 (index k-1 = camera k). Only those a layout region shows are decoded. */ + additionalCameras?: ClipCameraInput[]; } /** Which backend the compositor will run on. `"cpu"` = WARP rasterisation + software @@ -153,6 +190,17 @@ export interface CompositorViewAddon { * still frame): `null` comes back WITHOUT cloning the buffer or crossing IPC. * Pass `sinceGen = 0` to force delivery of the current frame. */ readFrame(id: number, sinceGen: number): NativeFramePacket | null; + /** Deliver this view's frames as shared GPU textures (`readSharedFrame`) rather than RAM + * pixels (`readFrame`). Returns `false` where that cannot work (not Windows, software + * backend): the view keeps reading back. Optional: an older `.node` predates it. */ + setSharedFrames?(id: number, enabled: boolean): boolean; + /** The latest frame left in the view's shared texture ring, if newer than `sinceGen`. + * `null` too while the view reads back to RAM. Throws the render thread's fatal error, + * like `readFrame`. */ + readSharedFrame?(id: number, sinceGen: number): NativeSharedFramePacket | null; + /** Chromium let go of frame `gen` in `slot`, in every process: the render thread may write + * that slot again. A no-op for a destroyed view. */ + releaseSharedFrame?(id: number, slot: number, gen: number): void; setParam(id: number, key: string, value: CompositorParamValue): void; setPlaying(id: number, playing: boolean): void; /** Seeks the view to source-media `seconds` for the active clip. */ @@ -160,6 +208,8 @@ export interface CompositorViewAddon { /** Installs the app scene (JSON `SceneDescription`) — layout preset etc. drive the render * instead of the fixture. Invalid JSON is ignored native-side. */ setScene(id: number, sceneJson: string): void; + /** `additionalCameras`: cameras 2-4 of the clip (index k-1 = camera k, empty `path` = none). + * An addon built before it ignores the argument. */ setActiveClip( id: number, screenPath: string, @@ -167,6 +217,7 @@ export interface CompositorViewAddon { webcamOffsetSec: number, clipIndex: number, sourceTimeSec: number, + additionalCameras?: ClipCameraInput[], ): void; destroyView(id: number): void; /** Renders the fixture to `outPath` (C8), auto-pausing live previews. `onProgress` diff --git a/electron/native/pipewire-capture/src/dmabuf_import.rs b/electron/native/pipewire-capture/src/dmabuf_import.rs index 4dc60412f6..59e900a84a 100644 --- a/electron/native/pipewire-capture/src/dmabuf_import.rs +++ b/electron/native/pipewire-capture/src/dmabuf_import.rs @@ -266,8 +266,10 @@ impl DmabufImporter { // Output size = the recorded (out) size. For a monitor that equals the // source; for a window it is the crop rectangle, and the per-frame crop // fields set in `import` pick which region of the source is scaled into it. + // BT.709 studio range out, as `ensure_sws` converts the CPU path and as + // the compositor decodes; left to the driver it is not guaranteed (#926). let scale_args = std::ffi::CString::new(format!( - "w={}:h={}:format=nv12", + "w={}:h={}:format=nv12:out_color_matrix=bt709:out_range=limited: out_color_primaries=bt709:out_color_transfer=bt709", self.out_width, self.out_height )) .map_err(|_| "scale_vaapi args contained a NUL".to_owned())?; diff --git a/electron/native/pipewire-capture/src/encoder.rs b/electron/native/pipewire-capture/src/encoder.rs index a2e041bfe4..05191862a7 100644 --- a/electron/native/pipewire-capture/src/encoder.rs +++ b/electron/native/pipewire-capture/src/encoder.rs @@ -355,6 +355,12 @@ impl VideoEncoder { (*codec_ctx).framerate = ff::AVRational { num: params.fps, den: 1 }; (*codec_ctx).pix_fmt = backend.codec_pixel_format(); (*codec_ctx).bit_rate = params.bitrate; + // What the samples are (see `ensure_sws` and the dmabuf VPP), said in + // the stream so no player has to guess from the frame size (#926). + (*codec_ctx).color_range = ff::AVCOL_RANGE_MPEG; + (*codec_ctx).colorspace = ff::AVCOL_SPC_BT709; + (*codec_ctx).color_primaries = ff::AVCOL_PRI_BT709; + (*codec_ctx).color_trc = ff::AVCOL_TRC_BT709; // Half a second, not the two seconds a streaming preset would use. // // This file is an EDITING SOURCE, and the editor scrubs it. Seeking @@ -787,6 +793,17 @@ impl VideoEncoder { ff::err_to_string(initialised) )); } + // BT.709 studio range, which is what the compositor decodes every + // recording as. swscale's default matrix is BT.601, so left alone every + // Linux take played back with shifted hue and saturation + // (getopenscreen/openscreen#926). Full range in: the source is RGB. + let bt709 = ff::sws_getCoefficients(ff::SWS_CS_ITU709 as i32); + let unity = 1 << 16; + if ff::sws_setColorspaceDetails(self.sws, bt709, 1, bt709, 0, 0, unity, unity) < 0 { + return Err(format!( + "swscale refused a BT.709 conversion from pixel format {src_format} to {dst_format}" + )); + } // Read back rather than assume: libswscale is free to ignore a thread // count it cannot use for a given conversion, and a request that was // silently dropped looks exactly like threading that did not help. @@ -1053,6 +1070,40 @@ struct MuxTrack { stream_time_base: ff::AVRational, } +/// The mov muxer options that make the output fragmented. +/// +/// A fragment is cut at a video keyframe once it holds at least one second, the +/// same floor the Windows sink uses (`kFragmentDurationHns` in +/// wgc-capture/src/mf_encoder.cpp). The GOP is half a second (see +/// `VideoEncoder::open_backend`), so a fragment is two GOPs and a kill loses at +/// most about a second. `frag_keyframe` alone would cut every half second; +/// `frag_duration` alone would cut mid-GOP, leaving fragments that open on a +/// frame nothing can decode without the previous one. `min_frag_duration` is +/// the floor that waits for the keyframe. `default_base_moof` makes each moof's +/// data offsets relative to that moof rather than to the file, so a fragment +/// parses on its own. +/// +/// `flush_packets` hands every closed fragment to the kernel at once. Without +/// it the moov and the fragments wait in avio's 32 KB buffer, and a kill loses +/// that too: measured, a low-bitrate take killed after three seconds left a +/// 28-byte file. It costs one write per fragment, since the muxer holds each +/// fragment's samples until it closes. +/// +/// `delay_moov` is for the audio track. AAC's first packet sits 1024 samples +/// before zero (encoder priming), and an empty moov written with the header +/// cannot know that, so libavformat drops the edit list and shifts every track +/// 21 ms later instead: measured, the first video frame then lasts 55 ms and +/// every later one lands 21 ms after its capture time, against a cursor track +/// that does not move. Delaying the moov to the first fragment lets it carry +/// the same edit list the plain MP4 had; timestamps match it packet for packet. +/// The price is that a take killed in its first second keeps nothing, as every +/// killed take did before. +const FRAGMENT_OPTIONS: [(&CStr, &CStr); 3] = [ + (c"movflags", c"+frag_keyframe+empty_moov+default_base_moof+delay_moov"), + (c"min_frag_duration", c"1000000"), + (c"flush_packets", c"1"), +]; + /// An MP4 writer, taking however many streams [`Self::add_stream`] is called for. /// /// WHAT A RECORDING ACTUALLY USES IS TWO: one H.264 video stream and one AAC @@ -1067,11 +1118,15 @@ struct MuxTrack { /// and it also removes the question of reconciling two capture clocks in the /// container: there is only one audio timeline now, built in `AudioMix::pump`. /// +/// The file is a FRAGMENTED MP4, like the Windows and macOS helpers write +/// (#338): a moov with no samples in front, then self-describing moof+mdat pairs. A plain +/// MP4 has no index until the trailer, so a killed helper, a crash or a power +/// loss used to leave gigabytes of frames and no way to read them (#944). Now it +/// plays up to the last complete fragment. See [`FRAGMENT_OPTIONS`]. +/// /// `+faststart` is not used: it rewrites the whole file on close, which on a -/// long recording means copying gigabytes. The moov atom is written at the end -/// as usual, and the app reads these files locally, where a trailing moov costs -/// nothing. This is the difference from the WebM path, which had NO index at all -/// and could not be seeked even locally (see electron/recording/webm-seek-index.ts). +/// long recording means copying gigabytes, and a fragmented file has its moov +/// in front anyway. pub struct Muxer { fmt: *mut ff::AVFormatContext, tracks: Vec, @@ -1153,9 +1208,21 @@ impl Muxer { if self.tracks.is_empty() { return Err("the output has no streams".to_owned()); } - // SAFETY: `fmt` is ours and every stream was added through add_stream. + // SAFETY: `fmt` is ours and every stream was added through add_stream; + // the dictionary is ours and freed before leaving the block. unsafe { - let header = ff::avformat_write_header(self.fmt, ptr::null_mut()); + let mut options: *mut ff::AVDictionary = ptr::null_mut(); + for (key, value) in FRAGMENT_OPTIONS { + ff::av_dict_set(&mut options, key.as_ptr(), value.as_ptr(), 0); + } + let header = ff::avformat_write_header(self.fmt, &mut options); + // The muxer takes out every option it knows; one left over is a typo + // that would silently bring back the unfragmented file. + let unknown = ff::av_dict_count(options); + ff::av_dict_free(&mut options); + if header >= 0 && unknown > 0 { + return Err(format!("the mp4 muxer ignored {unknown} fragment option(s)")); + } if header < 0 { return Err(format!( "avformat_write_header: {}", @@ -1217,7 +1284,7 @@ impl Drop for Muxer { fn drop(&mut self) { // SAFETY: `fmt` is either null or ours. `header_written` is still true // only on the error path — `finish` clears it — and a file left without - // its trailer would be unplayable, so write one on the way out. + // its trailer would lose its last fragment, so write one on the way out. unsafe { if self.fmt.is_null() { return; @@ -1423,6 +1490,111 @@ mod tests { assert!(size > 1024, "the muxed file is {size} bytes, which cannot be 120 frames"); } + #[test] + fn software_path_converts_and_tags_bt709() { + // The compositor decodes every recording as BT.709 studio range; swscale's + // default is BT.601, which puts pure red at Y 81, Cb 90 (#926). + let (width, height) = (64, 32); + let mut encoder = VideoEncoder::open( + VideoParams { width, height, fps: 30, bitrate: 2_000_000 }, + Some(Backend::Software), + |_, _| {}, + ) + .expect("the software encoder always opens"); + let stride = width as usize * 4; + let mut frame = vec![0u8; stride * height as usize]; + for pixel in frame.chunks_exact_mut(4) { + pixel.copy_from_slice(&[0, 0, 255, 255]); // BGRA pure red + } + encoder.stage(&frame, stride, ff::AV_PIX_FMT_BGRA).expect("stage"); + // SAFETY: `stage` just filled `sw_frame`, a YUV420P frame of this size. + let (y, cb, cr, ctx) = unsafe { + let staged = encoder.sw_frame; + (*(*staged).data[0], *(*staged).data[1], *(*staged).data[2], &*encoder.codec_ctx) + }; + let close = |got: u8, want: i32| (i32::from(got) - want).abs() <= 1; + assert!(close(y, 63) && close(cb, 102) && close(cr, 240), "red came out Y {y} Cb {cb} Cr {cr}"); + assert_eq!(ctx.color_range, ff::AVCOL_RANGE_MPEG); + assert_eq!(ctx.colorspace, ff::AVCOL_SPC_BT709); + assert_eq!(ctx.color_primaries, ff::AVCOL_PRI_BT709); + assert_eq!(ctx.color_trc, ff::AVCOL_TRC_BT709); + } + + #[test] + fn a_killed_muxer_leaves_a_file_that_opens_and_holds_its_frames() { + // Three seconds at 30 fps with audio, then a simulated kill: no trailer, + // no flush of the write buffer, no interleave queue drained (#944). + let (width, height, fps) = (64, 32, 30); + // Per process, so two test runs on one machine never share the file. + let output = std::env::temp_dir() + .join(format!("openscreen-killed-muxer-{}.mp4", std::process::id())); + let _ = std::fs::remove_file(&output); + let mut video = VideoEncoder::open( + VideoParams { width, height, fps, bitrate: 1_000_000 }, + Some(Backend::Software), + |_, _| {}, + ) + .expect("the software encoder always opens"); + let mut audio = AudioEncoder::open(128_000).expect("aac"); + let mut muxer = Muxer::create(&output).expect("muxer"); + let video_track = muxer.add_stream(video.codec_context()).expect("video track"); + let audio_track = muxer.add_stream(audio.codec_context()).expect("audio track"); + muxer.write_header().expect("header"); + + let stride = width as usize * 4; + let mut frame = vec![0u8; stride * height as usize]; + let samples = vec![0.0f32; (AUDIO_SAMPLE_RATE / fps) as usize * AUDIO_CHANNELS]; + for pts in 0..3 * i64::from(fps) { + frame.fill(pts as u8); + video + .submit(&frame, stride, ff::AV_PIX_FMT_BGRA, pts, |p| muxer.write(video_track, p)) + .expect("video"); + audio.push(&samples, |p| muxer.write(audio_track, p)).expect("audio"); + } + std::mem::forget(muxer); + + // SAFETY: ffmpeg's own demuxer on the file just written; the context is + // closed before the block ends. + let (streams, video_frames, second_frame_at) = unsafe { + let path = CString::new(output.as_os_str().as_encoded_bytes()).unwrap(); + let mut input: *mut ff::AVFormatContext = ptr::null_mut(); + let opened = + ff::avformat_open_input(&mut input, path.as_ptr(), ptr::null(), ptr::null_mut()); + assert!(opened >= 0, "a killed take must open: {}", ff::err_to_string(opened)); + let probed = ff::avformat_find_stream_info(input, ptr::null_mut()); + assert!(probed >= 0, "stream info: {}", ff::err_to_string(probed)); + let packet = ff::av_packet_alloc(); + let mut video_frames = 0; + let mut second_frame_at = 0.0; + while ff::av_read_frame(input, packet) >= 0 { + let stream = *(*input).streams.add((*packet).stream_index as usize); + if (*(*stream).codecpar).codec_type == ff::AVMEDIA_TYPE_VIDEO { + if video_frames == 1 { + let base = (*stream).time_base; + second_frame_at = (*packet).pts as f64 * f64::from(base.num) + / f64::from(base.den); + } + video_frames += 1; + } + ff::av_packet_unref(packet); + } + let mut packet = packet; + ff::av_packet_free(&mut packet); + let streams = (*input).nb_streams; + ff::avformat_close_input(&mut input); + (streams, video_frames, second_frame_at) + }; + assert_eq!(streams, 2, "both tracks are declared up front"); + // Fragments close at the keyframes at 1 s and 2 s; only the one in + // flight, the last second, is lost. + assert!(video_frames >= 2 * fps, "only {video_frames} frames survived the kill"); + // Without an edit list for AAC's priming, every frame after the first + // slides 21 ms off its capture time (see FRAGMENT_OPTIONS). + let one_frame = 1.0 / f64::from(fps); + assert!((second_frame_at - one_frame).abs() < 1e-3, "frame 1 at {second_frame_at} s"); + let _ = std::fs::remove_file(&output); + } + #[test] fn radv_perftest_is_added_without_clobbering_an_existing_value() { std::env::set_var("RADV_PERFTEST", "gpl"); diff --git a/electron/native/pipewire-capture/src/main.rs b/electron/native/pipewire-capture/src/main.rs index 44d96ae7df..13c987ae97 100644 --- a/electron/native/pipewire-capture/src/main.rs +++ b/electron/native/pipewire-capture/src/main.rs @@ -662,8 +662,31 @@ fn run( // means the import never works, which would otherwise record nothing — see // MAX_CONSECUTIVE_IMPORT_FAILURES. let mut consecutive_drops: u32 = 0; + // When the capture clock last advanced. See the heartbeat at the top of the loop. + let mut last_advance = Instant::now(); loop { + // THE HEARTBEAT, CHECKED ON EVERY PASS, NOT ONLY ON A TIMEOUT. `advance` + // holds the last picture forward and drains the audio rings, and it used + // to run only on a new frame or when `recv_timeout` expired. Any other + // message restarts that timeout, and cursor messages arrive at the + // compositor's rate while the mouse moves: a static screen with a moving + // mouse never timed out, so nothing drained and the 2 s rings dropped + // their oldest audio (getopenscreen/openscreen#936). + if heartbeat_due(last_advance, Instant::now(), config.tick) { + last_advance = Instant::now(); + if let Some(capture) = capture.as_mut() { + if let Err(message) = capture.advance() { + let _ = emitter.emit(&Event::Error { + code: "encode-failed".to_owned(), + message, + }); + exit_code = 1; + break; + } + } + } + // Return PipeWire buffers whose dmabuf imports completed last iteration // (or that were superseded on the capture thread). This MUST run on this // loop, not the PipeWire thread — `Session::requeue` takes the thread-loop @@ -918,6 +941,7 @@ fn run( } } if let Some(capture) = capture.as_mut() { + last_advance = Instant::now(); if let Err(message) = capture.advance() { let _ = emitter.emit(&Event::Error { code: "encode-failed".to_owned(), @@ -1239,19 +1263,8 @@ fn run( emit_sample(emitter, &cursor, content_rect(&capture, size), &mut pending_asset, None); last_emit = Instant::now(); } - // The heartbeat that keeps the output at a constant frame rate - // while the screen is static: no frame arrived, but the clock - // moved, so the last picture is held forward. - if let Some(capture) = capture.as_mut() { - if let Err(message) = capture.advance() { - let _ = emitter.emit(&Event::Error { - code: "encode-failed".to_owned(), - message, - }); - exit_code = 1; - break; - } - } + // No advance here: a timeout means a tick has passed, so the + // heartbeat at the top of the next pass holds the picture forward. } Err(RecvTimeoutError::Disconnected) => break, @@ -1271,9 +1284,9 @@ fn run( /// Writes the trailer and reports what the recording cost. /// -/// Runs even when the loop broke on an error: a file whose moov atom was never -/// written is unplayable, and a partial recording is worth more to the user than -/// none. +/// Runs even when the loop broke on an error: a file whose trailer was never +/// written loses its last fragment, and a partial recording is worth more to the +/// user than none. fn finish_capture( emitter: &mut Emitter, capture: Capture, @@ -1349,6 +1362,11 @@ fn finish_capture( /// only rectangle the file shows. Before then, and for a cursor-only session /// that opens no encoder at all, the whole negotiated stream, which is what the /// consumer of a cursor-only recording is compositing over. +/// Whether a tick has passed since the capture clock last advanced. +fn heartbeat_due(last_advance: Instant, now: Instant, tick: Duration) -> bool { + now.saturating_duration_since(last_advance) >= tick +} + fn content_rect(capture: &Option, size: Option<(i32, i32)>) -> Option { match capture { Some(capture) if capture.started() => Some(capture.content_rect()), @@ -1563,6 +1581,28 @@ mod cursor_sample_tests { assert!(buffer.is_empty(), "emitted {}", String::from_utf8_lossy(&buffer)); } + /// #936: messages arriving faster than the tick must not hold the heartbeat + /// off. The loop checks it on every pass, so what matters is that it comes + /// due once a tick has passed, however many passes happened in between. + #[test] + fn the_heartbeat_comes_due_every_tick_whatever_else_arrives() { + let tick = Duration::from_millis(16); + let start = Instant::now(); + let mut last_advance = start; + let mut advances = 0; + // A cursor message every 7 ms (144 Hz) for one second of loop passes. + for pass in 0..=(1000 / 7) { + let now = start + Duration::from_millis(pass * 7); + if heartbeat_due(last_advance, now, tick) { + last_advance = now; + advances += 1; + } + } + // Every 21 ms at worst (three 7 ms passes): about 47 a second, never zero. + assert!(advances >= 1000 / 21, "{advances} advances in a second of cursor traffic"); + assert!(!heartbeat_due(start, start + Duration::from_millis(15), tick)); + } + /// A cursor-only session opens no encoder, so it falls back to the whole /// stream; once pixels are flowing the encoder's rectangle wins. #[test] diff --git a/electron/native/screencapturekit/Sources/OpenScreenCaptureCore/CaptureSizing.swift b/electron/native/screencapturekit/Sources/OpenScreenCaptureCore/CaptureSizing.swift index 9741d95599..0b69fedfe8 100644 --- a/electron/native/screencapturekit/Sources/OpenScreenCaptureCore/CaptureSizing.swift +++ b/electron/native/screencapturekit/Sources/OpenScreenCaptureCore/CaptureSizing.swift @@ -72,3 +72,24 @@ private func evenFloor(_ value: Int) -> Int { let clamped = max(2, value) return clamped - (clamped % 2) } + +/// The video bitrate for the size the stream actually got, the rule the Linux helper uses +/// (`default_bitrate` in `pipewire-capture/src/capture.rs`): 0.1 bit per pixel per frame, +/// about 12 Mbit/s at 1920×1080/60, kept between 2 and 60 Mbit/s. +/// +/// The renderer used to send `computeBitrate(TARGET_WIDTH, TARGET_HEIGHT)`, the 4K ceiling with +/// its high-frame-rate boost: 76.5 Mbit/s for every take, a 1080p one included, 4.25× what +/// Windows spends on the same capture (getopenscreen/openscreen#924). It cannot know the real +/// size, which `captureOutputSize` settles here, so it now sends none. +public func defaultVideoBitrate(width: Int, height: Int, fps: Int) -> Int { + let bitsPerPixel = 0.1 + let pixelsPerSecond = Double(max(1, width)) * Double(max(1, height)) * Double(max(1, fps)) + return min(max(Int(pixelsPerSecond * bitsPerPixel), 2_000_000), 60_000_000) +} + +/// The H.264 keyframe interval, in frames: one second of video, the interval Windows uses. +/// Left to VideoToolbox it had no bound, and the editor's scrub cost is the GOP length: a +/// seek decodes from the keyframe before it (getopenscreen/openscreen#937). +public func videoKeyFrameInterval(fps: Int) -> Int { + max(1, fps) +} diff --git a/electron/native/screencapturekit/Sources/OpenScreenCaptureCore/VideoColor.swift b/electron/native/screencapturekit/Sources/OpenScreenCaptureCore/VideoColor.swift new file mode 100644 index 0000000000..35b30eed0e --- /dev/null +++ b/electron/native/screencapturekit/Sources/OpenScreenCaptureCore/VideoColor.swift @@ -0,0 +1,22 @@ +import AVFoundation +import CoreGraphics + +// The colour every recording is decoded as: BT.709 primaries, transfer and matrix, studio +// range. The compositor assumes it for every file, and the Windows (#929) and Linux (#932) +// helpers write it. Left to the defaults, ScreenCaptureKit hands over the display's own colour +// space (P3 on most Macs) and VideoToolbox picks the matrix and the tags +// (getopenscreen/openscreen#943). +// +// Range has no key here: it comes from the pixel format, and the stream captures `420v`, the +// studio-range one. + +/// The matrix ScreenCaptureKit converts to YCbCr with. It has to be the one the file is tagged +/// with in `videoColorProperties`, which `VideoColorTests` pins. +public let captureYCbCrMatrix: CFString = CGDisplayStream.yCbCrMatrix_ITU_R_709_2 + +/// `AVVideoColorPropertiesKey` for the H.264 writer input. +public let videoColorProperties: [String: String] = [ + AVVideoColorPrimariesKey: AVVideoColorPrimaries_ITU_R_709_2, + AVVideoTransferFunctionKey: AVVideoTransferFunction_ITU_R_709_2, + AVVideoYCbCrMatrixKey: AVVideoYCbCrMatrix_ITU_R_709_2, +] diff --git a/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/ScreenCaptureRecorder.swift b/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/ScreenCaptureRecorder.swift index 4e0723f555..f08c447fdf 100644 --- a/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/ScreenCaptureRecorder.swift +++ b/electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/ScreenCaptureRecorder.swift @@ -152,6 +152,9 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { private var totalPausedDuration = CMTime.zero /// Sample queue only. See `VideoTimestampGate` for the failure it exists to prevent. private var videoTimestampGate = VideoTimestampGate() + /// Sample queue only. Frames the writer input was not ready for, or refused, reported in + /// `recording-stopped` so a macOS drop rate can be measured at all (#937). + private var droppedVideoFrames = 0 private var nativeMicrophoneEnabled = false private var outputWidth = 1920 private var outputHeight = 1080 @@ -454,8 +457,11 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { "captureBounds": captureBoundsPayload(), ]) } else if !appended { + droppedVideoFrames += 1 reportWriterFailure("video append") } + } else { + droppedVideoFrames += 1 } } @@ -701,7 +707,15 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { configuration.minimumFrameInterval = CMTime(value: 1, timescale: CMTimeScale(max(1, request.video.fps))) configuration.queueDepth = 6 configuration.showsCursor = !request.video.hideSystemCursor - configuration.pixelFormat = kCVPixelFormatType_32BGRA + // Studio-range BT.709 YCbCr from ScreenCaptureKit itself, which the encoder takes as is + // and the compositor decodes (#943). BGRA left the matrix to VideoToolbox. Nothing here + // reads the pixels: a frame goes through `retimedSampleBuffer` (timing only) to + // `append`, and `isCompleteFrame` reads attachments. 1.5 bytes a pixel instead of 4, + // times `queueDepth`. + configuration.pixelFormat = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + configuration.colorMatrix = captureYCbCrMatrix + // Unset, the buffers carry the display's colour space: P3 on most Macs. + configuration.colorSpaceName = CGColorSpace.sRGB configuration.sampleRate = 48_000 configuration.channelCount = 2 configuration.excludesCurrentProcessAudio = true @@ -754,8 +768,12 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { AVVideoCodecKey: AVVideoCodecType.h264, AVVideoWidthKey: outputWidth, AVVideoHeightKey: outputHeight, + // BT.709 tags, the colour the stream is captured in (#943). + AVVideoColorPropertiesKey: videoColorProperties, AVVideoCompressionPropertiesKey: [ - AVVideoAverageBitRateKey: request.video.bitrate ?? 18_000_000, + // From the size this stream really got. The renderer sends none (#924). + AVVideoAverageBitRateKey: request.video.bitrate + ?? defaultVideoBitrate(width: outputWidth, height: outputHeight, fps: request.video.fps), AVVideoExpectedSourceFrameRateKey: request.video.fps, // Without this the encoder defaults to B-frames, and a reordered // stream needs a composition offset per sample. AVAssetWriter emits @@ -784,6 +802,11 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { // particular run happened to die. Reordering off is 3/3 clean across // both rates, and a SIGKILL at 25s still leaves 27 readable `moof`. AVVideoAllowFrameReorderingKey: false, + // A keyframe every second, by frame count and by time: a still screen + // delivers fewer frames than the fps, and the count alone would then + // stretch the GOP, which is the editor's scrub cost (#937). + AVVideoMaxKeyFrameIntervalKey: videoKeyFrameInterval(fps: request.video.fps), + AVVideoMaxKeyFrameIntervalDurationKey: 1, ], ] let input = AVAssetWriterInput(mediaType: .video, outputSettings: settings) @@ -906,7 +929,9 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { } } - let refusedVideoFrames = sampleQueue.sync { videoTimestampGate.rejectedCount } + let (refusedVideoFrames, droppedFrames) = sampleQueue.sync { + (videoTimestampGate.rejectedCount, droppedVideoFrames) + } if refusedVideoFrames > 1 { emit([ "event": "warning", @@ -929,6 +954,7 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { emit([ "event": "recording-stopped", "screenPath": request.outputs.screenPath, + "droppedVideoFrames": droppedFrames, ]) } else { emitError( diff --git a/electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/CaptureSizingTests.swift b/electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/CaptureSizingTests.swift index 53029a31aa..0d318585fe 100644 --- a/electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/CaptureSizingTests.swift +++ b/electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/CaptureSizingTests.swift @@ -134,4 +134,31 @@ final class CaptureSizingTests: XCTestCase { XCTAssertEqual(result.height, 1080, "scale \(scale)") } } + + // MARK: - Bitrate from the real size (#924) + + func testBitrateFollowsTheCapturedSizeNotTheCeiling() { + // 1080p60 used to be asked for 76.5 Mbit/s, the 4K ceiling's figure. + XCTAssertEqual(defaultVideoBitrate(width: 1920, height: 1080, fps: 60), 12_441_600) + XCTAssertEqual(defaultVideoBitrate(width: 2560, height: 1440, fps: 60), 22_118_400) + XCTAssertEqual(defaultVideoBitrate(width: 3840, height: 2160, fps: 60), 49_766_400) + } + + func testBitrateStaysWithinItsFloorAndCeiling() { + // A small window still gets enough bits to stay sharp; 4K at 120 fps does not ask for + // more than any disk wants. + XCTAssertEqual(defaultVideoBitrate(width: 320, height: 240, fps: 30), 2_000_000) + XCTAssertEqual(defaultVideoBitrate(width: 3840, height: 2160, fps: 120), 60_000_000) + } + + // MARK: - Keyframe interval (#937) + + func testKeyFrameIntervalIsOneSecondOfFrames() { + XCTAssertEqual(videoKeyFrameInterval(fps: 60), 60) + XCTAssertEqual(videoKeyFrameInterval(fps: 30), 30) + } + + func testKeyFrameIntervalNeverFallsBelowOneFrame() { + XCTAssertEqual(videoKeyFrameInterval(fps: 0), 1) + } } diff --git a/electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/VideoColorTests.swift b/electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/VideoColorTests.swift new file mode 100644 index 0000000000..2a423d2b90 --- /dev/null +++ b/electron/native/screencapturekit/Tests/OpenScreenCaptureCoreTests/VideoColorTests.swift @@ -0,0 +1,18 @@ +import AVFoundation +import XCTest + +import OpenScreenCaptureCore + +/// The file has to say what its samples are: a matrix that converts with one standard and tags +/// another is the shifted-hue defect #943 exists to remove. +final class VideoColorTests: XCTestCase { + func testCaptureMatrixIsTheOneTheFileIsTaggedWith() { + XCTAssertEqual(captureYCbCrMatrix as String, videoColorProperties[AVVideoYCbCrMatrixKey]) + } + + func testFileIsTaggedBT709() { + XCTAssertEqual(videoColorProperties[AVVideoColorPrimariesKey], "ITU_R_709_2") + XCTAssertEqual(videoColorProperties[AVVideoTransferFunctionKey], "ITU_R_709_2") + XCTAssertEqual(videoColorProperties[AVVideoYCbCrMatrixKey], "ITU_R_709_2") + } +} diff --git a/electron/native/wgc-capture/CMakeLists.txt b/electron/native/wgc-capture/CMakeLists.txt index c0921486fd..009a04677c 100644 --- a/electron/native/wgc-capture/CMakeLists.txt +++ b/electron/native/wgc-capture/CMakeLists.txt @@ -39,8 +39,15 @@ add_executable(wgc-capture src/audio_sample_utils.cpp src/audio_sample_utils.h src/desktop_icon_cover.cpp + src/device_selection.cpp + src/device_selection.h src/desktop_icon_cover.h src/dpi_awareness.h + src/realtime_scheduling.h + src/frame_visibility.cpp + src/frame_visibility.h + src/json_fields.cpp + src/json_fields.h src/dshow_webcam_capture.cpp src/dshow_webcam_capture.h src/main.cpp @@ -55,6 +62,12 @@ add_executable(wgc-capture src/wasapi_render_keepalive.cpp src/wasapi_render_keepalive.h src/webcam_capture.cpp + src/webcam_loss.cpp + src/webcam_loss.h + src/webcam_format.cpp + src/webcam_format.h + src/webcam_config.cpp + src/webcam_config.h src/webcam_capture.h src/wgc_session.cpp src/wgc_session.h @@ -69,6 +82,7 @@ target_compile_definitions(wgc-capture PRIVATE target_compile_options(wgc-capture PRIVATE /EHsc /W4 /utf-8) target_link_libraries(wgc-capture PRIVATE + avrt d3d11 dxgi mf @@ -77,11 +91,13 @@ target_link_libraries(wgc-capture PRIVATE mfuuid runtimeobject windowsapp + winmm ) add_executable(cursor-sampler src/cursor-sampler.cpp src/dpi_awareness.h + src/realtime_scheduling.h ) target_compile_definitions(cursor-sampler PRIVATE @@ -92,8 +108,10 @@ target_compile_definitions(cursor-sampler PRIVATE target_compile_options(cursor-sampler PRIVATE /EHsc /W4 /utf-8) target_link_libraries(cursor-sampler PRIVATE + avrt gdi32 gdiplus + winmm ) add_executable(audio_sample_utils_test @@ -111,9 +129,124 @@ target_compile_definitions(audio_sample_utils_test PRIVATE target_compile_options(audio_sample_utils_test PRIVATE /EHsc /W4 /utf-8) target_link_libraries(audio_sample_utils_test PRIVATE + avrt mf mfplat mfreadwrite mfuuid ole32 + winmm ) + +add_executable(webcam_format_test + src/webcam_format.cpp + src/webcam_format.h + src/webcam_format_test.cpp +) + +target_compile_definitions(webcam_format_test PRIVATE + NOMINMAX + WIN32_LEAN_AND_MEAN + _WIN32_WINNT=0x0A00 +) + +target_compile_options(webcam_format_test PRIVATE /EHsc /W4 /utf-8) + +add_executable(frame_visibility_test + src/frame_visibility.cpp + src/frame_visibility.h + src/frame_visibility_test.cpp +) + +target_compile_definitions(frame_visibility_test PRIVATE + NOMINMAX + WIN32_LEAN_AND_MEAN + _WIN32_WINNT=0x0A00 +) + +target_compile_options(frame_visibility_test PRIVATE /EHsc /W4 /utf-8) + +add_executable(webcam_snapshot_test + src/dshow_webcam_capture.h + src/webcam_snapshot_test.cpp +) + +target_compile_definitions(webcam_snapshot_test PRIVATE + NOMINMAX + WIN32_LEAN_AND_MEAN + _WIN32_WINNT=0x0A00 +) + +target_compile_options(webcam_snapshot_test PRIVATE /EHsc /W4 /utf-8) + +add_executable(mf_encoder_color_test + src/audio_sample_utils.cpp + src/audio_sample_utils.h + src/mf_encoder.cpp + src/mf_encoder.h + src/mf_encoder_color_test.cpp +) + +target_compile_definitions(mf_encoder_color_test PRIVATE + NOMINMAX + WIN32_LEAN_AND_MEAN + _WIN32_WINNT=0x0A00 +) + +target_compile_options(mf_encoder_color_test PRIVATE /EHsc /W4 /utf-8) + +target_link_libraries(mf_encoder_color_test PRIVATE + avrt + d3d11 + dxgi + mf + mfplat + mfreadwrite + mfuuid + ole32 + propsys +) + +add_executable(webcam_config_test + src/json_fields.cpp + src/json_fields.h + src/webcam_config.cpp + src/webcam_config.h + src/webcam_config_test.cpp +) + +target_compile_definitions(webcam_config_test PRIVATE + NOMINMAX + WIN32_LEAN_AND_MEAN + _WIN32_WINNT=0x0A00 +) + +target_compile_options(webcam_config_test PRIVATE /EHsc /W4 /utf-8) + +add_executable(device_selection_test + src/device_selection.cpp + src/device_selection.h + src/device_selection_test.cpp +) + +target_compile_definitions(device_selection_test PRIVATE + NOMINMAX + WIN32_LEAN_AND_MEAN + _WIN32_WINNT=0x0A00 +) + +target_compile_options(device_selection_test PRIVATE /EHsc /W4 /utf-8) + +add_executable(webcam_loss_test + src/webcam_loss.cpp + src/webcam_loss.h + src/webcam_loss_test.cpp +) + +target_compile_definitions(webcam_loss_test PRIVATE + NOMINMAX + WIN32_LEAN_AND_MEAN + _WIN32_WINNT=0x0A00 +) + +target_compile_options(webcam_loss_test PRIVATE /EHsc /W4 /utf-8) diff --git a/electron/native/wgc-capture/src/audio_sample_utils.cpp b/electron/native/wgc-capture/src/audio_sample_utils.cpp index 54e77e40f4..4ffe3ff50b 100644 --- a/electron/native/wgc-capture/src/audio_sample_utils.cpp +++ b/electron/native/wgc-capture/src/audio_sample_utils.cpp @@ -1,4 +1,5 @@ #include "audio_sample_utils.h" +#include "realtime_scheduling.h" #include @@ -478,6 +479,18 @@ void mixAudioInPlace( } } +namespace { + +// How far behind real time AudioMixer writes; see mixLoop. Wider than the capture +// threads' poll jitter with room for a stall, and harmless beyond that: the +// output timeline does not move with it. +// ponytail: absorbs jitter, not clock drift. A device slower than the steady clock +// spends the cushion over a long take, then leaves one gap as it re-anchors; +// resampling each source to the mixer clock is the upgrade if that is ever heard. +constexpr uint32_t MixerCushionMs = 100; + +} // namespace + AudioMixer::AudioMixer( const AudioInputFormat& format, const AudioInputFormat& systemFormat, @@ -510,34 +523,59 @@ bool AudioMixer::start() { systemDecimator_.reset(); microphoneDecimator_.reset(); thread_ = std::thread([this] { + const MmcssThread mmcss(L"Pro Audio"); mixLoop(); }); return true; } +void AudioMixer::resetSources() { + systemQueue_.clear(); + microphoneQueue_.clear(); + systemDecimator_.reset(); + microphoneDecimator_.reset(); + systemStarved_ = false; + microphoneStarved_ = false; +} + void AudioMixer::beginTimeline() { { std::scoped_lock lock(mutex_); - systemQueue_.clear(); - microphoneQueue_.clear(); - systemDecimator_.reset(); - microphoneDecimator_.reset(); + resetSources(); emittedFrames_ = 0; + mixedFrames_ = 0; timelineStarted_ = true; + // Here, not when mixLoop next wakes: the video's T0 is taken right after + // this call, and every millisecond the mixer took to wake would have + // placed the whole track that much early -- tens of them on a busy machine. + clockStart_ = std::chrono::steady_clock::now(); + clockAnchored_ = true; } cv_.notify_all(); } void AudioMixer::setPaused(bool paused) { { - std::scoped_lock lock(mutex_); - paused_ = paused; - if (paused_) { - systemQueue_.clear(); - microphoneQueue_.clear(); - systemDecimator_.reset(); - microphoneDecimator_.reset(); + std::unique_lock lock(mutex_); + if (paused && !paused_) { + // The queues still hold what the cushion kept back; mixLoop writes it + // up to this instant, and the resume clears whatever is left. + pausedAt_ = std::chrono::steady_clock::now(); + pauseFlushed_ = !timelineStarted_; + } else if (!paused && paused_) { + // A resume right behind the pause would otherwise clear the cushion + // before mixLoop has written it: the last 100 ms of voice before the + // pause, gone (measured, 80 to 90 ms). + cv_.wait(lock, [this] { return pauseFlushed_ || stopRequested_.load(); }); + resetSources(); + // Resumed at this instant, where the pause left off: the flush has + // run, so `emittedFrames_` is the pause point and mixLoop is idle. + clockStart_ = std::chrono::steady_clock::now() - + std::chrono::duration_cast(std::chrono::duration( + static_cast(emittedFrames_) / format_.sampleRate)); + clockAnchored_ = true; } + paused_ = paused; } cv_.notify_all(); } @@ -560,7 +598,7 @@ void AudioMixer::pushSystem(const BYTE* data, DWORD byteCount) { if (paused_) { return; } - append(systemQueue_, data, byteCount, systemFormat_, 1.0, systemDecimator_); + append(systemQueue_, systemStarved_, data, byteCount, systemFormat_, 1.0, systemDecimator_); } cv_.notify_all(); } @@ -581,13 +619,16 @@ void AudioMixer::pushMicrophone(const BYTE* data, DWORD byteCount) { // in: the queue would hold a flat-topped signal and the mix would add // further distortion on top of it, with whatever headroom the opposite // polarity of the system stream offered already destroyed. - append(microphoneQueue_, data, byteCount, microphoneFormat_, 1.0, microphoneDecimator_); + append( + microphoneQueue_, microphoneStarved_, data, byteCount, microphoneFormat_, 1.0, + microphoneDecimator_); } cv_.notify_all(); } void AudioMixer::append( std::vector& queue, + bool& starved, const BYTE* data, DWORD byteCount, const AudioInputFormat& sourceFormat, @@ -598,16 +639,35 @@ void AudioMixer::append( } convertAudioWithGain(data, byteCount, sourceFormat, format_, gain, gainBuffer_, decimator); + // A source that ran dry is coming back: loopback after a silence, or a device + // that stalled for longer than the cushion. Its packet belongs at now, which + // is as far ahead of what mixLoop has taken as real time is -- the cushion + // plus however late mixLoop is running, which on a loaded machine is tens of + // milliseconds. Queued at the front, it would land that much early. + if (starved) { + uint64_t lagFrames = 0; + if (clockAnchored_) { + const double elapsed = + std::chrono::duration(std::chrono::steady_clock::now() - clockStart_).count(); + const auto nowFrames = static_cast(std::max(0.0, elapsed) * format_.sampleRate); + lagFrames = nowFrames > mixedFrames_ ? nowFrames - mixedFrames_ : 0; + } + queue.assign(static_cast(lagFrames) * format_.blockAlign, 0); + starved = false; + } queue.insert(queue.end(), gainBuffer_.begin(), gainBuffer_.end()); } -bool AudioMixer::pop(std::vector& queue, std::vector& chunk, size_t byteCount) { +bool AudioMixer::pop( + std::vector& queue, bool& starved, std::vector& chunk, size_t byteCount) { + chunk.assign(byteCount, 0); + if (queue.size() < byteCount) { + starved = true; + } if (queue.empty()) { - chunk.assign(byteCount, 0); return false; } - chunk.assign(byteCount, 0); const size_t copiedBytes = std::min(byteCount, queue.size()); std::memcpy(chunk.data(), queue.data(), copiedBytes); queue.erase(queue.begin(), queue.begin() + static_cast(copiedBytes)); @@ -631,72 +691,60 @@ bool AudioMixer::pop(std::vector& queue, std::vector& chunk, size_t * to cause a system-audio desync it merely stopped concealing * (getopenscreen/openscreen#406). * - * `audioClockStart` is anchored so that `emittedFrames_` always describes the + * A cushion behind real time (getopenscreen/openscreen#911). The capture threads + * poll WASAPI and push whatever has piled up -- every 15.6 ms at the default + * timer resolution, and later than that on a busy machine -- so a packet can + * reach its queue after the chunk it belongs to is due. Mixing at real time + * zero-filled that chunk and the packet played one chunk late: a hole of up to + * 10 ms in the middle of a continuous voice, heard as crackle. Measured on real + * takes: ten or more such holes in 25 s. Writing `MixerCushionMs` behind real time + * gives a late packet that long to arrive. The timestamps do not move -- they + * still come from `emittedFrames_` -- so the cushion only delays the writing, + * and a pause or a stop writes what it still holds up to that instant. + * + * `clockStart_` is anchored so that `emittedFrames_` always describes the * time elapsed since the timeline began; re-deriving it on resume is what lets a * pause interrupt the clock without shifting everything recorded after it. */ void AudioMixer::mixLoop() { const uint32_t chunkFrames = std::max(1, format_.sampleRate / 100); const size_t chunkBytes = static_cast(chunkFrames) * format_.blockAlign; + const uint64_t cushionFrames = static_cast(format_.sampleRate) * MixerCushionMs / 1000; std::vector mixedChunk; std::vector sourceChunk; - std::chrono::steady_clock::time_point audioClockStart; - bool audioClockAnchored = false; + // This loop's copy of `clockStart_`, taken under the lock at the top of each + // pass: beginTimeline and a resume move it from other threads. + std::chrono::steady_clock::time_point clockStart; const auto framesToDuration = [&](uint64_t frames) { return std::chrono::duration_cast( std::chrono::duration(static_cast(frames) / format_.sampleRate)); }; + const auto framesAt = [&](std::chrono::steady_clock::time_point time) { + const double elapsed = std::chrono::duration(time - clockStart).count(); + return static_cast(std::max(0.0, elapsed) * format_.sampleRate); + }; - while (true) { - { - std::unique_lock lock(mutex_); - cv_.wait_for(lock, std::chrono::milliseconds(20), [&] { - return stopRequested_.load() || (timelineStarted_ && !paused_); - }); - - if (stopRequested_) { - break; - } - if (!timelineStarted_ || paused_) { - // A pause stops the clock rather than resetting it: the anchor is - // re-derived from `emittedFrames_` on resume, so what follows keeps - // the position it would have had. - audioClockAnchored = false; - continue; - } - } - - const auto now = std::chrono::steady_clock::now(); - if (!audioClockAnchored) { - audioClockStart = now - framesToDuration(emittedFrames_); - audioClockAnchored = true; - } - - // How much of the timeline real time has covered. Emitting up to here -- - // from the queues where they have data, from silence where they do not -- - // is what keeps the audio clock pinned to the take rather than to whether - // anything happened to be playing. - const auto elapsed = std::chrono::duration(now - audioClockStart).count(); - const uint64_t targetFrames = static_cast(elapsed * format_.sampleRate); - + // Writes every chunk that ends by `targetFrames` -- from the queues where they + // have data, from silence where they do not, which is what keeps the audio + // clock pinned to the take rather than to whether anything happened to be + // playing. False once the output refuses a chunk. + const auto emitUntil = [&](uint64_t targetFrames) { while (emittedFrames_ + chunkFrames <= targetFrames) { { std::scoped_lock lock(mutex_); - if (stopRequested_ || !timelineStarted_ || paused_) { - break; - } mixedChunk.assign(chunkBytes, 0); if (includeSystem_) { - pop(systemQueue_, sourceChunk, chunkBytes); + pop(systemQueue_, systemStarved_, sourceChunk, chunkBytes); mixAudioInPlace(mixedChunk, sourceChunk.data(), static_cast(sourceChunk.size()), format_); } if (includeMicrophone_) { - pop(microphoneQueue_, sourceChunk, chunkBytes); + pop(microphoneQueue_, microphoneStarved_, sourceChunk, chunkBytes); mixAudioInPlace( mixedChunk, sourceChunk.data(), static_cast(sourceChunk.size()), format_, microphoneGain_); } + mixedFrames_ = emittedFrames_ + chunkFrames; } const int64_t timestampHns = @@ -705,15 +753,70 @@ void AudioMixer::mixLoop() { static_cast((static_cast(chunkFrames) * HnsPerSecond) / format_.sampleRate); if (!output_(mixedChunk.data(), static_cast(mixedChunk.size()), timestampHns, durationHns)) { stopRequested_ = true; - break; + return false; } emittedFrames_ += chunkFrames; } + return true; + }; - if (stopRequested_) { + while (true) { + bool stopping = false; + bool running = false; + bool flush = false; + std::chrono::steady_clock::time_point pausedAt; + { + std::unique_lock lock(mutex_); + cv_.wait_for(lock, std::chrono::milliseconds(20), [&] { + return stopRequested_.load() || (timelineStarted_ && !paused_); + }); + stopping = stopRequested_; + running = timelineStarted_ && !paused_; + pausedAt = pausedAt_; + clockStart = clockStart_; + if (stopping || !running) { + flush = clockAnchored_; + clockAnchored_ = false; + } + } + + if (stopping || !running) { + // A pause stops the clock rather than resetting it: the anchor is + // re-derived from `emittedFrames_` on resume, so what follows keeps + // the position it would have had -- once the cushion is written up to + // the pause, which is where that position is. A stop writes it up to + // now, unless it lands on a pause this loop has not handled yet: the + // paused span is out of the video, so it stays out of the audio. + const bool written = + !flush || emitUntil(framesAt(running ? std::chrono::steady_clock::now() : pausedAt)); + { + std::scoped_lock lock(mutex_); + pauseFlushed_ = true; + } + cv_.notify_all(); + if (!written || stopping) { + break; + } + continue; + } + + const uint64_t realFrames = framesAt(std::chrono::steady_clock::now()); + if (!emitUntil(realFrames > cushionFrames ? realFrames - cushionFrames : 0)) { break; } - std::this_thread::sleep_until(audioClockStart + framesToDuration(emittedFrames_ + chunkFrames)); + // Woken early by a pause or a stop, so that one writes the cushion out at once. + std::unique_lock lock(mutex_); + cv_.wait_until( + lock, clockStart + framesToDuration(emittedFrames_ + chunkFrames + cushionFrames), + [&] { return stopRequested_.load() || paused_; }); } + // Whatever ended the loop, a resume waiting on the pause flush must not wait + // for a loop that is gone. Set under the lock, so the wakeup cannot slip + // between the waiter's check and its sleep. + { + std::scoped_lock lock(mutex_); + pauseFlushed_ = true; + } + cv_.notify_all(); } diff --git a/electron/native/wgc-capture/src/audio_sample_utils.h b/electron/native/wgc-capture/src/audio_sample_utils.h index f869d5e941..81d99d021f 100644 --- a/electron/native/wgc-capture/src/audio_sample_utils.h +++ b/electron/native/wgc-capture/src/audio_sample_utils.h @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -103,12 +104,14 @@ class AudioMixer { private: void append( std::vector& queue, + bool& starved, const BYTE* data, DWORD byteCount, const AudioInputFormat& sourceFormat, double gain, AudioDecimatorState& decimator); - bool pop(std::vector& queue, std::vector& chunk, size_t byteCount); + bool pop(std::vector& queue, bool& starved, std::vector& chunk, size_t byteCount); + void resetSources(); void mixLoop(); AudioInputFormat format_{}; @@ -124,10 +127,23 @@ class AudioMixer { std::vector microphoneQueue_; AudioDecimatorState systemDecimator_; AudioDecimatorState microphoneDecimator_; + // Set when pop() runs a source dry, cleared by its next packet (see append). + bool systemStarved_ = false; + bool microphoneStarved_ = false; std::vector gainBuffer_; std::thread thread_; std::atomic stopRequested_ = false; bool timelineStarted_ = false; bool paused_ = false; + std::chrono::steady_clock::time_point pausedAt_{}; + // Set by mixLoop once it has written the cushion up to `pausedAt_`; a resume + // waits for it, or it would clear audio that belongs before the pause. + bool pauseFlushed_ = true; + // mixLoop's clock, shared so a source coming back after running dry can be + // placed at real time (see append). Written by mixLoop under `mutex_`. + std::chrono::steady_clock::time_point clockStart_{}; + bool clockAnchored_ = false; + // Frames mixLoop has taken from the queues so far, under `mutex_`. + uint64_t mixedFrames_ = 0; uint64_t emittedFrames_ = 0; }; diff --git a/electron/native/wgc-capture/src/audio_sample_utils_test.cpp b/electron/native/wgc-capture/src/audio_sample_utils_test.cpp index b28f738c88..73995bd464 100644 --- a/electron/native/wgc-capture/src/audio_sample_utils_test.cpp +++ b/electron/native/wgc-capture/src/audio_sample_utils_test.cpp @@ -1,4 +1,5 @@ #include "audio_sample_utils.h" +#include "realtime_scheduling.h" #include #include @@ -215,6 +216,38 @@ HRESULT trySetAacPcmRateWithVideo(UINT32 sampleRate) { } // namespace int main() { + // --- Timer resolution (getopenscreen/openscreen#921) ---------------------- + // + // First, before anything in this process could have raised it: a 5 ms sleep + // is what the WASAPI poll asks for, and at the default 15.625 ms tick it + // lasts a whole tick. Median of 20, so one preempted sleep cannot decide it. + { + const auto medianSleepMs = [] { + std::vector samples; + for (int i = 0; i < 20; i += 1) { + const auto start = std::chrono::steady_clock::now(); + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + samples.push_back( + std::chrono::duration(std::chrono::steady_clock::now() - start).count()); + } + std::sort(samples.begin(), samples.end()); + return samples[samples.size() / 2]; + }; + const double before = medianSleepMs(); + const HighResolutionTiming timing; + // Load only ever lengthens a sleep, so a median under 8 ms cannot be + // faked by a busy host, and at the default tick it cannot happen at all. + // A failing round is retried, so a transient load does not fail the build. + double after = medianSleepMs(); + for (int retry = 0; retry < 2 && after >= 8.0; retry += 1) { + after = std::min(after, medianSleepMs()); + } + char detail[96]{}; + sprintf_s(detail, "5 ms sleep: median %.2f ms before, %.2f ms after", before, after); + std::cout << "TIMER_RAW " << detail << std::endl; + expect("timer-5ms-sleep-lasts-about-5ms", after < 8.0, detail); + } + const AudioInputFormat diagnostic = makeFormat(MFAudioFormat_Float, 96000, 8, 32); const AudioInputFormat snapped = makeAacCompatibleAudioFormat(diagnostic); expect( @@ -1446,6 +1479,187 @@ int main() { } } + // --- Capture jitter: the mixer's cushion (getopenscreen/openscreen#911) --- + // + // The capture threads poll WASAPI every 5 ms -- 15.6 ms at the default timer + // resolution -- and push whatever packets have piled up, while the mixer emits + // on its own clock. With no margin between the two, every packet that arrived + // just after the mixer's tick was zero-filled: holes of up to 10 ms in the + // middle of the voice, heard as a crackle in the recording itself. `capture` + // below is that loop without WASAPI: a 10 ms packet becomes available every + // 10 ms from `from`, and a poller that sleeps 5 ms pushes what has piled up. + { + using std::chrono::milliseconds; + using Clock = std::chrono::steady_clock; + const AudioInputFormat f32 = makeFormat(MFAudioFormat_Float, 48000, 2, 32); + const auto dcPacket = [&](float value) { + std::vector bytes(480 * f32.blockAlign, 0); + auto* samples = reinterpret_cast(bytes.data()); + for (size_t i = 0; i < 480 * 2; i += 1) { + samples[i] = value; + } + return bytes; + }; + const auto capture = [](const auto& push, Clock::time_point from, int packets, + int stallAtPacket, int stallMs) { + int pushed = 0; + bool stalled = false; + while (pushed < packets) { + const auto now = Clock::now(); + const int available = now < from + ? 0 + : std::min(packets, static_cast((now - from) / milliseconds(10))); + for (; pushed < available; pushed += 1) { + push(); + } + if (!stalled && pushed >= stallAtPacket) { + stalled = true; + std::this_thread::sleep_for(milliseconds(stallMs)); + } + std::this_thread::sleep_for(milliseconds(5)); + } + }; + // Left channel of everything the mixer writes, from beginTimeline to stop. + const auto runTake = [&](bool includeSystem, bool includeMic, const auto& drive) { + std::mutex guard; + std::vector left; + AudioMixer mixer( + target48k, f32, f32, includeSystem, includeMic, 1.0, + [&](const BYTE* data, DWORD byteCount, int64_t, int64_t) { + std::scoped_lock lock(guard); + const auto* samples = reinterpret_cast(data); + for (size_t i = 0; i < byteCount / target48k.blockAlign; i += 1) { + left.push_back(samples[i * 2]); + } + return true; + }); + expect("jitter-mixer-start", mixer.start(), ""); + mixer.beginTimeline(); + drive(mixer, Clock::now()); + mixer.stop(); + std::scoped_lock lock(guard); + return left; + }; + const auto msAt = [](size_t frame) { return static_cast(frame) * 1000.0 / 48000.0; }; + + // (1) A microphone streaming a steady DC through ordinary poll jitter and one + // 40 ms stall: every sample between its first and its last must be voiced. + { + const auto packet = dcPacket(0.5f); + const auto left = runTake(false, true, [&](AudioMixer& mixer, Clock::time_point t0) { + capture( + [&] { mixer.pushMicrophone(packet.data(), static_cast(packet.size())); }, + t0, 120, 50, 40); + }); + const auto voiced = [](int16_t s) { return s != 0; }; + const auto first = std::find_if(left.begin(), left.end(), voiced); + const auto last = std::find_if(left.rbegin(), left.rend(), voiced).base(); + const size_t span = first < last ? static_cast(last - first) : 0; + const size_t holes = first < last ? static_cast(std::count(first, last, int16_t{0})) : 0; + char detail[128]{}; + sprintf_s(detail, "span=%zu holes=%zu of 57600 pushed", span, holes); + std::cout << "JITTER_RAW streaming " << detail << std::endl; + expect("mixer-streaming-source-has-no-holes", span >= 110 * 480 && holes == 0, detail); + } + + // (2) The cushion must not move a source that starts after silence -- loopback + // delivers nothing while nothing plays. Played from 400 ms, it lands there. + { + const auto packet = dcPacket(0.5f); + const auto left = runTake(true, false, [&](AudioMixer& mixer, Clock::time_point t0) { + capture( + [&] { mixer.pushSystem(packet.data(), static_cast(packet.size())); }, + t0 + milliseconds(400), 30, 1000, 0); + }); + const auto first = std::find_if(left.begin(), left.end(), [](int16_t s) { return s != 0; }); + const double at = msAt(static_cast(first - left.begin())); + char detail[96]{}; + sprintf_s(detail, "first sound at %.1f ms, played at 400 ms", at); + std::cout << "JITTER_RAW late-source " << detail << std::endl; + expect("mixer-late-source-lands-when-it-played", first != left.end() && std::abs(at - 400.0) <= 40.0, detail); + } + + // (3) Nor across a pause: what the cushion still holds at the pause is written + // before it, so what follows the resume starts where the pause began. + { + const auto beforePause = dcPacket(0.5f); + const auto afterPause = dcPacket(-0.5f); + double pausedAtMs = 0.0; + const auto left = runTake(false, true, [&](AudioMixer& mixer, Clock::time_point t0) { + capture( + [&] { mixer.pushMicrophone(beforePause.data(), static_cast(beforePause.size())); }, + t0, 40, 1000, 0); + pausedAtMs = std::chrono::duration(Clock::now() - t0).count(); + mixer.setPaused(true); + std::this_thread::sleep_for(milliseconds(200)); + mixer.setPaused(false); + capture( + [&] { mixer.pushMicrophone(afterPause.data(), static_cast(afterPause.size())); }, + Clock::now(), 30, 1000, 0); + }); + const auto first = std::find_if(left.begin(), left.end(), [](int16_t s) { return s < 0; }); + const double at = msAt(static_cast(first - left.begin())); + char detail[96]{}; + sprintf_s(detail, "resumed sound at %.1f ms, paused at %.1f ms", at, pausedAtMs); + std::cout << "JITTER_RAW pause " << detail << std::endl; + expect("mixer-resume-continues-at-the-pause", first != left.end() && std::abs(at - pausedAtMs) <= 40.0, detail); + } + + // (5) A pause followed by a stop, as the helper ends every take (#942): + // the track ends at the pause, not at the stop that comes later. + { + const auto packet = dcPacket(0.5f); + double pausedAtMs = 0.0; + const auto left = runTake(false, true, [&](AudioMixer& mixer, Clock::time_point t0) { + capture( + [&] { mixer.pushMicrophone(packet.data(), static_cast(packet.size())); }, + t0, 40, 1000, 0); + pausedAtMs = std::chrono::duration(Clock::now() - t0).count(); + mixer.setPaused(true); + std::this_thread::sleep_for(milliseconds(300)); + }); + const double lengthMs = msAt(left.size()); + char detail[96]{}; + sprintf_s(detail, "track %.1f ms long, paused at %.1f ms", lengthMs, pausedAtMs); + std::cout << "JITTER_RAW stop-after-pause " << detail << std::endl; + expect("mixer-track-ends-at-the-pause-before-a-stop", std::abs(lengthMs - pausedAtMs) <= 20.0, detail); + } + + // (4) The same with a resume that follows the pause at once, before the + // mixer has had a chance to see the pause: the resume must wait for the + // cushion to be written, or it throws the cushion away and what follows + // lands a cushion early. + { + const auto beforePause = dcPacket(0.5f); + const auto afterPause = dcPacket(-0.5f); + double pausedAtMs = 0.0; + const auto left = runTake(false, true, [&](AudioMixer& mixer, Clock::time_point t0) { + capture( + [&] { mixer.pushMicrophone(beforePause.data(), static_cast(beforePause.size())); }, + t0, 40, 1000, 0); + pausedAtMs = std::chrono::duration(Clock::now() - t0).count(); + mixer.setPaused(true); + mixer.setPaused(false); + capture( + [&] { mixer.pushMicrophone(afterPause.data(), static_cast(afterPause.size())); }, + Clock::now(), 30, 1000, 0); + }); + const auto first = std::find_if(left.begin(), left.end(), [](int16_t s) { return s < 0; }); + const double at = msAt(static_cast(first - left.begin())); + // 40 packets were pushed before the pause: 400 ms of voice, all of it + // due before the pause, none of it to be thrown away with the cushion. + const auto lastBefore = std::find_if(left.rbegin(), left.rend(), [](int16_t s) { return s > 0; }); + const double voiceEnd = msAt(static_cast(left.rend() - lastBefore)); + char detail[128]{}; + sprintf_s( + detail, "voice before the pause ends at %.1f ms of 400, resumed at %.1f ms, paused at %.1f ms", + voiceEnd, at, pausedAtMs); + std::cout << "JITTER_RAW instant-resume " << detail << std::endl; + expect("mixer-instant-resume-keeps-the-voice-before-it", voiceEnd >= 390.0, detail); + expect("mixer-instant-resume-continues-at-the-pause", first != left.end() && std::abs(at - pausedAtMs) <= 40.0, detail); + } + } + // --- mixAudioInPlace, per output format ----------------------------------- // // The gain rides in mixAudioInPlace on every format branch, but the diff --git a/electron/native/wgc-capture/src/cursor-sampler.cpp b/electron/native/wgc-capture/src/cursor-sampler.cpp index aa58a1cd7b..cf4947c556 100644 --- a/electron/native/wgc-capture/src/cursor-sampler.cpp +++ b/electron/native/wgc-capture/src/cursor-sampler.cpp @@ -3,6 +3,7 @@ #include #include "dpi_awareness.h" +#include "realtime_scheduling.h" #include #include @@ -429,6 +430,7 @@ int main(int argc, char* argv[]) { std::cerr << "ERROR: Could not enable per-monitor-v2 DPI awareness" << std::endl; return 1; } + const HighResolutionTiming highResolutionTiming; if (argc < 2) { std::cerr << "Usage: cursor-sampler [windowHandle]" << std::endl; diff --git a/electron/native/wgc-capture/src/device_selection.cpp b/electron/native/wgc-capture/src/device_selection.cpp new file mode 100644 index 0000000000..3211cbd1d2 --- /dev/null +++ b/electron/native/wgc-capture/src/device_selection.cpp @@ -0,0 +1,158 @@ +#include "device_selection.h" + +#include +#include + +namespace { + +/** + * Does one of these appear inside the other as WHOLE WORDS? + * + * Plain containment answered for devices that merely share a spelling: a + * requested "Logi" is inside "Logitech", and "Micro" inside "Microphone", + * neither of them as a word. Matching on that resolved a camera nobody asked + * for -- and resolving one is exactly what stops the request reaching the + * DirectShow fallback, where the cameras Media Foundation cannot enumerate live. + * + * Both sides arrive normalized, so a boundary is the start of the string, its + * end, or a space. + */ +bool containsAsWords(const std::wstring& haystack, const std::wstring& needle) { + if (haystack.empty() || needle.empty()) { + return false; + } + size_t pos = haystack.find(needle); + while (pos != std::wstring::npos) { + const bool startsOnBoundary = pos == 0 || haystack[pos - 1] == L' '; + const size_t after = pos + needle.size(); + const bool endsOnBoundary = after == haystack.size() || haystack[after] == L' '; + if (startsOnBoundary && endsOnBoundary) { + return true; + } + pos = haystack.find(needle, pos + 1); + } + return false; +} + +bool containsInsensitive(const std::wstring& haystack, const std::wstring& needle) { + return containsAsWords(haystack, needle) || containsAsWords(needle, haystack); +} + +std::wstring normalizeDeviceName(const std::wstring& value) { + std::wstring normalized; + normalized.reserve(value.size()); + bool lastWasSpace = true; + for (const wchar_t ch : value) { + if (std::iswalnum(ch)) { + normalized.push_back(static_cast(std::towlower(ch))); + lastWasSpace = false; + continue; + } + if (!lastWasSpace) { + normalized.push_back(L' '); + lastWasSpace = true; + } + } + while (!normalized.empty() && normalized.back() == L' ') { + normalized.pop_back(); + } + return normalized; +} + +std::wstring toLower(const std::wstring& value) { + std::wstring lowered; + lowered.reserve(value.size()); + for (const wchar_t ch : value) { + lowered.push_back(static_cast(std::towlower(ch))); + } + return lowered; +} + +} // namespace + +bool DeviceClaims::contains(const std::wstring& identity) const { + const std::wstring normalized = normalizeDeviceIdentity(identity); + return !normalized.empty() && + std::find(identities_.begin(), identities_.end(), normalized) != identities_.end(); +} + +void DeviceClaims::add(const std::wstring& identity) { + const std::wstring normalized = normalizeDeviceIdentity(identity); + if (!normalized.empty() && + std::find(identities_.begin(), identities_.end(), normalized) == identities_.end()) { + identities_.push_back(normalized); + } +} + +std::wstring normalizeDeviceIdentity(const std::wstring& identity) { + const std::wstring lowered = toLower(identity); + const size_t interfaceClass = lowered.rfind(L"#{"); + if (interfaceClass == std::wstring::npos || interfaceClass == 0) { + return lowered; + } + const size_t classEnd = lowered.find(L'}', interfaceClass); + if (classEnd == std::wstring::npos) { + return lowered; + } + return lowered.substr(0, interfaceClass) + lowered.substr(classEnd + 1); +} + +int deviceMatchScore( + const std::wstring& candidateName, + const std::wstring& candidateLink, + const std::wstring& requestedName, + const std::wstring& requestedId) { + int score = 0; + const auto normalizedName = normalizeDeviceName(candidateName); + const auto normalizedLink = normalizeDeviceName(candidateLink); + const auto normalizedRequestedName = normalizeDeviceName(requestedName); + const auto normalizedRequestedId = normalizeDeviceName(requestedId); + + if (!normalizedRequestedName.empty()) { + if (normalizedName == normalizedRequestedName) { + score = std::max(score, 1000); + } + if (containsInsensitive(normalizedName, normalizedRequestedName)) { + score = std::max(score, 900); + } + if (containsInsensitive(normalizedLink, normalizedRequestedName)) { + score = std::max(score, 800); + } + } + + if (!normalizedRequestedId.empty()) { + if (containsInsensitive(normalizedLink, normalizedRequestedId)) { + score = std::max(score, 700); + } + if (containsInsensitive(normalizedName, normalizedRequestedId)) { + score = std::max(score, 600); + } + } + + return score; +} + +int selectUnclaimedDevice( + const std::vector& candidates, + const std::wstring& requestedName, + const std::wstring& requestedId, + const DeviceClaims& claims) { + const bool requested = !requestedName.empty() || !requestedId.empty(); + int selected = -1; + int bestScore = 0; + for (size_t index = 0; index < candidates.size(); ++index) { + if (claims.contains(candidates[index].identity)) { + continue; + } + const int score = + deviceMatchScore(candidates[index].name, candidates[index].identity, requestedName, requestedId); + if (requested && score <= 0) { + continue; + } + if (selected < 0 || score > bestScore) { + selected = static_cast(index); + bestScore = score; + } + } + return selected; +} diff --git a/electron/native/wgc-capture/src/device_selection.h b/electron/native/wgc-capture/src/device_selection.h new file mode 100644 index 0000000000..bcdc2ad8f8 --- /dev/null +++ b/electron/native/wgc-capture/src/device_selection.h @@ -0,0 +1,89 @@ +#pragma once + +#include +#include + +/** + * Which camera devices are already recording in this take. + * + * Two webcams of the same model report the same friendly name, and the id the + * browser hands us is a salted hash that never matches a device path, so name + * matching alone sends every such camera to the FIRST physical device. The + * second open then fails as busy and that camera is lost. Each camera that + * opens adds its device identity here, and later cameras of the take skip it. + * + * Identities are compared normalized (see `normalizeDeviceIdentity`), so a + * device opened through Media Foundation is recognized when it turns up again + * on the DirectShow fallback. + */ +class DeviceClaims { +public: + bool contains(const std::wstring& identity) const; + /** Ignores an empty identity: a device we cannot name cannot be claimed. */ + void add(const std::wstring& identity); + +private: + std::vector identities_; +}; + +/** One device a capture backend enumerated. */ +struct DeviceCandidate { + std::wstring name; + /** The Media Foundation symbolic link or the DirectShow DevicePath. */ + std::wstring identity; +}; + +/** + * The part of a device interface path that names the physical device. + * + * Media Foundation's symbolic link and DirectShow's DevicePath are the same + * `\\?\usb#vid_…##{interface class}\` string, except that + * each registers the camera under its own interface class GUID (measured on a + * Snapdragon front camera: KSCATEGORY_VIDEO_CAMERA against KSCATEGORY_VIDEO). + * Dropping the `#{…}` class and lowercasing leaves the device instance plus the + * reference string, which both share. The reference string is kept because it + * tells apart two cameras of one device (a colour and an IR sensor). A value + * without that shape (a moniker display name) is only lowercased. + */ +std::wstring normalizeDeviceIdentity(const std::wstring& identity); + +/** + * How well a candidate answers a requested name, or 0 for "not this one". + * + * Only decisive matches count: the names being equal once normalized, or one + * containing the other -- which is the ordinary case, since Chromium appends USB + * ids to what the driver reports. + * + * A further tier used to score shared WORDS, to bridge names differing more than + * that. It bridged names that were not the same device. "Logi Capture" and + * "Logitech StreamCam" share no word, yet "logi" sits inside "logitech" and that + * scored high enough to win -- so asking for a camera Media Foundation cannot + * enumerate opened a DIFFERENT camera, instead of returning nothing and letting + * the DirectShow fallback find the real one (getopenscreen/openscreen#405). + * + * Returning 0 is what makes that fallback reachable, so it is a real answer + * rather than a weak match. Keep this in step with + * `electron/recording/deviceNameMatching.ts`, which states the same rules for + * the Electron side and carries their unit tests. + */ +int deviceMatchScore( + const std::wstring& candidateName, + const std::wstring& candidateLink, + const std::wstring& requestedName, + const std::wstring& requestedId); + +/** + * The candidate a camera should open, or -1 for none. + * + * Claimed candidates are skipped; among the rest the best `deviceMatchScore` + * wins, the earlier one on a tie. With nothing requested that is the first + * unclaimed device. With a name or id requested, a candidate scoring 0 is never + * picked -- the same rule as before claims existed, and what lets the caller + * fall back to DirectShow. With an empty claim set this is exactly the old + * selection. + */ +int selectUnclaimedDevice( + const std::vector& candidates, + const std::wstring& requestedName, + const std::wstring& requestedId, + const DeviceClaims& claims); diff --git a/electron/native/wgc-capture/src/device_selection_test.cpp b/electron/native/wgc-capture/src/device_selection_test.cpp new file mode 100644 index 0000000000..aa6329f66e --- /dev/null +++ b/electron/native/wgc-capture/src/device_selection_test.cpp @@ -0,0 +1,100 @@ +#include "device_selection.h" + +#include +#include +#include + +namespace { +int failures = 0; +void expect(bool ok, const char* label) { + if (!ok) { + std::printf("FAIL %s\n", label); + ++failures; + } +} +} // namespace + +int main() { + const std::vector twins = { + {L"USB Camera", L"\\\\?\\usb#vid_0c45&pid_6366&mi_00#7&aaa&0&0000#{e5323777-f976-4f5b-9b55-b94699c46e44}\\global"}, + {L"USB Camera", L"\\\\?\\usb#vid_0c45&pid_6366&mi_00#7&bbb&0&0000#{e5323777-f976-4f5b-9b55-b94699c46e44}\\global"}, + }; + + DeviceClaims none; + expect(selectUnclaimedDevice(twins, L"USB Camera", L"", none) == 0, + "same names, nothing claimed: first"); + + DeviceClaims first; + first.add(twins[0].identity); + expect(selectUnclaimedDevice(twins, L"USB Camera", L"", first) == 1, + "same names, first claimed: second"); + + DeviceClaims both; + both.add(twins[0].identity); + both.add(twins[1].identity); + expect(selectUnclaimedDevice(twins, L"USB Camera", L"", both) == -1, + "same names, both claimed: none"); + expect(selectUnclaimedDevice(twins, L"", L"", both) == -1, + "nothing requested, both claimed: none"); + + DeviceClaims upper; + upper.add(L"\\\\?\\USB#VID_0C45&PID_6366&MI_00#7&AAA&0&0000#{E5323777-F976-4F5B-9B55-B94699C46E44}\\GLOBAL"); + expect(selectUnclaimedDevice(twins, L"USB Camera", L"", upper) == 1, + "claim comparison is case-insensitive"); + + // DirectShow registers the same device under its own interface class GUID. + DeviceClaims viaDirectShow; + viaDirectShow.add( + L"\\\\?\\usb#vid_0c45&pid_6366&mi_00#7&aaa&0&0000#{65e8773d-8f56-11d0-a3b9-00a0c9223196}\\global"); + expect(viaDirectShow.contains(twins[0].identity), "MF link and DirectShow path are one device"); + expect(!viaDirectShow.contains(twins[1].identity), "the twin is a different device"); + + const std::vector mixed = { + {L"Studio Cam", L"\\\\?\\usb#vid_1#a#{e5323777-f976-4f5b-9b55-b94699c46e44}\\global"}, + {L"Studio Cam Pro", L"\\\\?\\usb#vid_2#b#{e5323777-f976-4f5b-9b55-b94699c46e44}\\global"}, + {L"Other Camera", L"\\\\?\\usb#vid_3#c#{e5323777-f976-4f5b-9b55-b94699c46e44}\\global"}, + }; + DeviceClaims exact; + exact.add(mixed[0].identity); + expect(selectUnclaimedDevice(mixed, L"Studio Cam", L"", none) == 0, "exact name wins unclaimed"); + expect(selectUnclaimedDevice(mixed, L"Studio Cam", L"", exact) == 1, + "claimed exact match loses to a weaker unclaimed match"); + DeviceClaims bothStudio = exact; + bothStudio.add(mixed[1].identity); + expect(selectUnclaimedDevice(mixed, L"Studio Cam", L"", bothStudio) == -1, + "a non-matching device is never picked for a request"); + expect(selectUnclaimedDevice(mixed, L"Nonexistent", L"", none) == -1, + "no match for a request: none"); + expect(selectUnclaimedDevice(mixed, L"", L"", none) == 0, "nothing requested: first device"); + expect(selectUnclaimedDevice(mixed, L"", L"", exact) == 1, + "nothing requested: first unclaimed device"); + expect(selectUnclaimedDevice({}, L"", L"", none) == -1, "no candidates: none"); + + // Measured on an ARM64 dev machine: the front camera as Media Foundation and + // as DirectShow list it. + DeviceClaims frontCamera; + frontCamera.add( + L"\\\\?\\display#qcom_avstream_8380#3&2dd9d5f4&0&uid32768#{e5323777-f976-4f5b-9b55-b94699c46e44}" + L"\\{4faeafd4-041b-4e46-85fd-400473891182}"); + expect(frontCamera.contains( + L"\\\\?\\DISPLAY#QCOM_AVSTREAM_8380#3&2DD9D5F4&0&UID32768#{65E8773D-8F56-11D0-A3B9-00A0C9223196}" + L"\\{4FAEAFD4-041B-4E46-85FD-400473891182}"), + "measured MF link and DirectShow path are one device"); + expect(!frontCamera.contains( + L"\\\\?\\display#qcom_avstream_8380#3&2dd9d5f4&0&uid32768#{e5323777-f976-4f5b-9b55-b94699c46e44}" + L"\\{00000000-0000-0000-0000-000000000001}"), + "another reference string on the same device is another camera"); + + DeviceClaims empty; + empty.add(L""); + expect(!empty.contains(L""), "an empty identity is never claimed"); + expect(normalizeDeviceIdentity(L"@device:sw:{ABC}") == L"@device:sw:{abc}", + "identity without an interface class is only lowercased"); + + if (failures == 0) { + std::printf("device_selection_test: all assertions passed\n"); + return 0; + } + std::printf("device_selection_test: %d failure(s)\n", failures); + return 1; +} diff --git a/electron/native/wgc-capture/src/dshow_webcam_capture.cpp b/electron/native/wgc-capture/src/dshow_webcam_capture.cpp index f39e4b065a..837f70f68b 100644 --- a/electron/native/wgc-capture/src/dshow_webcam_capture.cpp +++ b/electron/native/wgc-capture/src/dshow_webcam_capture.cpp @@ -1,5 +1,9 @@ #include "dshow_webcam_capture.h" +#include "realtime_scheduling.h" +#include "webcam_format.h" +#include "webcam_loss.h" + #include #include #include @@ -29,6 +33,19 @@ ISampleGrabber : public IUnknown { virtual HRESULT STDMETHODCALLTYPE SetCallback(IUnknown* callback, long whichMethodToCallback) = 0; }; +/** + * Is this subtype something the graph must DECODE before this class can unpack it? + * + * Listed positively -- an unknown subtype counts as compressed -- because + * guessing wrong this way only costs resolution, while guessing wrong the other + * way costs the camera track entirely. + */ +bool isCompressedDshowSubtype(const GUID& subtype) { + return !(subtype == MEDIASUBTYPE_YUY2 || subtype == MEDIASUBTYPE_NV12 || + subtype == MEDIASUBTYPE_RGB32 || subtype == MEDIASUBTYPE_RGB24 || + subtype == MEDIASUBTYPE_UYVY || subtype == MEDIASUBTYPE_YV12); +} + bool succeeded(HRESULT hr, const char* label) { if (SUCCEEDED(hr)) { return true; @@ -93,6 +110,63 @@ std::array yuvToBgr(int y, int u, int v) { return {clampToByte(blue), clampToByte(green), clampToByte(red)}; } +std::wstring readPropertyString(IPropertyBag* properties, const wchar_t* name) { + VARIANT value; + VariantInit(&value); + std::wstring result; + if (SUCCEEDED(properties->Read(name, &value, nullptr)) && value.vt == VT_BSTR && value.bstrVal) { + result = value.bstrVal; + } + VariantClear(&value); + return result; +} + +/** + * The video input devices DirectShow lists for the filter `clsid`. + * + * The fallback opens its filter by CLSID, which says nothing about WHICH + * device that is. The moniker does: its DevicePath is the same interface path + * Media Foundation reports as the symbolic link, so a device already opened + * there is recognized here. A moniker without a DevicePath (a plain software + * filter) is named by its display name instead. + */ +std::vector enumerateDevicesForClsid(const CLSID& clsid) { + std::vector candidates; + Microsoft::WRL::ComPtr deviceEnumerator; + if (FAILED(CoCreateInstance(CLSID_SystemDeviceEnum, nullptr, CLSCTX_INPROC_SERVER, IID_PPV_ARGS(&deviceEnumerator)))) { + return candidates; + } + Microsoft::WRL::ComPtr monikers; + // S_FALSE: the category is empty. + if (deviceEnumerator->CreateClassEnumerator(CLSID_VideoInputDeviceCategory, &monikers, 0) != S_OK || !monikers) { + return candidates; + } + Microsoft::WRL::ComPtr moniker; + while (monikers->Next(1, &moniker, nullptr) == S_OK) { + Microsoft::WRL::ComPtr properties; + if (SUCCEEDED(moniker->BindToStorage(nullptr, nullptr, IID_PPV_ARGS(&properties)))) { + CLSID monikerClsid{}; + const std::wstring clsidText = readPropertyString(properties.Get(), L"CLSID"); + if (!clsidText.empty() && SUCCEEDED(CLSIDFromString(clsidText.c_str(), &monikerClsid)) && + IsEqualCLSID(monikerClsid, clsid)) { + DeviceCandidate candidate{ + readPropertyString(properties.Get(), L"FriendlyName"), + readPropertyString(properties.Get(), L"DevicePath")}; + if (candidate.identity.empty()) { + LPOLESTR displayName = nullptr; + if (SUCCEEDED(moniker->GetDisplayName(nullptr, nullptr, &displayName)) && displayName) { + candidate.identity = displayName; + CoTaskMemFree(displayName); + } + } + candidates.push_back(std::move(candidate)); + } + } + moniker.Reset(); + } + return candidates; +} + } // namespace struct DirectShowWebcamCapture::Impl { @@ -103,6 +177,8 @@ struct DirectShowWebcamCapture::Impl { Microsoft::WRL::ComPtr sampleGrabber; Microsoft::WRL::ComPtr nullRenderer; Microsoft::WRL::ComPtr mediaControl; + /** Where the graph says the device left; optional, see captureLoop. */ + Microsoft::WRL::ComPtr mediaEvent; bool comInitialized = false; bool running = false; }; @@ -112,10 +188,97 @@ DirectShowWebcamCapture::~DirectShowWebcamCapture() { delete impl_; } -bool DirectShowWebcamCapture::buildGraph(const CLSID& sourceClsid, const GUID* preferredSubtype) { +void DirectShowWebcamCapture::applyPreferredFormat(int requestedWidth, int requestedHeight, int requestedFps) { + Microsoft::WRL::ComPtr streamConfig; + if (FAILED(impl_->captureGraph->FindInterface( + &PIN_CATEGORY_CAPTURE, + &MEDIATYPE_Video, + impl_->captureFilter.Get(), + IID_PPV_ARGS(&streamConfig)))) { + // Virtual cameras commonly expose a single fixed format and no + // IAMStreamConfig at all. Nothing to choose from; let the graph connect. + return; + } + + int capCount = 0; + int capSize = 0; + if (FAILED(streamConfig->GetNumberOfCapabilities(&capCount, &capSize)) || + capSize != sizeof(VIDEO_STREAM_CONFIG_CAPS)) { + return; + } + + std::vector formats; + formats.reserve(static_cast(std::max(0, capCount))); + for (int index = 0; index < capCount; ++index) { + VIDEO_STREAM_CONFIG_CAPS caps{}; + AM_MEDIA_TYPE* mediaType = nullptr; + if (FAILED(streamConfig->GetStreamCaps(index, &mediaType, reinterpret_cast(&caps))) || !mediaType) { + continue; + } + if (mediaType->formattype == FORMAT_VideoInfo && mediaType->pbFormat) { + const auto* videoInfo = reinterpret_cast(mediaType->pbFormat); + // AvgTimePerFrame is in 100ns units; 333333 is 30 fps. + const int fps = videoInfo->AvgTimePerFrame > 0 + ? static_cast((10'000'000LL + videoInfo->AvgTimePerFrame / 2) / videoInfo->AvgTimePerFrame) + : 0; + formats.push_back(WebcamFormat{ + std::abs(videoInfo->bmiHeader.biWidth), + std::abs(videoInfo->bmiHeader.biHeight), + fps, + isCompressedDshowSubtype(mediaType->subtype)}); + } + freeMediaType(*mediaType); + CoTaskMemFree(mediaType); + } + + const WebcamFormat chosen = chooseWebcamFormat( + formats, + requestedWidth > 0 ? requestedWidth : 1920, + requestedHeight > 0 ? requestedHeight : 1080, + std::max(1, requestedFps)); + + // Second pass: set the first capability that matches the choice. GetStreamCaps + // hands back the media type we must give to SetFormat, so it has to be + // re-read rather than cached across the loop above. + for (int index = 0; index < capCount; ++index) { + VIDEO_STREAM_CONFIG_CAPS caps{}; + AM_MEDIA_TYPE* mediaType = nullptr; + if (FAILED(streamConfig->GetStreamCaps(index, &mediaType, reinterpret_cast(&caps))) || !mediaType) { + continue; + } + bool matched = false; + if (mediaType->formattype == FORMAT_VideoInfo && mediaType->pbFormat) { + auto* videoInfo = reinterpret_cast(mediaType->pbFormat); + if (std::abs(videoInfo->bmiHeader.biWidth) == chosen.width && + std::abs(videoInfo->bmiHeader.biHeight) == chosen.height && + isCompressedDshowSubtype(mediaType->subtype) == chosen.compressed) { + if (chosen.fps > 0) { + videoInfo->AvgTimePerFrame = 10'000'000LL / chosen.fps; + } + matched = SUCCEEDED(streamConfig->SetFormat(mediaType)); + if (matched) { + std::cerr << "INFO: DirectShow webcam format " << chosen.width << "x" << chosen.height + << "@" << chosen.fps << std::endl; + } + } + } + freeMediaType(*mediaType); + CoTaskMemFree(mediaType); + if (matched) { + return; + } + } +} + +bool DirectShowWebcamCapture::buildGraph( + const CLSID& sourceClsid, + const GUID* preferredSubtype, + int preferredWidth, + int preferredHeight) { // Every attempt starts from empty filters. A RenderStream that fails can // leave pins connected behind it, and retrying on top of that half-built // graph is how you get a second failure that says nothing about the format. + impl_->mediaEvent.Reset(); impl_->mediaControl.Reset(); impl_->nullRenderer.Reset(); impl_->sampleGrabber.Reset(); @@ -175,6 +338,8 @@ bool DirectShowWebcamCapture::buildGraph(const CLSID& sourceClsid, const GUID* p return false; } + applyPreferredFormat(preferredWidth, preferredHeight, fps_); + return succeeded(impl_->captureGraph->RenderStream( &PIN_CATEGORY_CAPTURE, &MEDIATYPE_Video, @@ -190,7 +355,8 @@ bool DirectShowWebcamCapture::initialize( const std::wstring& directShowClsid, int requestedWidth, int requestedHeight, - int requestedFps) { + int requestedFps, + const DeviceClaims& claims) { (void)deviceId; stop(); delete impl_; @@ -217,6 +383,27 @@ bool DirectShowWebcamCapture::initialize( } selectedDeviceName_ = deviceName.empty() ? directShowClsid : deviceName; + // Every device this filter stands for has already been matched by name in + // Electron, so the only thing left to choose on is which one is free. With + // no moniker naming the filter, the CLSID itself is the identity. + const std::vector candidates = enumerateDevicesForClsid(selectedClsid); + if (candidates.empty()) { + deviceIdentity_ = directShowClsid; + if (claims.contains(deviceIdentity_)) { + std::cerr << "ERROR: DirectShow webcam filter is already recording in this take" << std::endl; + return false; + } + } else { + const int selectedIndex = selectUnclaimedDevice(candidates, L"", L"", claims); + if (selectedIndex < 0) { + std::cerr << "ERROR: Every DirectShow webcam for this filter is already recording in this take" + << std::endl; + return false; + } + deviceIdentity_ = candidates[selectedIndex].identity; + } + std::wcerr << L"INFO: DirectShow webcam device " << deviceIdentity_ << std::endl; + // The camera's own format first, a forced RGB32 conversion only if we cannot // read it. // @@ -231,14 +418,14 @@ bool DirectShowWebcamCapture::initialize( // it connects with a subtype this file cannot read // (getopenscreen/openscreen#387). Naming a concrete subtype on the retry is // what makes DirectShow insert a colour converter for it. - if (!buildGraph(selectedClsid, nullptr)) { + if (!buildGraph(selectedClsid, nullptr, requestedWidth, requestedHeight)) { return false; } if (!resolveConnectedFormat(requestedWidth, requestedHeight, false)) { std::cerr << "WARNING: DirectShow webcam speaks a format this build cannot unpack; " "asking for RGB32 so the graph converts it" << std::endl; - if (!buildGraph(selectedClsid, &MEDIASUBTYPE_RGB32)) { + if (!buildGraph(selectedClsid, &MEDIASUBTYPE_RGB32, requestedWidth, requestedHeight)) { return false; } if (!resolveConnectedFormat(requestedWidth, requestedHeight, true)) { @@ -251,6 +438,10 @@ bool DirectShowWebcamCapture::initialize( if (!succeeded(impl_->graph.As(&impl_->mediaControl), "QueryInterface(IMediaControl)")) { return false; } + // Best-effort: without it a lost device goes unnoticed, as it always did. + if (FAILED(impl_->graph.As(&impl_->mediaEvent))) { + impl_->mediaEvent.Reset(); + } return true; } @@ -291,9 +482,28 @@ bool DirectShowWebcamCapture::resolveConnectedFormat( sourceStride_ = ((width_ * bitsPerPixel + 31) / 32) * 4; } sourceTopDown_ = pixelFormat_ != PixelFormat::Bgra || videoInfo->bmiHeader.biHeight < 0; + // The rate the graph settled on, not the one that was asked for. + // + // `chooseWebcamFormat` deliberately settles for less than the target + // when a camera offers nothing faster, and a driver may pick its own + // nearest rate after SetFormat regardless. `fps()` feeds three things + // that all have to agree with the frames actually arriving: the + // `webcam-format` event, the webcam encoder's nominal rate, and the + // constant-rate write interval in main.cpp. Left at the requested + // value, a 24 fps camera asked for 30 is encoded and paced as 30. + if (videoInfo->AvgTimePerFrame > 0) { + const int negotiated = static_cast( + (10'000'000LL + videoInfo->AvgTimePerFrame / 2) / videoInfo->AvgTimePerFrame); + if (negotiated > 0 && negotiated != fps_) { + std::cerr << "INFO: DirectShow webcam negotiated " << negotiated << " fps (asked for " + << fps_ << ")" << std::endl; + fps_ = std::clamp(negotiated, 1, 60); + } + } } std::cerr << "INFO: DirectShow webcam connected subtype " << guidToString(connectedType.subtype) - << " " << width_ << "x" << height_ << " stride=" << sourceStride_ << std::endl; + << " " << width_ << "x" << height_ << "@" << fps_ << " stride=" << sourceStride_ + << std::endl; freeMediaType(connectedType); if (width_ <= 0 || height_ <= 0) { width_ = requestedWidth > 0 ? requestedWidth : 1280; @@ -344,6 +554,7 @@ void DirectShowWebcamCapture::stop() { impl_->mediaControl->Stop(); } impl_->running = false; + impl_->mediaEvent.Reset(); impl_->mediaControl.Reset(); impl_->nullRenderer.Reset(); impl_->sampleGrabber.Reset(); @@ -358,8 +569,14 @@ void DirectShowWebcamCapture::stop() { } void DirectShowWebcamCapture::captureLoop() { + const MmcssThread mmcss(L"Capture"); const HRESULT coinitHr = CoInitializeEx(nullptr, COINIT_MULTITHREADED); while (!stopRequested_ && impl_ && impl_->sampleGrabber) { + // The sample grabber keeps handing back its last buffer after the + // camera is gone, so the frames cannot tell; the graph's events can. + if (deviceLeftGraph()) { + break; + } long bufferSize = 0; HRESULT hr = impl_->sampleGrabber->GetCurrentBuffer(&bufferSize, nullptr); if (SUCCEEDED(hr) && bufferSize > 0) { @@ -376,6 +593,32 @@ void DirectShowWebcamCapture::captureLoop() { } } +bool DirectShowWebcamCapture::deviceLeftGraph() { + if (!impl_->mediaEvent) { + return false; + } + long code = 0; + LONG_PTR param1 = 0; + LONG_PTR param2 = 0; + // A zero timeout drains what is queued without waiting for more. + while (SUCCEEDED(impl_->mediaEvent->GetEvent(&code, ¶m1, ¶m2, 0))) { + impl_->mediaEvent->FreeEventParams(code, param1, param2); + // EC_DEVICE_LOST's second parameter is 0 when the device was removed, + // 1 when it came back; a stream error or abort stops the graph. + const bool lost = (code == EC_DEVICE_LOST && param2 == 0) || code == EC_ERRORABORT || + code == EC_STREAM_ERROR_STOPPED; + if (lost && !lost_.exchange(true)) { + reportWebcamLost(selectedDeviceName_, static_cast(code)); + return true; + } + } + return false; +} + +bool DirectShowWebcamCapture::isLost() const { + return lost_; +} + void DirectShowWebcamCapture::storeFrame(const BYTE* buffer, long length) { const int destinationStride = width_ * 4; const int sourceStride = sourceStride_ > 0 ? sourceStride_ : destinationStride; @@ -452,17 +695,10 @@ void DirectShowWebcamCapture::storeFrame(const BYTE* buffer, long length) { latestFrameSequence_ += 1; } -bool DirectShowWebcamCapture::copyLatestFrame(WebcamFrameSnapshot& destination) { +bool DirectShowWebcamCapture::copyLatestFrame(WebcamFrameSnapshot& destination, uint64_t lastSeenSequence) { std::scoped_lock lock(frameMutex_); - if (latestFrame_.empty() || width_ <= 0 || height_ <= 0) { - return false; - } - - destination.data = latestFrame_; - destination.width = width_; - destination.height = height_; - destination.sequence = latestFrameSequence_; - return true; + return snapshotWebcamFrame( + latestFrame_, width_, height_, latestFrameSequence_, lastSeenSequence, destination); } int DirectShowWebcamCapture::width() const { @@ -477,6 +713,10 @@ int DirectShowWebcamCapture::fps() const { return fps_; } +const std::wstring& DirectShowWebcamCapture::deviceIdentity() const { + return deviceIdentity_; +} + const std::wstring& DirectShowWebcamCapture::selectedDeviceName() const { return selectedDeviceName_; } diff --git a/electron/native/wgc-capture/src/dshow_webcam_capture.h b/electron/native/wgc-capture/src/dshow_webcam_capture.h index cf510e2dfc..5e26098d09 100644 --- a/electron/native/wgc-capture/src/dshow_webcam_capture.h +++ b/electron/native/wgc-capture/src/dshow_webcam_capture.h @@ -1,5 +1,7 @@ #pragma once +#include "device_selection.h" + #include #include @@ -16,6 +18,33 @@ struct WebcamFrameSnapshot { uint64_t sequence = 0; }; +/** + * Copies the latest camera frame into `destination`, unless the caller already has it. + * + * The video writer asks on every screen tick, 60 times a second, while a camera + * delivers 30: copying first and comparing sequences after cost a full frame + * copied for nothing on every other tick. Returns false, leaving `destination` + * untouched, when there is no frame yet or its sequence is `lastSeenSequence`. + * Both capture backends route `copyLatestFrame` through this. + */ +inline bool snapshotWebcamFrame( + const std::vector& frame, + int width, + int height, + uint64_t sequence, + uint64_t lastSeenSequence, + WebcamFrameSnapshot& destination) { + if (frame.empty() || width <= 0 || height <= 0 || sequence == lastSeenSequence) { + return false; + } + + destination.data = frame; + destination.width = width; + destination.height = height; + destination.sequence = sequence; + return true; +} + class DirectShowWebcamCapture { public: DirectShowWebcamCapture() = default; @@ -30,15 +59,26 @@ class DirectShowWebcamCapture { const std::wstring& directShowClsid, int requestedWidth, int requestedHeight, - int requestedFps); + int requestedFps, + const DeviceClaims& claims); bool start(); void stop(); - bool copyLatestFrame(WebcamFrameSnapshot& destination); + bool copyLatestFrame(WebcamFrameSnapshot& destination, uint64_t lastSeenSequence); int width() const; int height() const; int fps() const; const std::wstring& selectedDeviceName() const; + /** + * The opened device's DevicePath (the moniker's display name when it has + * none, the filter CLSID when no moniker names it), for the take's claims. + */ + const std::wstring& deviceIdentity() const; + /** + * Did the graph report the device gone (EC_DEVICE_LOST removal, a stream + * error, or an abort)? Latched; safe to ask from any thread. + */ + bool isLost() const; void storeFrame(const BYTE* buffer, long length); private: @@ -50,6 +90,8 @@ class DirectShowWebcamCapture { struct Impl; void captureLoop(); + /** Drains the graph's queued events; true once one of them says the device left. */ + bool deviceLeftGraph(); /** * Builds source -> sample grabber -> null renderer and connects it. * @@ -59,7 +101,21 @@ class DirectShowWebcamCapture { * without leaving the graph half-built, so the caller can retry with a * different constraint. */ - bool buildGraph(const CLSID& sourceClsid, const GUID* preferredSubtype); + bool buildGraph( + const CLSID& sourceClsid, + const GUID* preferredSubtype, + int preferredWidth, + int preferredHeight); + /** + * Pins the capture pin to the best format the device offers, before the + * graph is rendered. + * + * Skipped, RenderStream's intelligent connect takes the pin's default + * format, which on a UVC camera is the first one it enumerates -- 640x480 + * on hardware that can do far better. Best-effort: a device without + * IAMStreamConfig, or one that rejects the format, still gets a graph. + */ + void applyPreferredFormat(int requestedWidth, int requestedHeight, int requestedFps); /** * Reads back what the graph actually negotiated and records how to unpack it. * @@ -72,6 +128,7 @@ class DirectShowWebcamCapture { Impl* impl_ = nullptr; std::thread thread_; std::atomic stopRequested_ = false; + std::atomic lost_ = false; std::mutex frameMutex_; std::vector latestFrame_; uint64_t latestFrameSequence_ = 0; @@ -82,4 +139,5 @@ class DirectShowWebcamCapture { bool sourceTopDown_ = false; PixelFormat pixelFormat_ = PixelFormat::Bgra; std::wstring selectedDeviceName_; + std::wstring deviceIdentity_; }; diff --git a/electron/native/wgc-capture/src/frame_visibility.cpp b/electron/native/wgc-capture/src/frame_visibility.cpp new file mode 100644 index 0000000000..d296b5f0ed --- /dev/null +++ b/electron/native/wgc-capture/src/frame_visibility.cpp @@ -0,0 +1,78 @@ +#include "frame_visibility.h" + +#include + +namespace { + +// Sampling at most this many pixels keeps the probe cheap on a 4K frame while +// still covering it evenly. +constexpr size_t kMaxSamples = 4096; + +// Luma is judged on a full-range 0..255 scale. A frame counts as visible when +// any sample is clearly above black, or when the picture is dim but not empty. +constexpr int kMaxLumaThreshold = 24; +constexpr int kAverageLumaThreshold = 4; + +// BT.709/BT.601 studio range: black sits at 16, white at 235. +constexpr int kStudioBlack = 16; +constexpr int kStudioSpan = 235 - kStudioBlack; + +bool isVisible(uint64_t lumaTotal, int maxLuma, size_t sampled) { + const uint64_t averageLuma = sampled > 0 ? lumaTotal / sampled : 0; + return maxLuma > kMaxLumaThreshold || averageLuma > kAverageLumaThreshold; +} + +} // namespace + +bool hasVisibleBgraContent(const std::vector& frame) { + if (frame.size() < 4) { + return false; + } + + uint64_t lumaTotal = 0; + int maxLuma = 0; + const size_t pixelCount = frame.size() / 4; + const size_t step = std::max(1, pixelCount / kMaxSamples); + size_t sampled = 0; + for (size_t pixel = 0; pixel < pixelCount; pixel += step) { + const size_t offset = pixel * 4; + const int b = frame[offset + 0]; + const int g = frame[offset + 1]; + const int r = frame[offset + 2]; + const int luma = (r * 54 + g * 183 + b * 19) >> 8; + lumaTotal += static_cast(luma); + maxLuma = std::max(maxLuma, luma); + sampled += 1; + } + + return isVisible(lumaTotal, maxLuma, sampled); +} + +bool hasVisibleNv12Content(const std::vector& frame) { + if (frame.size() < 6) { + return false; + } + + const size_t lumaCount = frame.size() * 2 / 3; + const size_t step = std::max(1, lumaCount / kMaxSamples); + uint64_t lumaTotal = 0; + int maxLuma = 0; + size_t sampled = 0; + for (size_t offset = 0; offset < lumaCount; offset += step) { + // Studio range mapped onto 0..255, so the thresholds mean the same here + // as they do for BGRA. Skip this and an all-black frame averages 16 -- + // past the average threshold on its own, so every black frame would + // count as a picture. + const int studio = static_cast(frame[offset]); + const int luma = std::clamp(((studio - kStudioBlack) * 255) / kStudioSpan, 0, 255); + lumaTotal += static_cast(luma); + maxLuma = std::max(maxLuma, luma); + sampled += 1; + } + + return isVisible(lumaTotal, maxLuma, sampled); +} + +bool hasVisibleWebcamContent(const std::vector& frame, bool isNv12) { + return isNv12 ? hasVisibleNv12Content(frame) : hasVisibleBgraContent(frame); +} diff --git a/electron/native/wgc-capture/src/frame_visibility.h b/electron/native/wgc-capture/src/frame_visibility.h new file mode 100644 index 0000000000..9f62c2298f --- /dev/null +++ b/electron/native/wgc-capture/src/frame_visibility.h @@ -0,0 +1,27 @@ +#pragma once + +#include +#include + +/** + * Does this frame contain any picture, or is it the blank one a camera emits + * while it is still warming up? + * + * Used before recording starts, to tell "the camera is running" from "the + * camera is open but has not produced anything yet". + */ +bool hasVisibleBgraContent(const std::vector& frame); + +/** + * The NV12 twin: luma is the Y plane, one byte per pixel, so no colour + * conversion is needed. The Y plane is the first two thirds of the buffer. + * + * A camera's NV12 is studio-range, where black is 16 rather than 0. Those + * values are normalised to full range before the thresholds are applied -- + * without that, the average of an all-black frame is 16, clears the average + * threshold on its own, and every black frame counts as visible. + */ +bool hasVisibleNv12Content(const std::vector& frame); + +/** Dispatches to the probe matching the frame's layout. */ +bool hasVisibleWebcamContent(const std::vector& frame, bool isNv12); diff --git a/electron/native/wgc-capture/src/frame_visibility_test.cpp b/electron/native/wgc-capture/src/frame_visibility_test.cpp new file mode 100644 index 0000000000..51e9f75313 --- /dev/null +++ b/electron/native/wgc-capture/src/frame_visibility_test.cpp @@ -0,0 +1,66 @@ +#include "frame_visibility.h" + +#include +#include +#include + +namespace { + +int failures = 0; + +void expectVisible(const std::string& label, bool actual, bool expected) { + if (actual == expected) { + return; + } + std::printf("FAIL %s: expected %s, got %s\n", label.c_str(), expected ? "visible" : "blank", + actual ? "visible" : "blank"); + ++failures; +} + +std::vector nv12(int width, int height, std::uint8_t luma) { + std::vector frame(static_cast(width) * height * 3 / 2, 128); + std::fill(frame.begin(), frame.begin() + static_cast(width) * height, luma); + return frame; +} + +std::vector bgra(int width, int height, std::uint8_t value) { + return std::vector(static_cast(width) * height * 4, value); +} + +} // namespace + +int main() { + // The regression this file exists for. A camera's NV12 is studio-range, so an + // all-black frame is a plane of 16, not 0. Averaged raw that is 16 -- past the + // average threshold on its own -- and every black frame counted as a picture, + // which is exactly what the warm-up probe exists to rule out. + expectVisible("studio-range black is blank", hasVisibleNv12Content(nv12(640, 480, 16)), false); + expectVisible("true zero is blank", hasVisibleNv12Content(nv12(640, 480, 0)), false); + + // Just above black stays blank; a clearly lit frame does not. + expectVisible("near-black is blank", hasVisibleNv12Content(nv12(640, 480, 20)), false); + expectVisible("mid grey is visible", hasVisibleNv12Content(nv12(640, 480, 128)), true); + expectVisible("studio white is visible", hasVisibleNv12Content(nv12(640, 480, 235)), true); + + // BGRA keeps its own behaviour: full-range, black is 0. + expectVisible("bgra black is blank", hasVisibleBgraContent(bgra(640, 480, 0)), false); + expectVisible("bgra grey is visible", hasVisibleBgraContent(bgra(640, 480, 128)), true); + + // Buffers too small to carry a frame must not be read past their end. + expectVisible("empty nv12", hasVisibleNv12Content({}), false); + expectVisible("empty bgra", hasVisibleBgraContent({}), false); + expectVisible("runt nv12", hasVisibleNv12Content({16, 16, 16}), false); + + // The dispatcher has to pick the layout it is told, not guess. The same bytes + // read as BGRA are bright; read as studio-range NV12 they are black. + const std::vector blackNv12 = nv12(320, 240, 16); + expectVisible("dispatch nv12", hasVisibleWebcamContent(blackNv12, true), false); + expectVisible("dispatch bgra", hasVisibleWebcamContent(bgra(320, 240, 200), false), true); + + if (failures == 0) { + std::printf("frame_visibility_test: all assertions passed\n"); + return 0; + } + std::printf("frame_visibility_test: %d assertion(s) failed\n", failures); + return 1; +} diff --git a/electron/native/wgc-capture/src/json_fields.cpp b/electron/native/wgc-capture/src/json_fields.cpp new file mode 100644 index 0000000000..e36a43e816 --- /dev/null +++ b/electron/native/wgc-capture/src/json_fields.cpp @@ -0,0 +1,125 @@ +#include "json_fields.h" + +#include +#include + +// Lightweight field lookups over the helper's single JSON argument. They match +// the first occurrence of a key anywhere in the document. + +bool findBool(const std::string& json, const std::string& key, bool fallback) { + auto pos = json.find("\"" + key + "\""); + if (pos == std::string::npos) { + return fallback; + } + pos = json.find(':', pos); + if (pos == std::string::npos) { + return fallback; + } + pos += 1; + while (pos < json.size() && std::isspace(static_cast(json[pos]))) { + pos += 1; + } + if (json.compare(pos, 4, "true") == 0) { + return true; + } + if (json.compare(pos, 5, "false") == 0) { + return false; + } + return fallback; +} + +int64_t findInt64(const std::string& json, const std::string& key, int64_t fallback) { + auto pos = json.find("\"" + key + "\""); + if (pos == std::string::npos) { + return fallback; + } + pos = json.find(':', pos); + if (pos == std::string::npos) { + return fallback; + } + pos += 1; + while (pos < json.size() && std::isspace(static_cast(json[pos]))) { + pos += 1; + } + try { + return std::stoll(json.substr(pos)); + } catch (...) { + return fallback; + } +} + +int findInt(const std::string& json, const std::string& key, int fallback) { + return static_cast(findInt64(json, key, fallback)); +} + +double findDouble(const std::string& json, const std::string& key, double fallback) { + auto pos = json.find("\"" + key + "\""); + if (pos == std::string::npos) { + return fallback; + } + pos = json.find(':', pos); + if (pos == std::string::npos) { + return fallback; + } + pos += 1; + while (pos < json.size() && std::isspace(static_cast(json[pos]))) { + pos += 1; + } + try { + return std::stod(json.substr(pos)); + } catch (...) { + return fallback; + } +} + +std::string findString(const std::string& json, const std::string& key) { + auto pos = json.find("\"" + key + "\""); + if (pos == std::string::npos) { + return {}; + } + pos = json.find(':', pos); + if (pos == std::string::npos) { + return {}; + } + pos += 1; + while (pos < json.size() && std::isspace(static_cast(json[pos]))) { + pos += 1; + } + if (pos >= json.size() || json[pos] != '"') { + return {}; + } + pos += 1; + + std::string result; + while (pos < json.size()) { + const char c = json[pos++]; + if (c == '"') { + break; + } + if (c == '\\' && pos < json.size()) { + const char escaped = json[pos++]; + switch (escaped) { + case '\\': + case '"': + case '/': + result.push_back(escaped); + break; + case 'n': + result.push_back('\n'); + break; + case 'r': + result.push_back('\r'); + break; + case 't': + result.push_back('\t'); + break; + default: + result.push_back(escaped); + break; + } + continue; + } + result.push_back(c); + } + return result; +} diff --git a/electron/native/wgc-capture/src/json_fields.h b/electron/native/wgc-capture/src/json_fields.h new file mode 100644 index 0000000000..0921b3381f --- /dev/null +++ b/electron/native/wgc-capture/src/json_fields.h @@ -0,0 +1,10 @@ +#pragma once + +#include +#include + +bool findBool(const std::string& json, const std::string& key, bool fallback); +int64_t findInt64(const std::string& json, const std::string& key, int64_t fallback); +int findInt(const std::string& json, const std::string& key, int fallback); +double findDouble(const std::string& json, const std::string& key, double fallback); +std::string findString(const std::string& json, const std::string& key); diff --git a/electron/native/wgc-capture/src/main.cpp b/electron/native/wgc-capture/src/main.cpp index c86524d6d3..77dc692db2 100644 --- a/electron/native/wgc-capture/src/main.cpp +++ b/electron/native/wgc-capture/src/main.cpp @@ -1,12 +1,16 @@ #include "audio_sample_utils.h" #include "desktop_icon_cover.h" #include "dpi_awareness.h" +#include "realtime_scheduling.h" #include "mf_encoder.h" #include "monitor_utils.h" #include "wasapi_device_watcher.h" #include "wasapi_loopback_capture.h" #include "wasapi_render_keepalive.h" +#include "frame_visibility.h" +#include "json_fields.h" #include "webcam_capture.h" +#include "webcam_config.h" #include "wgc_session.h" #include @@ -24,6 +28,7 @@ #include #include #include +#include namespace { @@ -35,7 +40,6 @@ struct CaptureConfig { std::string sourceId; std::string windowHandle; std::string outputPath; - std::string webcamOutputPath; int fps = 60; int width = 0; int height = 0; @@ -136,6 +140,49 @@ struct CaptureControl { } }; +// One camera of the take: its capture, its encoder and the writer-loop state that used +// to be single variables. All cameras share the recording's T0. +struct WebcamStream { + WebcamConfig config; + // The camera's position in the config list, kept when an earlier camera is + // dropped, so every event names the camera the caller asked for. + size_t index = 0; + WebcamCapture capture; + MFEncoder encoder; + bool active = false; + // False only for the legacy inline picture-in-picture camera, which has no + // file of its own and is drawn into the screen frame instead. + bool writeSeparate = false; + std::vector latestFrame; + int latestWidth = 0; + int latestHeight = 0; + uint64_t latestSequence = 0; + bool hasVisibleFrame = false; + int64_t lastTimestampHns = -1; + int64_t nextWriteDueHns = 0; + int64_t nominalIntervalHns = 0; + // Captured in the writer's pull block, submitted after it (issue #115). + Microsoft::WRL::ComPtr pendingSample; + // Owned here because the shutdown watchdog reads the current step's name + // through a raw pointer from another thread. + std::string finalizeStepName; + + // Takes the camera's newest frame if it carries a picture. Returns whether it did. + bool pullVisibleFrame() { + WebcamFrameSnapshot candidate; + if (!capture.copyLatestFrame(candidate, latestSequence) || + !hasVisibleWebcamContent(candidate.data, capture.deliversNv12())) { + return false; + } + latestFrame = std::move(candidate.data); + latestWidth = candidate.width; + latestHeight = candidate.height; + latestSequence = candidate.sequence; + hasVisibleFrame = true; + return true; + } +}; + int readEnvInt(const char* name, int fallback) { char raw[32]{}; const DWORD length = GetEnvironmentVariableA(name, raw, static_cast(sizeof(raw))); @@ -393,147 +440,13 @@ void reportCaptureAdapters(ID3D11Device* device, HMONITOR targetMonitor) { } } -bool hasVisibleBgraContent(const std::vector& frame) { - if (frame.size() < 4) { - return false; - } - - uint64_t lumaTotal = 0; - BYTE maxLuma = 0; - const size_t pixelCount = frame.size() / 4; - const size_t step = std::max(1, pixelCount / 4096); - size_t sampledPixels = 0; - for (size_t pixel = 0; pixel < pixelCount; pixel += step) { - const size_t offset = pixel * 4; - const BYTE b = frame[offset + 0]; - const BYTE g = frame[offset + 1]; - const BYTE r = frame[offset + 2]; - const BYTE luma = static_cast((static_cast(r) * 54 + static_cast(g) * 183 + static_cast(b) * 19) >> 8); - lumaTotal += luma; - maxLuma = std::max(maxLuma, luma); - sampledPixels += 1; - } - const uint64_t averageLuma = sampledPixels > 0 ? lumaTotal / sampledPixels : 0; - return maxLuma > 24 || averageLuma > 4; -} -bool findBool(const std::string& json, const std::string& key, bool fallback) { - auto pos = json.find("\"" + key + "\""); - if (pos == std::string::npos) { - return fallback; - } - pos = json.find(':', pos); - if (pos == std::string::npos) { - return fallback; - } - pos += 1; - while (pos < json.size() && std::isspace(static_cast(json[pos]))) { - pos += 1; - } - if (json.compare(pos, 4, "true") == 0) { - return true; - } - if (json.compare(pos, 5, "false") == 0) { - return false; - } - return fallback; -} - -int64_t findInt64(const std::string& json, const std::string& key, int64_t fallback) { - auto pos = json.find("\"" + key + "\""); - if (pos == std::string::npos) { - return fallback; - } - pos = json.find(':', pos); - if (pos == std::string::npos) { - return fallback; - } - pos += 1; - while (pos < json.size() && std::isspace(static_cast(json[pos]))) { - pos += 1; - } - try { - return std::stoll(json.substr(pos)); - } catch (...) { - return fallback; - } -} - -int findInt(const std::string& json, const std::string& key, int fallback) { - return static_cast(findInt64(json, key, fallback)); -} - -double findDouble(const std::string& json, const std::string& key, double fallback) { - auto pos = json.find("\"" + key + "\""); - if (pos == std::string::npos) { - return fallback; - } - pos = json.find(':', pos); - if (pos == std::string::npos) { - return fallback; - } - pos += 1; - while (pos < json.size() && std::isspace(static_cast(json[pos]))) { - pos += 1; - } - try { - return std::stod(json.substr(pos)); - } catch (...) { - return fallback; - } -} - -std::string findString(const std::string& json, const std::string& key) { - auto pos = json.find("\"" + key + "\""); - if (pos == std::string::npos) { - return {}; - } - pos = json.find(':', pos); - if (pos == std::string::npos) { - return {}; - } - pos += 1; - while (pos < json.size() && std::isspace(static_cast(json[pos]))) { - pos += 1; - } - if (pos >= json.size() || json[pos] != '"') { - return {}; - } - pos += 1; - - std::string result; - while (pos < json.size()) { - const char c = json[pos++]; - if (c == '"') { - break; - } - if (c == '\\' && pos < json.size()) { - const char escaped = json[pos++]; - switch (escaped) { - case '\\': - case '"': - case '/': - result.push_back(escaped); - break; - case 'n': - result.push_back('\n'); - break; - case 'r': - result.push_back('\r'); - break; - case 't': - result.push_back('\t'); - break; - default: - result.push_back(escaped); - break; - } - continue; - } - result.push_back(c); - } - return result; +// `code` stays ahead of `index`: older readers match on the `"code":…` substring. +void printWebcamUnavailable(size_t index, const std::string& deviceName, const char* message) { + std::cout << "{\"event\":\"warning\",\"code\":\"webcam-unavailable\",\"index\":" << index + << ",\"deviceName\":\"" << jsonEscape(deviceName) << "\",\"message\":\"" << message << "\"}" + << std::endl; } std::string parseWindowHandleFromSourceId(const std::string& sourceId) { @@ -607,7 +520,6 @@ bool parseConfig(const std::string& json, CaptureConfig& config) { config.webcamDeviceId = findString(json, "webcamDeviceId"); config.webcamDeviceName = findString(json, "webcamDeviceName"); config.webcamDirectShowClsid = findString(json, "webcamDirectShowClsid"); - config.webcamOutputPath = findString(json, "webcamPath"); config.webcamWidth = findInt(json, "webcamWidth", 0); config.webcamHeight = findInt(json, "webcamHeight", 0); config.webcamFps = findInt(json, "webcamFps", 0); @@ -673,6 +585,7 @@ int wmain(int argc, wchar_t* argv[]) { std::cerr << "ERROR: Could not enable per-monitor-v2 DPI awareness" << std::endl; return 1; } + const HighResolutionTiming highResolutionTiming; if (argc < 2) { std::cerr << "ERROR: Missing JSON config argument" << std::endl; @@ -681,8 +594,9 @@ int wmain(int argc, wchar_t* argv[]) { winrt::init_apartment(winrt::apartment_type::multi_threaded); + const std::string configJson = wideToUtf8(argv[1]); CaptureConfig config; - if (!parseConfig(wideToUtf8(argv[1]), config)) { + if (!parseConfig(configJson, config)) { std::cerr << "ERROR: Failed to parse config JSON" << std::endl; return 1; } @@ -753,7 +667,14 @@ int wmain(int argc, wchar_t* argv[]) { // A window is captured by whichever display it currently sits on, which // is the adapter that matters for the same reason a monitor's does. capturedMonitor = MonitorFromWindow(window, MONITOR_DEFAULTTONEAREST); - if (!session.initialize(window, config.fps, config.captureCursor)) { + // A window source would otherwise lose every context menu and dropdown, + // which are windows of their own (getopenscreen/openscreen#894). Windows 11 + // 24H2+ can draw them into this capture; OPENSCREEN_WGC_DISABLE_SECONDARY_ + // WINDOWS=1 restores the window-only capture. Never for a monitor source, + // which already contains everything on that screen. + const bool includeSecondaryWindows = + readEnvInt("OPENSCREEN_WGC_DISABLE_SECONDARY_WINDOWS", 0) != 1; + if (!session.initialize(window, config.fps, config.captureCursor, includeSecondaryWindows)) { std::cerr << "ERROR: Failed to initialize WGC window session" << std::endl; return 1; } @@ -775,37 +696,109 @@ int wmain(int argc, wchar_t* argv[]) { const int pixels = width * height; const int bitrate = pixels >= 3840 * 2160 ? 45'000'000 : pixels >= 2560 * 1440 ? 28'000'000 : 18'000'000; - WebcamCapture webcamCapture; - bool webcamActive = false; - bool writeSeparateWebcam = false; - if (config.webcamEnabled) { - if (!webcamCapture.initialize( - utf8ToWide(config.webcamDeviceId), - utf8ToWide(config.webcamDeviceName), - utf8ToWide(config.webcamDirectShowClsid), - config.webcamWidth, - config.webcamHeight, - config.webcamFps > 0 ? config.webcamFps : config.fps)) { + // Every camera of the take, in config order. The `webcams` list (or the + // legacy fields with a webcamPath) gives cameras that each write their own + // file. The legacy fields without a webcamPath are the one camera drawn + // into the screen frame as an inline picture-in-picture. + std::vector webcamConfigs = parseWebcamConfigs(configJson); + if (webcamConfigs.empty() && config.webcamEnabled) { + WebcamConfig inlineCamera; + inlineCamera.deviceId = config.webcamDeviceId; + inlineCamera.deviceName = config.webcamDeviceName; + inlineCamera.directShowClsid = config.webcamDirectShowClsid; + inlineCamera.width = config.webcamWidth; + inlineCamera.height = config.webcamHeight; + inlineCamera.fps = config.webcamFps; + webcamConfigs.push_back(std::move(inlineCamera)); + } + + std::vector> webcams; // unique_ptr: WebcamCapture/MFEncoder are not movable + // The devices this take's cameras opened, so two cameras of the same model + // (same name, and a browser id that matches no device) open two devices + // rather than the first one twice. A camera dropped later keeps its claim. + DeviceClaims deviceClaims; + for (size_t index = 0; index < webcamConfigs.size(); ++index) { + auto stream = std::make_unique(); + stream->config = webcamConfigs[index]; + stream->index = index; + stream->finalizeStepName = "webcam-encoder-finalize-" + std::to_string(index); + // Decided before initialize(), not after: it selects the capture pixel + // format, and only a camera going to its own file can use NV12 -- an inline + // picture-in-picture composite needs the frame as BGRA. + stream->writeSeparate = !stream->config.outputPath.empty(); + if (!stream->capture.initialize( + utf8ToWide(stream->config.deviceId), + utf8ToWide(stream->config.deviceName), + utf8ToWide(stream->config.directShowClsid), + stream->config.width, + stream->config.height, + stream->config.fps > 0 ? stream->config.fps : config.fps, + stream->writeSeparate, + deviceClaims)) { // Non-fatal: a screen+audio recording the user can still use is far // better than losing the whole recording because one camera device // didn't match. Report it so the renderer can inform the user (and, // historically, fall back to a browser-recorded webcam sidecar), but - // let capture continue without a native webcam track. - std::cerr << "WARNING: Failed to initialize native webcam capture; continuing without webcam" - << std::endl; - std::cout << "{\"event\":\"warning\",\"code\":\"webcam-unavailable\",\"message\":" - "\"Failed to initialize native webcam capture\"}" - << std::endl; - config.webcamEnabled = false; - } else { - std::cout << "{\"event\":\"webcam-format\",\"schemaVersion\":2,\"width\":" << webcamCapture.width() - << ",\"height\":" << webcamCapture.height() - << ",\"fps\":" << webcamCapture.fps() - << ",\"deviceName\":\"" << jsonEscape(wideToUtf8(webcamCapture.selectedDeviceName())) - << "\"}" << std::endl; - writeSeparateWebcam = !config.webcamOutputPath.empty(); + // let capture continue without this camera's track. + std::cerr << "WARNING: Failed to initialize native webcam capture for camera " << index + << "; continuing without it" << std::endl; + printWebcamUnavailable( + index, stream->config.deviceName, "Failed to initialize native webcam capture"); + continue; } - } + std::cout << "{\"event\":\"webcam-format\",\"schemaVersion\":2,\"index\":" << index + << ",\"width\":" << stream->capture.width() + << ",\"height\":" << stream->capture.height() + << ",\"fps\":" << stream->capture.fps() + << ",\"deviceName\":\"" << jsonEscape(wideToUtf8(stream->capture.selectedDeviceName())) + << "\"}" << std::endl; + stream->nominalIntervalHns = + static_cast(10'000'000ULL / std::max(1, stream->capture.fps())); + webcams.push_back(std::move(stream)); + } + // Only the legacy single camera without a file of its own is composited + // into the screen frame; every listed camera writes separately. + WebcamStream* inlineWebcam = + webcams.size() == 1 && !webcams[0]->writeSeparate ? webcams[0].get() : nullptr; + // A camera that opened but whose encoder or capture then would not start + // costs that camera, not the take -- with the screen and up to four cameras, + // running out of hardware encoder sessions is the likeliest way to get here. + // Only before the video writer runs: the vector is not touched after that. + const auto dropWebcam = [&](WebcamStream& stream, const char* message) { + std::cerr << "WARNING: " << message << " for camera " << stream.index << "; continuing without it" + << std::endl; + printWebcamUnavailable(stream.index, stream.config.deviceName, message); + stream.capture.stop(); + if (stream.writeSeparate) { + // Balances whatever initialize() got through, then removes the + // file it may have created: nothing was ever written to it. + stream.encoder.finalize(); + // A file that never got created is fine; one that could not be + // removed is left as a 0-byte stub, which the app cleans up. + if (!DeleteFileW(utf8ToWide(stream.config.outputPath).c_str())) { + const DWORD deleteError = GetLastError(); + if (deleteError != ERROR_FILE_NOT_FOUND) { + std::cerr << "WARNING: Could not remove the dropped camera file " + << stream.config.outputPath << " (GetLastError=" << deleteError << ")" + << std::endl; + } + } + } + stream.active = false; + if (inlineWebcam == &stream) { + inlineWebcam = nullptr; + } + }; + const auto eraseDroppedWebcams = [&](const std::vector& dropped) { + webcams.erase( + std::remove_if( + webcams.begin(), + webcams.end(), + [&](const std::unique_ptr& stream) { + return std::find(dropped.begin(), dropped.end(), stream.get()) != dropped.end(); + }), + webcams.end()); + }; WasapiLoopbackCapture loopbackCapture; WasapiLoopbackCapture microphoneCapture; @@ -930,13 +923,13 @@ int wmain(int argc, wchar_t* argv[]) { // // The other two conditions are unchanged and still required: software // encoding and inline webcam PiP both need the frame in system memory, - // which the DXGI path does not produce. config.webcamEnabled, not - // webcamActive -- the latter is only set once webcam capture has started, - // well after this. + // which the DXGI path does not produce. Decided from the initialized + // cameras, not from `active` -- that is only set once webcam capture has + // started, well after this. encoderOptions.useDxgiInput = readEnvInt("OPENSCREEN_WGC_ENABLE_DXGI_INPUT", 0) == 1 && !config.preferSoftwareEncoder && - (!config.webcamEnabled || writeSeparateWebcam); + inlineWebcam == nullptr; MFEncoder encoder; if (!encoder.initialize( @@ -975,27 +968,41 @@ int wmain(int argc, wchar_t* argv[]) { // on stop. << ",\"videoEncoderRuntime\":\"" << encoder.videoEncoderRuntime() << "\"}" << std::endl; - MFEncoder webcamEncoder; - if (writeSeparateWebcam) { + std::vector droppedWebcams; + for (const auto& stream : webcams) { + if (!stream->writeSeparate) { + continue; + } MFEncoderOptions webcamEncoderOptions = encoderOptions; webcamEncoderOptions.injectDefaultSinkWriterFailureOnce = false; webcamEncoderOptions.useDxgiInput = false; - const int webcamPixels = std::max(1, webcamCapture.width()) * std::max(1, webcamCapture.height()); - const int webcamBitrate = webcamPixels >= 1280 * 720 ? 8'000'000 : 4'000'000; - if (!webcamEncoder.initialize( - utf8ToWide(config.webcamOutputPath), - webcamCapture.width(), - webcamCapture.height(), - webcamCapture.fps(), + // The two-step ladder this replaces topped out at 8 Mbit/s for anything + // 720p or larger. That was sized for a camera nobody had configured + // above 640x480; now that the capture runs at the camera's real + // resolution, 8 Mbit/s starves a 1440p or 2160p frame badly enough to + // undo the extra pixels. The tiers mirror the screen ladder above. + const int webcamPixels = + std::max(1, stream->capture.width()) * std::max(1, stream->capture.height()); + const int webcamBitrate = webcamPixels >= 3840 * 2160 ? 40'000'000 + : webcamPixels >= 2560 * 1440 ? 24'000'000 + : webcamPixels >= 1920 * 1080 ? 16'000'000 + : webcamPixels >= 1280 * 720 ? 8'000'000 + : 4'000'000; + if (!stream->encoder.initialize( + utf8ToWide(stream->config.outputPath), + stream->capture.width(), + stream->capture.height(), + stream->capture.fps(), webcamBitrate, session.device(), session.context(), nullptr, webcamEncoderOptions)) { - std::cerr << "ERROR: Failed to initialize native webcam encoder" << std::endl; - return 1; + dropWebcam(*stream, "Failed to initialize native webcam encoder"); + droppedWebcams.push_back(stream.get()); } } + eraseDroppedWebcams(droppedWebcams); // By default, no mutex guards frame handoff: writeVideoFrames is the // only thread that ever touches WGC or latestFrameTexture. It pulls each @@ -1019,11 +1026,6 @@ int wmain(int argc, wchar_t* argv[]) { // them is the next bug report, and neither is worth a log line each. std::atomic contendedFrames = 0; Microsoft::WRL::ComPtr latestFrameTexture; - std::vector latestWebcamFrame; - int latestWebcamWidth = 0; - int latestWebcamHeight = 0; - uint64_t latestWebcamSequence = 0; - bool hasVisibleWebcamFrame = false; // Legacy-path-only state. frameMutex guards latestFrameTexture/ // legacyLatestFrameTimestampHns between WGC's callback thread (writer) @@ -1073,11 +1075,11 @@ int wmain(int argc, wchar_t* argv[]) { } auto writeVideoFrames = [&]() { + const MmcssThread mmcss(L"Capture"); const auto frameDuration = std::chrono::duration_cast( std::chrono::duration(1.0 / config.fps)); uint64_t frameIndex = 0; int64_t lastEncodedVideoTimestampHns = -1; - int64_t lastWebcamTimestampHns = -1; // Media Foundation's H.264 encoder MFT does not honor irregular input // sample times for a VFR source: it numbers output samples // sequentially at its configured nominal frame rate regardless of the @@ -1088,18 +1090,21 @@ int wmain(int argc, wchar_t* argv[]) { // webcam encoder on a real-time-paced cadence (duplicating the // latest available camera frame when the camera hasn't produced a // newer one yet), so "sample N is at N/fps" is actually correct. - int64_t nextWebcamWriteDueHns = 0; - const int64_t nominalWebcamIntervalHns = - static_cast(10'000'000ULL / std::max(1, webcamCapture.fps())); + // The cadence state is per camera: see WebcamStream. auto nextFrameDue = std::chrono::steady_clock::now(); int64_t firstFrameTimestampHns = -1; int64_t latestFrameTimestampHns = 0; while (!control.stopRequested && !encodeFailed) { Microsoft::WRL::ComPtr videoSample; - Microsoft::WRL::ComPtr webcamSample; bool hasVideoSample = false; - bool hasWebcamSample = false; + for (const auto& stream : webcams) { + stream->pendingSample.Reset(); + } + // Whether the picture this tick encodes differs from the last one: + // a new WGC frame, or a new camera frame drawn into it. The legacy + // callback path cannot tell, so it always reads back. + bool pictureChanged = legacyFrameCallback; std::unique_lock legacyLock; { @@ -1138,6 +1143,7 @@ int wmain(int argc, wchar_t* argv[]) { ID3D11Texture2D* wgcTexture = nullptr; int64_t wgcTimestampHns = 0; const bool gotFrame = session.tryGetNextFrame(&wgcTexture, &wgcTimestampHns); + pictureChanged = gotFrame; if (gotFrame) { if (!latestFrameTexture) { D3D11_TEXTURE2D_DESC desc{}; @@ -1171,22 +1177,35 @@ int wmain(int argc, wchar_t* argv[]) { continue; } } - if (webcamActive) { - WebcamFrameSnapshot candidateWebcamFrame; - if (webcamCapture.copyLatestFrame(candidateWebcamFrame) && - candidateWebcamFrame.sequence != latestWebcamSequence && - hasVisibleBgraContent(candidateWebcamFrame.data)) { - latestWebcamFrame = std::move(candidateWebcamFrame.data); - latestWebcamWidth = candidateWebcamFrame.width; - latestWebcamHeight = candidateWebcamFrame.height; - latestWebcamSequence = candidateWebcamFrame.sequence; - hasVisibleWebcamFrame = true; + // Screen first, then every camera, as before there was more than one. + for (const auto& stream : webcams) { + if (stream->active && stream->capture.isLost()) { + // Unplugged mid-take: its file ends here instead of + // repeating the last picture to the end, it is left out + // of `webcamPaths` and still finalized at stop. The + // screen and the other cameras go on. + std::cerr << "ERROR: Camera " << stream->index + << " was lost during the take; disabling it" << std::endl; + stream->active = false; + if (inlineWebcam == stream.get()) { + // Nor is its frozen picture drawn into the screen. + inlineWebcam = nullptr; + pictureChanged = true; + } + continue; + } + if (stream->active && stream->pullVisibleFrame()) { + pictureChanged = pictureChanged || !stream->writeSeparate; } } - const BgraFrameView webcamFrame{ - hasVisibleWebcamFrame && !latestWebcamFrame.empty() ? latestWebcamFrame.data() : nullptr, - latestWebcamWidth, - latestWebcamHeight, + // The frame composited into the screen picture, for the legacy + // inline camera only. + const BgraFrameView inlineWebcamFrame{ + inlineWebcam && inlineWebcam->hasVisibleFrame && !inlineWebcam->latestFrame.empty() + ? inlineWebcam->latestFrame.data() + : nullptr, + inlineWebcam ? inlineWebcam->latestWidth : 0, + inlineWebcam ? inlineWebcam->latestHeight : 0, }; const int64_t syntheticTimestampHns = static_cast((frameIndex * 10'000'000ULL) / config.fps); @@ -1204,16 +1223,24 @@ int wmain(int argc, wchar_t* argv[]) { frameTimestampHns = lastEncodedVideoTimestampHns + static_cast(10'000'000ULL / config.fps); } - if (writeSeparateWebcam && webcamFrame.data) { - // Anchor to the same recording-start origin as screen video/audio, - // using real elapsed host-clock time (not a synthetic frame-index - // clock) so a long recording can't accumulate clock-origin drift. - const auto elapsedSinceStart = std::chrono::steady_clock::now() - control.recordingStartedAt; - const int64_t elapsedHns = std::chrono::duration_cast< - std::chrono::duration>>(elapsedSinceStart) - .count(); - const int64_t targetElapsedHns = - std::max(0, elapsedHns - control.pausedDurationHns()); + // Anchor to the same recording-start origin as screen video/audio, + // using real elapsed host-clock time (not a synthetic frame-index + // clock) so a long recording can't accumulate clock-origin drift. + // Read once per tick: every camera of this tick shares it. + int64_t targetElapsedHns = -1; + for (const auto& stream : webcams) { + if (!stream->active || !stream->writeSeparate || !stream->hasVisibleFrame || + stream->latestFrame.empty()) { + continue; + } + if (targetElapsedHns < 0) { + const auto elapsedSinceStart = + std::chrono::steady_clock::now() - control.recordingStartedAt; + const int64_t elapsedHns = std::chrono::duration_cast< + std::chrono::duration>>(elapsedSinceStart) + .count(); + targetElapsedHns = std::max(0, elapsedHns - control.pausedDurationHns()); + } // The H.264 encoder MFT does not honor irregular per-sample // timestamps for a VFR source -- it numbers output samples // sequentially at its configured nominal rate regardless of the @@ -1222,27 +1249,40 @@ int wmain(int argc, wchar_t* argv[]) { // to feed the encoder *at* that nominal cadence, duplicating // the latest available camera frame when the camera hasn't // produced a newer one yet (VFR capture -> CFR encode resampling). - if (targetElapsedHns >= nextWebcamWriteDueHns) { - int64_t webcamTimestampHns = targetElapsedHns; - if (lastWebcamTimestampHns >= 0 && webcamTimestampHns <= lastWebcamTimestampHns) { - webcamTimestampHns = lastWebcamTimestampHns + nominalWebcamIntervalHns; - } - // Capture the sample here, but submit it to the sink - // writer OUTSIDE this block below (issue #115) so a - // slow WriteSample can't hold up the next frame pull. - hasWebcamSample = webcamEncoder.captureBgraSample(webcamFrame, webcamTimestampHns, webcamSample); - if (!hasWebcamSample) { - encodeFailed = true; - control.requestStop(); - break; - } - lastWebcamTimestampHns = webcamTimestampHns; - nextWebcamWriteDueHns += nominalWebcamIntervalHns; - if (nextWebcamWriteDueHns <= targetElapsedHns) { - // Fell behind (e.g. coming out of a pause, or a stall) -- - // resync to now instead of trying to catch up frame-by-frame. - nextWebcamWriteDueHns = targetElapsedHns + nominalWebcamIntervalHns; - } + if (targetElapsedHns < stream->nextWriteDueHns) { + continue; + } + int64_t webcamTimestampHns = targetElapsedHns; + if (stream->lastTimestampHns >= 0 && webcamTimestampHns <= stream->lastTimestampHns) { + webcamTimestampHns = stream->lastTimestampHns + stream->nominalIntervalHns; + } + const BgraFrameView webcamFrame{ + stream->latestFrame.data(), stream->latestWidth, stream->latestHeight}; + // Capture the sample here, but submit it to the sink + // writer OUTSIDE this block below (issue #115) so a + // slow WriteSample can't hold up the next frame pull. + const bool captured = + stream->capture.deliversNv12() + ? stream->encoder.captureNv12Sample( + Nv12FrameView{webcamFrame.data, webcamFrame.width, webcamFrame.height}, + webcamTimestampHns, + stream->pendingSample) + : stream->encoder.captureBgraSample( + webcamFrame, webcamTimestampHns, stream->pendingSample); + if (!captured) { + // One camera failing costs that camera, not the take. + std::cerr << "ERROR: Failed to capture a sample for camera " << stream->index + << "; disabling it" << std::endl; + stream->pendingSample.Reset(); + stream->active = false; + continue; + } + stream->lastTimestampHns = webcamTimestampHns; + stream->nextWriteDueHns += stream->nominalIntervalHns; + if (stream->nextWriteDueHns <= targetElapsedHns) { + // Fell behind (e.g. coming out of a pause, or a stall) -- + // resync to now instead of trying to catch up frame-by-frame. + stream->nextWriteDueHns = targetElapsedHns + stream->nominalIntervalHns; } } if (testStallReadbackMs > 0) { @@ -1266,7 +1306,12 @@ int wmain(int argc, wchar_t* argv[]) { // this struct's request: it falls back to the CPU path on // its own when the GPU path does not fit the machine. bool captured = false; - if (usesDxgiInput) { + if (!usesDxgiInput && !pictureChanged && + encoder.repeatLastVideoSample(frameTimestampHns, videoSample)) { + // Nothing new to read back (#925): the static screen, + // most of a demo, costs no GPU copy and no conversion. + captured = true; + } else if (usesDxgiInput) { captured = encoder.captureDxgiSample( latestFrameTexture.Get(), frameTimestampHns, @@ -1275,7 +1320,7 @@ int wmain(int argc, wchar_t* argv[]) { captured = encoder.captureVideoSample( latestFrameTexture.Get(), frameTimestampHns, - !writeSeparateWebcam && webcamFrame.data ? &webcamFrame : nullptr, + inlineWebcamFrame.data ? &inlineWebcamFrame : nullptr, videoSample); } if (!captured) { @@ -1315,10 +1360,15 @@ int wmain(int argc, wchar_t* argv[]) { // Stop detection has nothing to do with this ordering -- that is // CaptureControl::stopMutex/stopCv, checked by the loop condition // above, unrelated to sample submission (issue #252). - if (hasWebcamSample && !webcamEncoder.submitVideoSample(webcamSample.Get())) { - encodeFailed = true; - control.requestStop(); - break; + for (const auto& stream : webcams) { + if (stream->pendingSample && !stream->encoder.submitVideoSample(stream->pendingSample.Get())) { + // Disables this camera only; the screen and the other + // cameras keep recording. + std::cerr << "ERROR: Failed to submit a sample for camera " << stream->index + << "; disabling it" << std::endl; + stream->active = false; + } + stream->pendingSample.Reset(); } if (hasVideoSample && !encoder.submitVideoSample(videoSample.Get())) { encodeFailed = true; @@ -1444,41 +1494,52 @@ int wmain(int argc, wchar_t* argv[]) { stopRenderKeepAliveIfActive(); return 1; } - if (config.webcamEnabled) { - if (!webcamCapture.start()) { - microphoneCapture.stop(); - loopbackCapture.stop(); - stopDeviceWatchIfActive(); - stopRenderKeepAliveIfActive(); - if (audioMixer) { - audioMixer->stop(); - } - std::cerr << "ERROR: Failed to start native webcam capture" << std::endl; - return 1; + const auto stopWebcamCaptures = [&]() { + for (const auto& stream : webcams) { + stream->capture.stop(); } - webcamActive = true; + }; + droppedWebcams.clear(); + for (const auto& stream : webcams) { + if (!stream->capture.start()) { + dropWebcam(*stream, "Failed to start native webcam capture"); + droppedWebcams.push_back(stream.get()); + continue; + } + stream->active = true; + } + eraseDroppedWebcams(droppedWebcams); + if (!webcams.empty()) { + // One 3 s budget for all cameras together, not 3 s each: they warm up + // in parallel, and the screen recording should not start later per camera. const auto webcamDeadline = std::chrono::steady_clock::now() + std::chrono::seconds(3); - while (std::chrono::steady_clock::now() < webcamDeadline && !hasVisibleWebcamFrame) { - WebcamFrameSnapshot candidateWebcamFrame; - if (webcamCapture.copyLatestFrame(candidateWebcamFrame) && - hasVisibleBgraContent(candidateWebcamFrame.data)) { - latestWebcamFrame = std::move(candidateWebcamFrame.data); - latestWebcamWidth = candidateWebcamFrame.width; - latestWebcamHeight = candidateWebcamFrame.height; - latestWebcamSequence = candidateWebcamFrame.sequence; - hasVisibleWebcamFrame = true; + const auto allVisible = [&]() { + return std::all_of(webcams.begin(), webcams.end(), [](const auto& stream) { + return stream->hasVisibleFrame; + }); + }; + while (std::chrono::steady_clock::now() < webcamDeadline && !allVisible()) { + for (const auto& stream : webcams) { + if (!stream->hasVisibleFrame) { + stream->pullVisibleFrame(); + } + } + if (allVisible()) { break; } std::this_thread::sleep_for(std::chrono::milliseconds(20)); } - if (!hasVisibleWebcamFrame) { - std::cerr << "WARNING: Native webcam started but no visible frame was available before screen capture" - << std::endl; + for (const auto& stream : webcams) { + if (!stream->hasVisibleFrame) { + std::cerr << "WARNING: Native webcam " << stream->index + << " started but no visible frame was available before screen capture" + << std::endl; + } } } if (!session.start()) { - webcamCapture.stop(); + stopWebcamCaptures(); microphoneCapture.stop(); loopbackCapture.stop(); stopDeviceWatchIfActive(); @@ -1528,7 +1589,7 @@ int wmain(int argc, wchar_t* argv[]) { loopbackCapture.stop(); stopDeviceWatchIfActive(); stopRenderKeepAliveIfActive(); - webcamCapture.stop(); + stopWebcamCaptures(); if (audioMixer) { audioMixer->stop(); } @@ -1542,6 +1603,17 @@ int wmain(int argc, wchar_t* argv[]) { control.waitForStop(); + // The take ends here, for the sound as for the picture: the video writer + // stops on this same request. The mixer is clock-driven, so left running + // it would go on writing chunks through the half second the steps below + // take, and every take ended with ~0.5 s of audio past its video + // (getopenscreen/openscreen#942, measured on three takes). A pause writes + // what the cushion holds up to this instant and then nothing; one already + // in place is left where it is. + if (audioMixer) { + audioMixer->setPaused(true); + } + const auto stopStart = std::chrono::steady_clock::now(); auto stopElapsedMs = [&] { return std::chrono::duration_cast( @@ -1674,7 +1746,7 @@ int wmain(int argc, wchar_t* argv[]) { logStopStep("render-keepalive"); } beginStopStep("webcam", stepBudgetMs); - webcamCapture.stop(); + stopWebcamCaptures(); logStopStep("webcam"); beginStopStep("audio-mixer", stepBudgetMs); if (audioMixer) { @@ -1770,20 +1842,52 @@ int wmain(int argc, wchar_t* argv[]) { if (!encodeFailed && screenFinalized) { std::cout << "{\"event\":\"recording-stopped\",\"schemaVersion\":2,\"screenPath\":\"" << jsonEscape(config.outputPath) << "\""; - if (writeSeparateWebcam) { - std::cout << ",\"webcamPath\":\"" << jsonEscape(config.webcamOutputPath) << "\""; + // `webcamPath` stays for camera 0, as before; `webcamPaths` lists every + // camera still recording at stop, in index order. Both are printed + // before the camera files are finalized, for the reason above -- so they + // name the files that were being written, and a camera whose finalize + // fails below is reported on stderr, not removed from this list. + // `webcamPaths` is printed, possibly empty, whenever a camera wrote a + // file of its own: its presence is how the app knows a camera missing + // from it stopped early, rather than an older helper that never sent it. + std::vector recordedWebcams; + bool anySeparateWebcam = false; + for (const auto& stream : webcams) { + anySeparateWebcam = anySeparateWebcam || stream->writeSeparate; + if (stream->writeSeparate && stream->active) { + recordedWebcams.push_back(stream.get()); + } + } + if (!recordedWebcams.empty() && recordedWebcams.front()->index == 0) { + std::cout << ",\"webcamPath\":\"" << jsonEscape(recordedWebcams.front()->config.outputPath) + << "\""; + } + if (anySeparateWebcam) { + std::cout << ",\"webcamPaths\":["; + for (size_t i = 0; i < recordedWebcams.size(); ++i) { + std::cout << (i == 0 ? "\"" : ",\"") << jsonEscape(recordedWebcams[i]->config.outputPath) + << "\""; + } + std::cout << "]"; } std::cout << "}" << std::endl; std::cout << "Recording stopped. Output path: " << config.outputPath << std::endl; } + // Every camera that wrote a file is finalized, including one disabled + // mid-take: what it wrote before failing is still worth a playable index. bool webcamFinalized = true; - if (writeSeparateWebcam) { - beginStopStep("webcam-encoder-finalize", shutdownBudgetMs); - webcamFinalized = webcamEncoder.finalize(); - logStopStep("webcam-encoder-finalize"); - if (!webcamFinalized) { - std::cerr << "ERROR: Failed to finalize the webcam recording" << std::endl; + for (const auto& stream : webcams) { + if (!stream->writeSeparate) { + continue; + } + beginStopStep(stream->finalizeStepName.c_str(), shutdownBudgetMs); + const bool finalized = stream->encoder.finalize(); + logStopStep(stream->finalizeStepName.c_str()); + if (!finalized) { + std::cerr << "ERROR: Failed to finalize the webcam recording for camera " << stream->index + << std::endl; + webcamFinalized = false; } } diff --git a/electron/native/wgc-capture/src/mf_encoder.cpp b/electron/native/wgc-capture/src/mf_encoder.cpp index b4dabcc549..23bc69331e 100644 --- a/electron/native/wgc-capture/src/mf_encoder.cpp +++ b/electron/native/wgc-capture/src/mf_encoder.cpp @@ -10,6 +10,8 @@ #include #include +#include + #include #include #include @@ -592,6 +594,94 @@ struct StageGuard { } // namespace +// BGRA to NV12, BT.709 studio range, which is what the compositor decodes every +// recording as (getopenscreen/openscreen#923). Media Foundation's own colour +// converter, which the sink writer used to insert in front of the encoder for +// RGB32 input, produced BT.601 whatever the media types said. Chroma is the +// average of each 2x2 block, and the width must be even, as H.264 needs it anyway +// (main.cpp rounds the capture down); an odd last row repeats its neighbour. +// Fixed point: the coefficients are BT.709's scaled by 2^15 (luma) and 2^16 +// (chroma), so a solid primary lands within one code value of the table. +void convertBgraToNv12Bt709(const BYTE* bgra, int stride, int width, int height, BYTE* nv12) { + BYTE* luma = nv12; + BYTE* chroma = nv12 + static_cast(width) * height; + // Luma four pixels at a time in SSE2, which every x64 CPU has: this runs on + // the video writer's thread for every frame, and the scalar loop alone cost + // 2.5 ms a 1080p frame. 15-bit coefficients so they fit madd's int16 lanes. + const __m128i zero = _mm_setzero_si128(); + const __m128i lumaCoefficients = _mm_setr_epi16(2032, 20127, 5983, 0, 2032, 20127, 5983, 0); + const __m128i lumaRounding = _mm_set1_epi32(16384); + const __m128i lumaOffset = _mm_set1_epi32(16); + const auto lumaOfTwo = [&](__m128i pixels) { + // [b0*cb + g0*cg, r0*cr, b1*cb + g1*cg, r1*cr] -> each pixel's sum in lanes 0 and 2. + const __m128i products = _mm_madd_epi16(pixels, lumaCoefficients); + return _mm_add_epi32(products, _mm_shuffle_epi32(products, _MM_SHUFFLE(2, 3, 0, 1))); + }; + for (int y = 0; y < height; y += 1) { + const BYTE* row = bgra + static_cast(y) * stride; + BYTE* out = luma + static_cast(y) * width; + int x = 0; + for (; x + 4 <= width; x += 4) { + const __m128i pixels = _mm_loadu_si128(reinterpret_cast(row + x * 4)); + const __m128i first = lumaOfTwo(_mm_unpacklo_epi8(pixels, zero)); + const __m128i second = lumaOfTwo(_mm_unpackhi_epi8(pixels, zero)); + __m128i sums = _mm_castps_si128(_mm_shuffle_ps( + _mm_castsi128_ps(first), _mm_castsi128_ps(second), _MM_SHUFFLE(2, 0, 2, 0))); + sums = _mm_add_epi32(_mm_srai_epi32(_mm_add_epi32(sums, lumaRounding), 15), lumaOffset); + const __m128i bytes = _mm_packus_epi16(_mm_packs_epi32(sums, zero), zero); + const int packed = _mm_cvtsi128_si32(bytes); + std::memcpy(out + x, &packed, 4); + } + for (; x < width; x += 1) { + const int b = row[x * 4]; + const int g = row[x * 4 + 1]; + const int r = row[x * 4 + 2]; + out[x] = static_cast(((5983 * r + 20127 * g + 2032 * b + 16384) >> 15) + 16); + } + } + // Chroma the same way, two 2x2 blocks at a time: the four pixels of each + // block summed in int16 lanes (at most 1020), then one madd per channel. + const __m128i cbCoefficients = _mm_setr_epi16(28784, -22189, -6596, 0, 28784, -22189, -6596, 0); + const __m128i crCoefficients = _mm_setr_epi16(-2642, -26142, 28784, 0, -2642, -26142, 28784, 0); + const __m128i chromaRounding = _mm_set1_epi32(131072); + const __m128i chromaOffset = _mm_set1_epi32(128); + const auto chromaOfTwo = [&](__m128i blocks, __m128i coefficients) { + const __m128i products = _mm_madd_epi16(blocks, coefficients); + const __m128i sums = _mm_add_epi32(products, _mm_shuffle_epi32(products, _MM_SHUFFLE(2, 3, 0, 1))); + return _mm_shuffle_epi32(sums, _MM_SHUFFLE(2, 0, 2, 0)); + }; + for (int y = 0; y < height; y += 2) { + const BYTE* top = bgra + static_cast(y) * stride; + const BYTE* bottom = y + 1 < height ? top + stride : top; + BYTE* out = chroma + static_cast(y / 2) * width; + int x = 0; + for (; x + 4 <= width; x += 4) { + const __m128i upper = _mm_loadu_si128(reinterpret_cast(top + x * 4)); + const __m128i lower = _mm_loadu_si128(reinterpret_cast(bottom + x * 4)); + // [pixel 0 + pixel 1 | pixel 2 + pixel 3] of both rows, as int16 B, G, R, A. + const __m128i left = _mm_add_epi16(_mm_unpacklo_epi8(upper, zero), _mm_unpacklo_epi8(lower, zero)); + const __m128i right = _mm_add_epi16(_mm_unpackhi_epi8(upper, zero), _mm_unpackhi_epi8(lower, zero)); + const __m128i blocks = _mm_unpacklo_epi64( + _mm_add_epi16(left, _mm_srli_si128(left, 8)), _mm_add_epi16(right, _mm_srli_si128(right, 8))); + __m128i values = _mm_unpacklo_epi32( + chromaOfTwo(blocks, cbCoefficients), chromaOfTwo(blocks, crCoefficients)); + values = _mm_add_epi32(_mm_srai_epi32(_mm_add_epi32(values, chromaRounding), 18), chromaOffset); + const int packed = _mm_cvtsi128_si32(_mm_packus_epi16(_mm_packs_epi32(values, zero), zero)); + std::memcpy(out + x, &packed, 4); + } + for (; x < width; x += 2) { + const int right = (x + 1 < width ? x + 1 : x) * 4; + const int left = x * 4; + const int b = top[left] + top[right] + bottom[left] + bottom[right]; + const int g = top[left + 1] + top[right + 1] + bottom[left + 1] + bottom[right + 1]; + const int r = top[left + 2] + top[right + 2] + bottom[left + 2] + bottom[right + 2]; + // Sums of four, so the shift is two bits wider. + out[x] = static_cast(((-6596 * r - 22189 * g + 28784 * b + 131072) >> 18) + 128); + out[x + 1] = static_cast(((28784 * r - 26142 * g - 2642 * b + 131072) >> 18) + 128); + } + } +} + MFEncoder::~MFEncoder() { finalize(); } @@ -696,6 +786,7 @@ bool MFEncoder::initialize( if (!succeeded(MFStartup(MF_VERSION), "MFStartup")) { return false; } + mfStarted_ = true; if (useDxgiInput_ && !initializeDxgiPipeline()) { std::cerr << "WARNING: The GPU DXGI encode path is unavailable on this machine; " @@ -715,6 +806,21 @@ bool MFEncoder::initialize( setFrameSize(outputType.Get(), static_cast(width_), static_cast(height_)); setFrameRate(outputType.Get(), static_cast(fps_)); setPixelAspectRatio(outputType.Get()); + // Left unset, every encoder on the machine chose Constrained Baseline: one + // reference frame, CAVLC. Re-encoding real takes through the same NVENC MFT + // at an equal bitrate, High gained 1.7 dB PSNR on the screen and 1.9 VMAF on + // the webcam (getopenscreen/openscreen#922). Every hardware H.264 encoder and + // the Microsoft software one support it. No B-frames: the encoders add none + // unless asked, and they broke the fragmented writer on macOS. + outputType->SetUINT32(MF_MT_MPEG2_PROFILE, eAVEncH264VProfile_High); + // Carried on the H.264 type as well so the MP4 sink writes the matching + // colour tags instead of leaving players to guess from the frame size. + // Primaries and transfer too: without them the webcam track read + // `reserved`, and the screen track, untagged, read `unknown` throughout. + outputType->SetUINT32(MF_MT_VIDEO_NOMINAL_RANGE, MFNominalRange_16_235); + outputType->SetUINT32(MF_MT_YUV_MATRIX, MFVideoTransferMatrix_BT709); + outputType->SetUINT32(MF_MT_VIDEO_PRIMARIES, MFVideoPrimaries_BT709); + outputType->SetUINT32(MF_MT_TRANSFER_FUNCTION, MFVideoTransFunc_709); Microsoft::WRL::ComPtr inputType; if (!succeeded(MFCreateMediaType(&inputType), "MFCreateMediaType(input)")) { @@ -725,52 +831,50 @@ bool MFEncoder::initialize( // type the RGB32 path would have produced from scratch. Every attribute // one mode sets is deleted by the other; nothing carries over. auto configureVideoInputType = [&](bool dxgi) { + // NV12 on every path: GPU NV12, and system-memory NV12 -- from the + // camera as it came (the webcam), or converted here from BGRA + // (`convertBgraToNv12Bt709`). BGRA used to go in as RGB32, and the + // colour converter the sink writer put in front of the encoder turned it + // into BT.601, whatever the types said (#923). inputType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video); - inputType->SetGUID(MF_MT_SUBTYPE, dxgi ? MFVideoFormat_NV12 : MFVideoFormat_RGB32); + inputType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_NV12); inputType->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive); - if (dxgi) { - inputType->DeleteItem(MF_MT_DEFAULT_STRIDE); - // The video processor below converts full-range BGRA into - // studio-range BT.709, so say so. Left untagged, the encoder and - // the player each pick their own default (BT.601 is the common - // one) and the recording comes back with shifted colours the CPU - // path does not have. + if (!dxgi) { + // NV12's declared stride is the Y plane's, which is one byte per + // pixel -- not the four an RGB32 row needs. + inputType->SetUINT32(MF_MT_DEFAULT_STRIDE, static_cast(width_)); + // A camera's NV12 is studio-range, and at these sizes BT.709. + // Untagged, the encoder and the player each fall back to their own + // default and the recording comes back with shifted colours. inputType->SetUINT32(MF_MT_VIDEO_NOMINAL_RANGE, MFNominalRange_16_235); inputType->SetUINT32(MF_MT_YUV_MATRIX, MFVideoTransferMatrix_BT709); - } else { - inputType->SetUINT32(MF_MT_DEFAULT_STRIDE, static_cast(width_ * 4)); - inputType->DeleteItem(MF_MT_VIDEO_NOMINAL_RANGE); - inputType->DeleteItem(MF_MT_YUV_MATRIX); + setFrameSize(inputType.Get(), static_cast(width_), static_cast(height_)); + setFrameRate(inputType.Get(), static_cast(fps_)); + setPixelAspectRatio(inputType.Get()); + return; } + inputType->DeleteItem(MF_MT_DEFAULT_STRIDE); + // The video processor below converts full-range BGRA into + // studio-range BT.709, so say so. Left untagged, the encoder and + // the player each pick their own default (BT.601 is the common + // one) and the recording comes back with shifted colours. + inputType->SetUINT32(MF_MT_VIDEO_NOMINAL_RANGE, MFNominalRange_16_235); + inputType->SetUINT32(MF_MT_YUV_MATRIX, MFVideoTransferMatrix_BT709); setFrameSize(inputType.Get(), static_cast(width_), static_cast(height_)); setFrameRate(inputType.Get(), static_cast(fps_)); setPixelAspectRatio(inputType.Get()); }; - // Carried on the H.264 type as well so the MP4 sink writes the matching - // colour tags instead of leaving players to guess from the frame size. - auto configureOutputColorTags = [&](bool dxgi) { - if (dxgi) { - outputType->SetUINT32(MF_MT_VIDEO_NOMINAL_RANGE, MFNominalRange_16_235); - outputType->SetUINT32(MF_MT_YUV_MATRIX, MFVideoTransferMatrix_BT709); - } else { - outputType->DeleteItem(MF_MT_VIDEO_NOMINAL_RANGE); - outputType->DeleteItem(MF_MT_YUV_MATRIX); - } - }; - // The allocator is the last thing that can refuse the GPU path, and it can // only be built once the NV12 type exists. Falling back here costs nothing // but the type rewrite, because no sink writer has been created yet. configureVideoInputType(useDxgiInput_); - configureOutputColorTags(useDxgiInput_); if (useDxgiInput_ && !initializeSampleAllocator(inputType.Get())) { std::cerr << "WARNING: The DXGI sample allocator is unavailable on this machine; " << "using the CPU readback path." << std::endl; releaseDxgiPipeline(); useDxgiInput_ = false; configureVideoInputType(false); - configureOutputColorTags(false); } bool injectedDefaultSinkWriterFailure = false; @@ -869,8 +973,18 @@ bool MFEncoder::initialize( // track: an H.264 input type against an AAC stream sink has no encoder // that can bridge it, so a bad mapping fails loudly here instead of // quietly writing video samples into the audio track. - if (!succeeded(sinkWriter_->SetInputMediaType(videoStreamIndex_, inputType.Get(), nullptr), - "SetInputMediaType")) { + // No B-frames. High allows them and the Microsoft software encoder uses + // them unless told otherwise (has_b_frames=1, measured); their negative + // composition offsets are what broke the fragmented writer on macOS. + // Passed as encoding parameters because the encoder only reads this + // before its types are set: ICodecAPI::SetValue afterwards is refused. + Microsoft::WRL::ComPtr encodingParameters; + if (SUCCEEDED(MFCreateAttributes(&encodingParameters, 1))) { + encodingParameters->SetUINT32(CODECAPI_AVEncMPVDefaultBPictureCount, 0); + } + if (!succeeded( + sinkWriter_->SetInputMediaType(videoStreamIndex_, inputType.Get(), encodingParameters.Get()), + "SetInputMediaType")) { return false; } if (!forceSoftwareEncoder) { @@ -928,7 +1042,6 @@ bool MFEncoder::initialize( releaseDxgiPipeline(); useDxgiInput_ = false; configureVideoInputType(false); - configureOutputColorTags(false); if (configureSinkWriterAttempt(false, kVideoEncoderSelectionDefault, false, true)) { return true; } @@ -1078,19 +1191,24 @@ bool MFEncoder::copyFrameToBuffer( } const DWORD rowBytes = static_cast(width_ * 4); - const DWORD requiredBytes = rowBytes * static_cast(height_); - if (destinationSize < requiredBytes) { + if (destinationSize < static_cast(width_ * height_ * 3 / 2)) { context_->Unmap(stagingTexture_.Get(), 0); std::cerr << "ERROR: Media Foundation buffer is too small" << std::endl; return false; } auto* source = static_cast(mapped.pData); - for (int y = 0; y < height_; y += 1) { - std::memcpy(destination + rowBytes * y, source + mapped.RowPitch * y, rowBytes); - } if (webcamFrame) { - compositeWebcam(destination, width_, height_, *webcamFrame); + // The picture-in-picture is drawn in BGRA, so the frame goes through a + // copy it can be drawn on before the conversion. + bgraScratch_.resize(static_cast(rowBytes) * height_); + for (int y = 0; y < height_; y += 1) { + std::memcpy(bgraScratch_.data() + rowBytes * y, source + mapped.RowPitch * y, rowBytes); + } + compositeWebcam(bgraScratch_.data(), width_, height_, *webcamFrame); + convertBgraToNv12Bt709(bgraScratch_.data(), static_cast(rowBytes), width_, height_, destination); + } else { + convertBgraToNv12Bt709(source, static_cast(mapped.RowPitch), width_, height_, destination); } context_->Unmap(stagingTexture_.Get(), 0); @@ -1103,25 +1221,24 @@ bool MFEncoder::copyBgraFrameToBuffer(const BgraFrameView& frame, BYTE* destinat } const DWORD rowBytes = static_cast(width_ * 4); - const DWORD requiredBytes = rowBytes * static_cast(height_); - if (destinationSize < requiredBytes) { + if (destinationSize < static_cast(width_ * height_ * 3 / 2)) { std::cerr << "ERROR: Media Foundation webcam buffer is too small" << std::endl; return false; } if (frame.width == width_ && frame.height == height_) { - for (DWORD i = 0; i < requiredBytes; i += 4) { - destination[i] = frame.data[i]; - destination[i + 1] = frame.data[i + 1]; - destination[i + 2] = frame.data[i + 2]; - destination[i + 3] = 255; - } + // Straight from the camera's frame, never through a per-byte loop: one + // that forced alpha to 255 measured out at ~12 fps of real camera + // motion at 3840x2160, inside a file whose container claimed 30. The + // conversion ignores alpha anyway. + convertBgraToNv12Bt709(frame.data, static_cast(rowBytes), width_, height_, destination); return true; } + bgraScratch_.resize(static_cast(rowBytes) * height_); for (int y = 0; y < height_; y += 1) { const int sourceY = static_cast((static_cast(y) * frame.height) / height_); - BYTE* destinationRow = destination + rowBytes * y; + BYTE* destinationRow = bgraScratch_.data() + rowBytes * y; for (int x = 0; x < width_; x += 1) { const int sourceX = static_cast((static_cast(x) * frame.width) / width_); const BYTE* source = frame.data + (sourceY * frame.width + sourceX) * 4; @@ -1132,7 +1249,7 @@ bool MFEncoder::copyBgraFrameToBuffer(const BgraFrameView& frame, BYTE* destinat target[3] = 255; } } - + convertBgraToNv12Bt709(bgraScratch_.data(), static_cast(rowBytes), width_, height_, destination); return true; } @@ -1618,7 +1735,7 @@ bool MFEncoder::captureVideoSample( // which holds the shared frame-state mutex across this call but not // across submitVideoSample). Microsoft::WRL::ComPtr buffer; - const DWORD frameBytes = static_cast(width_ * height_ * 4); + const DWORD frameBytes = static_cast(width_ * height_ * 3 / 2); if (!succeeded(MFCreateMemoryBuffer(frameBytes, &buffer), "MFCreateMemoryBuffer")) { return false; } @@ -1645,6 +1762,76 @@ bool MFEncoder::captureVideoSample( sample->SetSampleTime(sampleTime); sample->SetSampleDuration(sampleDuration); + lastVideoBuffer_ = buffer; + outSample = sample; + return true; +} + +bool MFEncoder::repeatLastVideoSample(int64_t timestampHns, Microsoft::WRL::ComPtr& outSample) { + outSample.Reset(); + if (!lastVideoBuffer_) { + return false; + } + const int64_t sampleDuration = 10'000'000LL / fps_; + Microsoft::WRL::ComPtr sample; + if (!succeeded(MFCreateSample(&sample), "MFCreateSample(repeat)")) { + return false; + } + // Shared, not copied: a buffer is never written again once its sample is + // built, so two samples can hand the encoder the same bytes. + sample->AddBuffer(lastVideoBuffer_.Get()); + sample->SetSampleTime(nextSampleTime(timestampHns, sampleDuration)); + sample->SetSampleDuration(sampleDuration); + outSample = sample; + return true; +} + +bool MFEncoder::captureNv12Sample( + const Nv12FrameView& frame, + int64_t timestampHns, + Microsoft::WRL::ComPtr& outSample) { + outSample.Reset(); + + if (!frame.data || frame.width != width_ || frame.height != height_) { + // No rescaler here on purpose: the webcam encoder is created with the + // capture's own dimensions, so a mismatch means a bug upstream rather + // than a frame worth stretching. + std::cerr << "ERROR: NV12 webcam frame does not match the encoder's size" << std::endl; + return false; + } + + const int64_t sampleDuration = 10'000'000LL / fps_; + const int64_t sampleTime = nextSampleTime(timestampHns, sampleDuration); + const DWORD frameBytes = static_cast(width_ * height_ * 3 / 2); + + Microsoft::WRL::ComPtr buffer; + if (!succeeded(MFCreateMemoryBuffer(frameBytes, &buffer), "MFCreateMemoryBuffer(webcam NV12)")) { + return false; + } + + BYTE* data = nullptr; + DWORD maxLength = 0; + DWORD currentLength = 0; + if (!succeeded(buffer->Lock(&data, &maxLength, ¤tLength), "IMFMediaBuffer::Lock(webcam NV12)")) { + return false; + } + if (maxLength < frameBytes) { + buffer->Unlock(); + std::cerr << "ERROR: Media Foundation webcam NV12 buffer is too small" << std::endl; + return false; + } + std::memcpy(data, frame.data, frameBytes); + buffer->Unlock(); + buffer->SetCurrentLength(frameBytes); + + Microsoft::WRL::ComPtr sample; + if (!succeeded(MFCreateSample(&sample), "MFCreateSample(webcam NV12)")) { + return false; + } + sample->AddBuffer(buffer.Get()); + sample->SetSampleTime(sampleTime); + sample->SetSampleDuration(sampleDuration); + outSample = sample; return true; } @@ -1659,7 +1846,7 @@ bool MFEncoder::captureBgraSample( const int64_t sampleTime = nextSampleTime(timestampHns, sampleDuration); Microsoft::WRL::ComPtr buffer; - const DWORD frameBytes = static_cast(width_ * height_ * 4); + const DWORD frameBytes = static_cast(width_ * height_ * 3 / 2); if (!succeeded(MFCreateMemoryBuffer(frameBytes, &buffer), "MFCreateMemoryBuffer(webcam)")) { return false; } @@ -1798,6 +1985,9 @@ bool MFEncoder::finalize() { captureDevice_.Reset(); context_.Reset(); device_.Reset(); - MFShutdown(); + if (mfStarted_) { + MFShutdown(); + mfStarted_ = false; + } return ok; } diff --git a/electron/native/wgc-capture/src/mf_encoder.h b/electron/native/wgc-capture/src/mf_encoder.h index 8d1d6ae6e6..4c2034f873 100644 --- a/electron/native/wgc-capture/src/mf_encoder.h +++ b/electron/native/wgc-capture/src/mf_encoder.h @@ -11,6 +11,7 @@ #include #include #include +#include struct BgraFrameView { const BYTE* data = nullptr; @@ -18,6 +19,25 @@ struct BgraFrameView { int height = 0; }; +/** + * A planar NV12 frame: a width*height Y plane followed by an interleaved + * width/2 * height/2 UV plane, so width*height*3/2 bytes in total. + * + * Used for the webcam, whose camera can hand Media Foundation NV12 directly. + * Asking that same camera for RGB32 made the source reader decode and convert + * every frame, which measured at 92ms per 4K frame -- a hard ceiling near 11 + * fps -- against 32ms for NV12. + */ +struct Nv12FrameView { + const BYTE* data = nullptr; + int width = 0; + int height = 0; +}; + +// BGRA to NV12, BT.709 studio range, for every frame the encoder gets from +// system memory. `width` must be even. Exposed for mf_encoder_color_test. +void convertBgraToNv12Bt709(const BYTE* bgra, int stride, int width, int height, BYTE* nv12); + struct AudioInputFormat { GUID subtype = MFAudioFormat_PCM; UINT32 sampleRate = 0; @@ -118,10 +138,19 @@ class MFEncoder { int64_t timestampHns, const BgraFrameView* webcamFrame, Microsoft::WRL::ComPtr& outSample); + // The last frame captureVideoSample read back, again, at a new time: for a + // tick on which WGC delivered nothing new. No GPU copy, no Map, no + // conversion -- the readback a static screen used to pay on every tick + // (getopenscreen/openscreen#925). False before any frame was captured. + bool repeatLastVideoSample(int64_t timestampHns, Microsoft::WRL::ComPtr& outSample); bool captureDxgiSample( ID3D11Texture2D* texture, int64_t timestampHns, Microsoft::WRL::ComPtr& outSample); + bool captureNv12Sample( + const Nv12FrameView& frame, + int64_t timestampHns, + Microsoft::WRL::ComPtr& outSample); bool captureBgraSample( const BgraFrameView& frame, int64_t timestampHns, @@ -230,12 +259,22 @@ class MFEncoder { DWORD videoStreamIndex_ = 0; DWORD audioStreamIndex_ = 0; bool hasAudioStream_ = false; + // The BGRA frame when it has to be drawn on or rescaled before the NV12 + // conversion; reused so a frame does not allocate one. + std::vector bgraScratch_; + Microsoft::WRL::ComPtr lastVideoBuffer_; int width_ = 0; int height_ = 0; int fps_ = 60; int64_t firstTimestampHns_ = -1; int64_t lastTimestampHns_ = -1; bool finalized_ = false; + // Whether initialize() got as far as a successful MFStartup(). finalize() + // may only balance a startup this encoder made: MF's startup count is + // process-wide, and an encoder that was never initialized (a dropped + // camera's) would otherwise shut Media Foundation down under every other + // encoder and camera still running. + bool mfStarted_ = false; bool useDxgiInput_ = false; const char* videoEncoderSelection_ = kVideoEncoderSelectionDefault; const char* videoEncoderRuntime_ = kVideoEncoderRuntimeUnknown; diff --git a/electron/native/wgc-capture/src/mf_encoder_color_test.cpp b/electron/native/wgc-capture/src/mf_encoder_color_test.cpp new file mode 100644 index 0000000000..ac33a5e1b4 --- /dev/null +++ b/electron/native/wgc-capture/src/mf_encoder_color_test.cpp @@ -0,0 +1,475 @@ +// What the H.264 track actually says about itself (getopenscreen/openscreen#922, +// #923). Drives the real MFEncoder with synthetic BGRA frames -- three solid +// bands, red, green and blue -- into a temporary MP4, then reads the file back +// with ffprobe and ffmpeg: the profile, the colour tags, and the YUV values the +// encoder really produced for each band. The compositor decodes every +// recording as BT.709 limited range, so that is what the file has to be and +// has to say. +// +// Needs ffprobe and ffmpeg on PATH; without them the checks are skipped, not +// failed. The hardware encoder is covered when the host has one. + +#include "mf_encoder.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +int g_ran = 0; +int g_failed = 0; + +void expect(const std::string& name, bool ok, const std::string& detail) { + g_ran += 1; + std::cout << (ok ? "PASS " : "FAIL ") << name << (ok ? "" : " " + detail) << "\n"; + if (!ok) { + g_failed += 1; + } +} + +void skip(const std::string& name, const std::string& reason) { + std::cout << "SKIP " << name << " " << reason << "\n"; +} + +std::string run(const std::string& command) { + std::string output; + FILE* pipe = _popen(command.c_str(), "rb"); + if (!pipe) { + return output; + } + char buffer[65536]; + size_t read = 0; + while ((read = fread(buffer, 1, sizeof(buffer), pipe)) > 0) { + output.append(buffer, read); + } + _pclose(pipe); + return output; +} + +// GetTempPathA answers in the ANSI code page, which _popen also speaks; the +// encoder takes a wide path, so widen it through that code page, not per byte. +std::wstring widen(const std::string& ansi) { + const int size = MultiByteToWideChar(CP_ACP, 0, ansi.data(), static_cast(ansi.size()), nullptr, 0); + std::wstring result(size, L'\0'); + MultiByteToWideChar(CP_ACP, 0, ansi.data(), static_cast(ansi.size()), result.data(), size); + return result; +} + +bool toolsAvailable() { + return run("ffprobe -version 2>NUL").find("ffprobe") != std::string::npos && + run("ffmpeg -version 2>NUL").find("ffmpeg") != std::string::npos; +} + +std::string field(const std::string& probe, const std::string& key) { + const auto at = probe.find(key + "="); + if (at == std::string::npos) { + return ""; + } + const auto start = at + key.size() + 1; + const auto end = probe.find_first_of("\r\n", start); + return probe.substr(start, end - start); +} + +constexpr int kWidth = 480; +constexpr int kHeight = 272; +constexpr int kFrames = 30; + +struct Expected { + const char* name; + int y, cb, cr; +}; +// BT.709, studio range. BT.601 would put red at Y 81, Cb 90. +constexpr Expected kBands[] = {{"red", 63, 102, 240}, {"green", 173, 42, 26}, {"blue", 32, 240, 118}}; + +void checkEncoder(ID3D11Device* device, ID3D11DeviceContext* context, bool software) { + const std::string label = software ? "software" : "default"; + char tempDir[MAX_PATH]{}; + GetTempPathA(MAX_PATH, tempDir); + const std::string path = std::string(tempDir) + "openscreen-mf-encoder-color-" + label + ".mp4"; + const std::wstring widePath = widen(path); + DeleteFileA(path.c_str()); + + std::vector bgra(static_cast(kWidth) * kHeight * 4); + for (int y = 0; y < kHeight; y += 1) { + for (int x = 0; x < kWidth; x += 1) { + BYTE* pixel = &bgra[(static_cast(y) * kWidth + x) * 4]; + const int band = x * 3 / kWidth; + pixel[0] = band == 2 ? 255 : 0; // B + pixel[1] = band == 1 ? 255 : 0; // G + pixel[2] = band == 0 ? 255 : 0; // R + pixel[3] = 255; + } + } + + { + MFEncoder encoder; + MFEncoderOptions options; + options.preferSoftwareEncoder = software; + if (!encoder.initialize(widePath, kWidth, kHeight, 30, 2'000'000, device, context, nullptr, options)) { + expect("encoder-initialize-" + label, false, "initialize failed"); + return; + } + const std::string runtime = encoder.videoEncoderRuntime(); + std::cout << "COLOR_RAW " << label << " runtime=" << runtime << "\n"; + if (!software && runtime != kVideoEncoderRuntimeHardware) { + skip("encoder-" + label, "no hardware H.264 encoder on this host (" + runtime + ")"); + encoder.finalize(); + return; + } + const BgraFrameView frame{bgra.data(), kWidth, kHeight}; + bool wrote = true; + for (int i = 0; i < kFrames && wrote; i += 1) { + Microsoft::WRL::ComPtr sample; + wrote = encoder.captureBgraSample(frame, static_cast(i) * 333'333, sample) && + encoder.submitVideoSample(sample.Get()); + } + expect("encoder-writes-" + label, wrote && encoder.finalize(), "write or finalize failed"); + } + + const std::string probe = run( + "ffprobe -v error -select_streams v:0 -show_entries " + "stream=profile,has_b_frames,color_range,color_space,color_primaries,color_transfer -of default=nw=1 \"" + + path + "\""); + std::cout << "COLOR_RAW " << label << " profile=" << field(probe, "profile") + << " range=" << field(probe, "color_range") << " space=" << field(probe, "color_space") + << " primaries=" << field(probe, "color_primaries") + << " transfer=" << field(probe, "color_transfer") << "\n"; + expect("profile-high-" + label, field(probe, "profile") == "High", probe); + // B-frames broke the fragmented writer on macOS; the profile must not bring them. + expect("no-b-frames-" + label, field(probe, "has_b_frames") == "0", probe); + expect( + "colour-tags-bt709-limited-" + label, + field(probe, "color_range") == "tv" && field(probe, "color_space") == "bt709" && + field(probe, "color_primaries") == "bt709" && field(probe, "color_transfer") == "bt709", + probe); + + // The last frame, as the encoder wrote its samples: yuv420p out of a yuv420p + // stream is a straight copy, no range or matrix conversion on the way. + const std::string yuv = run( + "ffmpeg -v error -sseof -0.2 -i \"" + path + "\" -frames:v 1 -f rawvideo -pix_fmt yuv420p -"); + const size_t lumaSize = static_cast(kWidth) * kHeight; + if (yuv.size() < lumaSize * 3 / 2) { + expect("yuv-readback-" + label, false, "ffmpeg returned " + std::to_string(yuv.size()) + " bytes"); + return; + } + for (int band = 0; band < 3; band += 1) { + const int x = kWidth * (2 * band + 1) / 6; + const int y = kHeight / 2; + const int luma = static_cast(yuv[static_cast(y) * kWidth + x]); + const size_t chroma = static_cast(y / 2) * (kWidth / 2) + x / 2; + const int cb = static_cast(yuv[lumaSize + chroma]); + const int cr = static_cast(yuv[lumaSize + lumaSize / 4 + chroma]); + const Expected& want = kBands[band]; + char detail[128]{}; + sprintf_s( + detail, "%s Y=%d Cb=%d Cr=%d, BT.709 limited wants %d/%d/%d", want.name, luma, cb, cr, want.y, + want.cb, want.cr); + std::cout << "COLOR_RAW " << label << " " << detail << "\n"; + expect( + std::string("yuv-bt709-") + want.name + "-" + label, + std::abs(luma - want.y) <= 3 && std::abs(cb - want.cb) <= 3 && std::abs(cr - want.cr) <= 3, + detail); + } + DeleteFileA(path.c_str()); +} + +// The converter against BT.709 in double precision, on noise: solid bands +// cannot tell a 2x2 average from a wrong pairing of pixels, and 38 wide with a +// padded stride runs the SIMD body, its scalar tail and the row pitch. +void checkConverterAgainstReference() { + constexpr int width = 38; + constexpr int height = 22; + constexpr int stride = width * 4 + 24; + std::vector bgra(static_cast(stride) * height); + uint32_t seed = 12345; + for (BYTE& value : bgra) { + seed = seed * 1664525u + 1013904223u; + value = static_cast(seed >> 24); + } + std::vector nv12(static_cast(width) * height * 3 / 2); + convertBgraToNv12Bt709(bgra.data(), stride, width, height, nv12.data()); + + const double kr = 0.2126; + const double kb = 0.0722; + const auto channel = [&](int x, int y, int c) { return bgra[static_cast(y) * stride + x * 4 + c] / 255.0; }; + int worst = 0; + for (int y = 0; y < height; y += 1) { + for (int x = 0; x < width; x += 1) { + const double luma = kr * channel(x, y, 2) + (1 - kr - kb) * channel(x, y, 1) + kb * channel(x, y, 0); + const int want = static_cast(std::lround(16 + 219 * luma)); + worst = std::max(worst, std::abs(want - nv12[static_cast(y) * width + x])); + } + } + for (int y = 0; y < height; y += 2) { + for (int x = 0; x < width; x += 2) { + double r = 0, g = 0, b = 0; + for (int dy = 0; dy < 2; dy += 1) { + for (int dx = 0; dx < 2; dx += 1) { + r += channel(x + dx, y + dy, 2) / 4; + g += channel(x + dx, y + dy, 1) / 4; + b += channel(x + dx, y + dy, 0) / 4; + } + } + const double luma = kr * r + (1 - kr - kb) * g + kb * b; + const int cb = static_cast(std::lround(128 + 224 * (b - luma) / (2 * (1 - kb)))); + const int cr = static_cast(std::lround(128 + 224 * (r - luma) / (2 * (1 - kr)))); + const size_t at = static_cast(width) * height + static_cast(y / 2) * width + x; + worst = std::max({worst, std::abs(cb - nv12[at]), std::abs(cr - nv12[at + 1])}); + } + } + std::cout << "COLOR_RAW converter worst deviation from BT.709 = " << worst << " code values" << std::endl; + expect("converter-matches-bt709-reference", worst <= 1, "worst=" + std::to_string(worst)); +} + +// Not a pass/fail: what one 1080p frame costs to hand over and encode, for a +// before/after comparison of the conversion's cost. +void timeFullHd(ID3D11Device* device, ID3D11DeviceContext* context) { + constexpr int width = 1920; + constexpr int height = 1080; + constexpr int frames = 120; + char tempDir[MAX_PATH]{}; + GetTempPathA(MAX_PATH, tempDir); + const std::string path = std::string(tempDir) + "openscreen-mf-encoder-timing.mp4"; + const std::wstring widePath = widen(path); + std::vector bgra(static_cast(width) * height * 4); + for (size_t i = 0; i < bgra.size(); i += 1) { + bgra[i] = static_cast((i * 2654435761u) >> 24); + } + MFEncoder encoder; + if (!encoder.initialize(widePath, width, height, 60, 18'000'000, device, context, nullptr, {})) { + return; + } + const BgraFrameView frame{bgra.data(), width, height}; + double captureMs = 0.0; + double submitMs = 0.0; + for (int i = 0; i < frames; i += 1) { + Microsoft::WRL::ComPtr sample; + const auto start = std::chrono::steady_clock::now(); + encoder.captureBgraSample(frame, static_cast(i) * 166'667, sample); + const auto captured = std::chrono::steady_clock::now(); + encoder.submitVideoSample(sample.Get()); + const auto submitted = std::chrono::steady_clock::now(); + captureMs += std::chrono::duration(captured - start).count(); + submitMs += std::chrono::duration(submitted - captured).count(); + } + encoder.finalize(); + DeleteFileA(path.c_str()); + std::cout << "COLOR_RAW timing 1080p " << encoder.videoEncoderRuntime() << ": capture " + << captureMs / frames << " ms, submit " << submitMs / frames << " ms per frame" << std::endl; +} + +// A tick with no new WGC frame repeats the last buffer instead of reading the +// texture back again (getopenscreen/openscreen#925). The file must still hold +// one frame per tick, each the captured picture; and the repeat must cost next +// to nothing next to a readback, or it is not worth having. +void checkRepeatedFrames(ID3D11Device* device, ID3D11DeviceContext* context) { + const auto makeTexture = [&](int width, int height, Microsoft::WRL::ComPtr& texture) { + std::vector bgra(static_cast(width) * height * 4); + for (int y = 0; y < height; y += 1) { + for (int x = 0; x < width; x += 1) { + BYTE* pixel = &bgra[(static_cast(y) * width + x) * 4]; + pixel[2] = 255; // solid red + pixel[3] = 255; + } + } + D3D11_TEXTURE2D_DESC desc{}; + desc.Width = width; + desc.Height = height; + desc.MipLevels = 1; + desc.ArraySize = 1; + desc.Format = DXGI_FORMAT_B8G8R8A8_UNORM; + desc.SampleDesc.Count = 1; + desc.Usage = D3D11_USAGE_DEFAULT; + desc.BindFlags = D3D11_BIND_SHADER_RESOURCE; + D3D11_SUBRESOURCE_DATA initial{bgra.data(), static_cast(width * 4), 0}; + return SUCCEEDED(device->CreateTexture2D(&desc, &initial, &texture)); + }; + char tempDir[MAX_PATH]{}; + GetTempPathA(MAX_PATH, tempDir); + + { + const std::string path = std::string(tempDir) + "openscreen-mf-encoder-repeat.mp4"; + const std::wstring widePath = widen(path); + Microsoft::WRL::ComPtr texture; + MFEncoder encoder; + if (!makeTexture(kWidth, kHeight, texture) || + !encoder.initialize(widePath, kWidth, kHeight, 30, 2'000'000, device, context, nullptr, {})) { + expect("repeat-setup", false, "texture or encoder"); + return; + } + Microsoft::WRL::ComPtr sample; + bool wrote = !encoder.repeatLastVideoSample(0, sample) && // nothing to repeat yet + encoder.captureVideoSample(texture.Get(), 0, nullptr, sample) && encoder.submitVideoSample(sample.Get()); + for (int i = 1; i < kFrames && wrote; i += 1) { + wrote = encoder.repeatLastVideoSample(static_cast(i) * 333'333, sample) && + encoder.submitVideoSample(sample.Get()); + } + expect("repeat-writes", wrote && encoder.finalize(), "write or finalize failed"); + const std::string frames = run( + "ffprobe -v error -count_frames -select_streams v:0 -show_entries stream=nb_read_frames " + "-of default=nw=1:nk=1 \"" + path + "\""); + const std::string yuv = run( + "ffmpeg -v error -sseof -0.2 -i \"" + path + "\" -frames:v 1 -f rawvideo -pix_fmt yuv420p -"); + const int luma = yuv.empty() ? -1 : static_cast(yuv[static_cast(kHeight / 2) * kWidth + kWidth / 2]); + std::cout << "REPEAT_RAW frames=" << std::atoi(frames.c_str()) << " last-frame Y=" << luma << std::endl; + expect("repeat-keeps-one-frame-per-tick", std::atoi(frames.c_str()) == kFrames, frames); + expect("repeat-keeps-the-picture", std::abs(luma - 63) <= 3, "Y=" + std::to_string(luma)); + DeleteFileA(path.c_str()); + } + + { + const std::string path = std::string(tempDir) + "openscreen-mf-encoder-repeat-timing.mp4"; + const std::wstring widePath = widen(path); + Microsoft::WRL::ComPtr texture; + MFEncoder encoder; + if (!makeTexture(1920, 1080, texture) || + !encoder.initialize(widePath, 1920, 1080, 60, 18'000'000, device, context, nullptr, {})) { + return; + } + double readbackMs = 0.0; + double repeatMs = 0.0; + constexpr int rounds = 60; + for (int i = 0; i < rounds; i += 1) { + Microsoft::WRL::ComPtr sample; + const auto start = std::chrono::steady_clock::now(); + encoder.captureVideoSample(texture.Get(), static_cast(2 * i) * 166'667, nullptr, sample); + const auto read = std::chrono::steady_clock::now(); + encoder.submitVideoSample(sample.Get()); + const auto repeatStart = std::chrono::steady_clock::now(); + encoder.repeatLastVideoSample(static_cast(2 * i + 1) * 166'667, sample); + const auto repeated = std::chrono::steady_clock::now(); + encoder.submitVideoSample(sample.Get()); + readbackMs += std::chrono::duration(read - start).count(); + repeatMs += std::chrono::duration(repeated - repeatStart).count(); + } + encoder.finalize(); + DeleteFileA(path.c_str()); + std::cout << "REPEAT_RAW 1080p readback " << readbackMs / rounds << " ms, repeat " << repeatMs / rounds + << " ms per frame" << std::endl; + } +} + +// An encoder that was never initialized -- a camera the helper dropped before +// its encoder was set up -- must not touch Media Foundation when it is +// finalized or destroyed. Its finalize() used to call an unmatched MFShutdown(), +// which in the helper left every other encoder writing nothing (Finalize: +// MF_E_SINK_NO_SAMPLES_PROCESSED). With an initialized encoder alive, as here, +// that takes MF's count to zero under a live sink writer, and the live encoder +// then blocks instead of failing -- so the whole sequence runs on its own thread +// with a deadline, and a regression fails this test instead of hanging the +// build. Needs no ffmpeg. +void checkUninitializedEncoderLeavesOthersRunning() { + Microsoft::WRL::ComPtr device; + Microsoft::WRL::ComPtr context; + if (FAILED(D3D11CreateDevice( + nullptr, D3D_DRIVER_TYPE_HARDWARE, nullptr, D3D11_CREATE_DEVICE_BGRA_SUPPORT, nullptr, 0, + D3D11_SDK_VERSION, &device, nullptr, &context)) && + FAILED(D3D11CreateDevice( + nullptr, D3D_DRIVER_TYPE_WARP, nullptr, D3D11_CREATE_DEVICE_BGRA_SUPPORT, nullptr, 0, + D3D11_SDK_VERSION, &device, nullptr, &context))) { + skip("uninitialized-encoder-leaves-mf-running", "no D3D11 device"); + return; + } + char tempDir[MAX_PATH]{}; + GetTempPathA(MAX_PATH, tempDir); + const std::string path = std::string(tempDir) + "openscreen-mf-encoder-uninitialized-peer.mp4"; + DeleteFileA(path.c_str()); + + // 0 running, 1 passed, 2 failed, 3 skipped. + std::atomic outcome = 0; + std::thread worker([&] { + CoInitializeEx(nullptr, COINIT_MULTITHREADED); + { + MFEncoder live; + if (!live.initialize( + widen(path), kWidth, kHeight, 30, 2'000'000, device.Get(), context.Get(), nullptr, {})) { + outcome = 3; + } else { + { + MFEncoder neverInitialized; + neverInitialized.finalize(); + } // and destroyed, which finalizes again + Microsoft::WRL::ComPtr probe; + std::vector bgra(static_cast(kWidth) * kHeight * 4, 0x80); + const BgraFrameView frame{bgra.data(), kWidth, kHeight}; + bool wrote = SUCCEEDED(MFCreateSample(&probe)); + for (int i = 0; i < kFrames && wrote; i += 1) { + Microsoft::WRL::ComPtr sample; + wrote = live.captureBgraSample(frame, static_cast(i) * 333'333, sample) && + live.submitVideoSample(sample.Get()); + } + outcome = wrote && live.finalize() ? 1 : 2; + } + } + CoUninitialize(); + }); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(30); + while (outcome == 0 && std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + } + if (outcome == 0) { + expect("uninitialized-encoder-leaves-mf-running", false, "the live encoder blocked in Media Foundation"); + std::cout << "ran " << g_ran << " tests\n" << g_failed << " failed" << std::endl; + // The blocked thread can be neither joined nor safely abandoned. + TerminateProcess(GetCurrentProcess(), 1); + } + worker.join(); + DeleteFileA(path.c_str()); + if (outcome == 3) { + skip("uninitialized-encoder-leaves-mf-running", "encoder initialize failed on this host"); + return; + } + expect("uninitialized-encoder-leaves-mf-running", outcome == 1, "write or finalize failed"); +} + +} // namespace + +int main() { + checkConverterAgainstReference(); + if (SUCCEEDED(CoInitializeEx(nullptr, COINIT_MULTITHREADED))) { + checkUninitializedEncoderLeavesOthersRunning(); + CoUninitialize(); + } + if (!toolsAvailable()) { + skip("mf-encoder-color", "ffprobe/ffmpeg not on PATH"); + return g_failed == 0 ? 0 : 1; + } + if (FAILED(CoInitializeEx(nullptr, COINIT_MULTITHREADED))) { + skip("mf-encoder-color", "CoInitializeEx failed"); + return 0; + } + Microsoft::WRL::ComPtr device; + Microsoft::WRL::ComPtr context; + const UINT flags = D3D11_CREATE_DEVICE_BGRA_SUPPORT | D3D11_CREATE_DEVICE_VIDEO_SUPPORT; + if (FAILED(D3D11CreateDevice( + nullptr, D3D_DRIVER_TYPE_HARDWARE, nullptr, flags, nullptr, 0, D3D11_SDK_VERSION, &device, + nullptr, &context)) && + FAILED(D3D11CreateDevice( + nullptr, D3D_DRIVER_TYPE_WARP, nullptr, D3D11_CREATE_DEVICE_BGRA_SUPPORT, nullptr, 0, + D3D11_SDK_VERSION, &device, nullptr, &context))) { + skip("mf-encoder-color", "no D3D11 device"); + return 0; + } + checkEncoder(device.Get(), context.Get(), true); + checkEncoder(device.Get(), context.Get(), false); + timeFullHd(device.Get(), context.Get()); + checkRepeatedFrames(device.Get(), context.Get()); + + std::cout << "ran " << g_ran << " tests\n"; + if (g_failed != 0) { + std::cout << g_failed << " failed\n"; + return 1; + } + return 0; +} diff --git a/electron/native/wgc-capture/src/realtime_scheduling.h b/electron/native/wgc-capture/src/realtime_scheduling.h new file mode 100644 index 0000000000..dc8ca5abc6 --- /dev/null +++ b/electron/native/wgc-capture/src/realtime_scheduling.h @@ -0,0 +1,77 @@ +#pragma once + +#include +#include +#include + +#include + +// Every wait in the capture helpers -- the video writer's deadline, the WASAPI +// poll, the audio mixer's chunk clock, the cursor sampler's interval -- lands on +// the system timer tick, and a process that asks for nothing gets the default +// 15.625 ms one. Measured on a real take (getopenscreen/openscreen#920, #921): +// webcam frames spaced in whole ticks, 25.9 unique frames/s for a 30 fps camera; +// the cursor sampled at 22.7 Hz for 30 asked; WASAPI packets handed over late +// enough to leave holes in the voice (#911). +// +// Held for the life of the process. Windows 11 ignores the timer request of a +// process it throttles (a minimized or occluded app, which a recording helper +// always is), so the process is also opted out of power throttling for both +// execution speed and timer resolution. Either call failing is logged and +// survived: the recording still works at the coarser tick, as it always has. +class HighResolutionTiming { +public: + HighResolutionTiming() { + PROCESS_POWER_THROTTLING_STATE throttling{}; + throttling.Version = PROCESS_POWER_THROTTLING_CURRENT_VERSION; + throttling.ControlMask = + PROCESS_POWER_THROTTLING_EXECUTION_SPEED | PROCESS_POWER_THROTTLING_IGNORE_TIMER_RESOLUTION; + throttling.StateMask = 0; + if (!SetProcessInformation( + GetCurrentProcess(), ProcessPowerThrottling, &throttling, sizeof(throttling))) { + std::cerr << "WARNING: Could not opt out of power throttling (error " << GetLastError() + << ")" << std::endl; + } + periodSet_ = timeBeginPeriod(1) == TIMERR_NOERROR; + if (!periodSet_) { + std::cerr << "WARNING: Could not raise the timer resolution to 1 ms" << std::endl; + } + } + ~HighResolutionTiming() { + if (periodSet_) { + timeEndPeriod(1); + } + } + HighResolutionTiming(const HighResolutionTiming&) = delete; + HighResolutionTiming& operator=(const HighResolutionTiming&) = delete; + +private: + bool periodSet_ = false; +}; + +// Registers the calling thread with MMCSS for as long as it lives, so the +// scheduler favours it over ordinary work while the machine is busy -- the +// stalls that outlast the timer tick. `task` is a task name from the registry's +// MMCSS profile: "Pro Audio" for the audio threads, "Capture" for video. A thread +// that cannot register runs at its normal priority, as before. +class MmcssThread { +public: + explicit MmcssThread(const wchar_t* task) { + DWORD taskIndex = 0; + handle_ = AvSetMmThreadCharacteristicsW(task, &taskIndex); + if (!handle_) { + std::wcerr << L"WARNING: Could not register thread with MMCSS task \"" << task + << L"\" (error " << GetLastError() << L")" << std::endl; + } + } + ~MmcssThread() { + if (handle_) { + AvRevertMmThreadCharacteristics(handle_); + } + } + MmcssThread(const MmcssThread&) = delete; + MmcssThread& operator=(const MmcssThread&) = delete; + +private: + HANDLE handle_ = nullptr; +}; diff --git a/electron/native/wgc-capture/src/wasapi_loopback_capture.cpp b/electron/native/wgc-capture/src/wasapi_loopback_capture.cpp index c19c441604..875dda566a 100644 --- a/electron/native/wgc-capture/src/wasapi_loopback_capture.cpp +++ b/electron/native/wgc-capture/src/wasapi_loopback_capture.cpp @@ -1,4 +1,5 @@ #include "wasapi_loopback_capture.h" +#include "realtime_scheduling.h" #include #include @@ -352,6 +353,7 @@ bool WasapiLoopbackCapture::start(AudioCallback callback) { } thread_ = std::thread([this] { + const MmcssThread mmcss(L"Pro Audio"); captureLoop(); }); return true; diff --git a/electron/native/wgc-capture/src/webcam_capture.cpp b/electron/native/wgc-capture/src/webcam_capture.cpp index 6377a799b3..45d2ebdd79 100644 --- a/electron/native/wgc-capture/src/webcam_capture.cpp +++ b/electron/native/wgc-capture/src/webcam_capture.cpp @@ -1,13 +1,17 @@ #include "webcam_capture.h" +#include "realtime_scheduling.h" +#include "webcam_format.h" +#include "webcam_loss.h" + #include #include #include #include #include -#include #include +#include namespace { @@ -33,114 +37,6 @@ std::wstring readAllocatedString(IMFActivate* activate, REFGUID key) { return result; } -/** - * Does one of these appear inside the other as WHOLE WORDS? - * - * Plain containment answered for devices that merely share a spelling: a - * requested "Logi" is inside "Logitech", and "Micro" inside "Microphone", - * neither of them as a word. Matching on that resolved a camera nobody asked - * for -- and resolving one is exactly what stops the request reaching the - * DirectShow fallback, where the cameras Media Foundation cannot enumerate live. - * - * Both sides arrive normalized, so a boundary is the start of the string, its - * end, or a space. - */ -bool containsAsWords(const std::wstring& haystack, const std::wstring& needle) { - if (haystack.empty() || needle.empty()) { - return false; - } - size_t pos = haystack.find(needle); - while (pos != std::wstring::npos) { - const bool startsOnBoundary = pos == 0 || haystack[pos - 1] == L' '; - const size_t after = pos + needle.size(); - const bool endsOnBoundary = after == haystack.size() || haystack[after] == L' '; - if (startsOnBoundary && endsOnBoundary) { - return true; - } - pos = haystack.find(needle, pos + 1); - } - return false; -} - -bool containsInsensitive(const std::wstring& haystack, const std::wstring& needle) { - return containsAsWords(haystack, needle) || containsAsWords(needle, haystack); -} - -std::wstring normalizeDeviceName(const std::wstring& value) { - std::wstring normalized; - normalized.reserve(value.size()); - bool lastWasSpace = true; - for (const wchar_t ch : value) { - if (std::iswalnum(ch)) { - normalized.push_back(static_cast(std::towlower(ch))); - lastWasSpace = false; - continue; - } - if (!lastWasSpace) { - normalized.push_back(L' '); - lastWasSpace = true; - } - } - while (!normalized.empty() && normalized.back() == L' ') { - normalized.pop_back(); - } - return normalized; -} - -/** - * How well a candidate answers a requested name, or 0 for "not this one". - * - * Only decisive matches count: the names being equal once normalized, or one - * containing the other -- which is the ordinary case, since Chromium appends USB - * ids to what the driver reports. - * - * A further tier used to score shared WORDS, to bridge names differing more than - * that. It bridged names that were not the same device. "Logi Capture" and - * "Logitech StreamCam" share no word, yet "logi" sits inside "logitech" and that - * scored high enough to win -- so asking for a camera Media Foundation cannot - * enumerate opened a DIFFERENT camera, instead of returning nothing and letting - * the DirectShow fallback find the real one (getopenscreen/openscreen#405). - * - * Returning 0 is what makes that fallback reachable, so it is a real answer - * rather than a weak match. Keep this in step with - * `electron/recording/deviceNameMatching.ts`, which states the same rules for - * the Electron side and carries their unit tests. - */ -int deviceMatchScore( - const std::wstring& candidateName, - const std::wstring& candidateLink, - const std::wstring& requestedName, - const std::wstring& requestedId) { - int score = 0; - const auto normalizedName = normalizeDeviceName(candidateName); - const auto normalizedLink = normalizeDeviceName(candidateLink); - const auto normalizedRequestedName = normalizeDeviceName(requestedName); - const auto normalizedRequestedId = normalizeDeviceName(requestedId); - - if (!normalizedRequestedName.empty()) { - if (normalizedName == normalizedRequestedName) { - score = std::max(score, 1000); - } - if (containsInsensitive(normalizedName, normalizedRequestedName)) { - score = std::max(score, 900); - } - if (containsInsensitive(normalizedLink, normalizedRequestedName)) { - score = std::max(score, 800); - } - } - - if (!normalizedRequestedId.empty()) { - if (containsInsensitive(normalizedLink, normalizedRequestedId)) { - score = std::max(score, 700); - } - if (containsInsensitive(normalizedName, normalizedRequestedId)) { - score = std::max(score, 600); - } - } - - return score; -} - } // namespace WebcamCapture::~WebcamCapture() { @@ -153,53 +49,49 @@ bool WebcamCapture::initialize( const std::wstring& directShowClsid, int requestedWidth, int requestedHeight, - int requestedFps) { + int requestedFps, + bool preferNv12, + DeviceClaims& claims) { fps_ = std::clamp(requestedFps > 0 ? requestedFps : 30, 1, 60); usingDirectShow_ = false; - selectedMatchScore_ = 0; - if (!succeeded(MFStartup(MF_VERSION), "MFStartup(webcam)")) { - if (directShowCapture_.initialize(deviceId, deviceName, directShowClsid, requestedWidth, requestedHeight, fps_)) { - usingDirectShow_ = true; - return true; + selectedIdentity_.clear(); + const auto initializeDirectShow = [&]() { + if (!directShowCapture_.initialize( + deviceId, deviceName, directShowClsid, requestedWidth, requestedHeight, fps_, claims)) { + if (!deviceId.empty() || !deviceName.empty()) { + std::cerr << "ERROR: Requested webcam device was not found by native Windows webcam providers" + << std::endl; + } + return false; } - return false; + usingDirectShow_ = true; + claims.add(directShowCapture_.deviceIdentity()); + return true; + }; + + if (!succeeded(MFStartup(MF_VERSION), "MFStartup(webcam)")) { + return initializeDirectShow(); } mfStarted_ = true; - if (!selectDevice(deviceId, deviceName)) { + if (!selectDevice(deviceId, deviceName, claims)) { if (mfStarted_) { MFShutdown(); mfStarted_ = false; } - if (directShowCapture_.initialize(deviceId, deviceName, directShowClsid, requestedWidth, requestedHeight, fps_)) { - usingDirectShow_ = true; - return true; - } - return false; + return initializeDirectShow(); } - if ((!deviceId.empty() || !deviceName.empty()) && selectedMatchScore_ <= 0) { - if (mediaSource_) { - mediaSource_->Shutdown(); - } - sourceReader_.Reset(); - mediaSource_.Reset(); - if (mfStarted_) { - MFShutdown(); - mfStarted_ = false; - } - if (directShowCapture_.initialize(deviceId, deviceName, directShowClsid, requestedWidth, requestedHeight, fps_)) { - usingDirectShow_ = true; - return true; - } - std::cerr << "ERROR: Requested webcam device was not found by native Windows webcam providers" - << std::endl; + if (!configureReader(requestedWidth, requestedHeight, fps_, preferNv12)) { return false; } - - return configureReader(requestedWidth, requestedHeight, fps_); + claims.add(selectedIdentity_); + return true; } -bool WebcamCapture::selectDevice(const std::wstring& deviceId, const std::wstring& deviceName) { +bool WebcamCapture::selectDevice( + const std::wstring& deviceId, + const std::wstring& deviceName, + const DeviceClaims& claims) { Microsoft::WRL::ComPtr attributes; if (!succeeded(MFCreateAttributes(&attributes, 1), "MFCreateAttributes(webcam enumeration)")) { return false; @@ -222,37 +114,169 @@ bool WebcamCapture::selectDevice(const std::wstring& deviceId, const std::wstrin return false; } - UINT32 selectedIndex = 0; - int bestScore = 0; + std::vector candidates; + candidates.reserve(deviceCount); + bool anyClaimed = false; for (UINT32 index = 0; index < deviceCount; index += 1) { - const std::wstring name = readAllocatedString(devices[index], MF_DEVSOURCE_ATTRIBUTE_FRIENDLY_NAME); - const std::wstring symbolicLink = readAllocatedString(devices[index], MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_SYMBOLIC_LINK); - const int score = deviceMatchScore(name, symbolicLink, deviceName, deviceId); - std::wcerr << L"INFO: Native webcam candidate [" << index << L"] name=\"" << name << L"\" score=" << score << std::endl; - if (score > bestScore) { - selectedIndex = index; - bestScore = score; - } - } - - if ((!deviceId.empty() || !deviceName.empty()) && bestScore <= 0) { + DeviceCandidate candidate{ + readAllocatedString(devices[index], MF_DEVSOURCE_ATTRIBUTE_FRIENDLY_NAME), + readAllocatedString(devices[index], MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_SYMBOLIC_LINK)}; + const bool claimed = claims.contains(candidate.identity); + anyClaimed = anyClaimed || claimed; + std::wcerr << L"INFO: Native webcam candidate [" << index << L"] name=\"" << candidate.name + << L"\" score=" << deviceMatchScore(candidate.name, candidate.identity, deviceName, deviceId) + << (claimed ? L" (already recording in this take)" : L"") << std::endl; + candidates.push_back(std::move(candidate)); + } + + const int selectedIndex = selectUnclaimedDevice(candidates, deviceName, deviceId, claims); + if (selectedIndex >= 0) { + selectedDeviceName_ = candidates[selectedIndex].name; + selectedIdentity_ = candidates[selectedIndex].identity; + hr = devices[selectedIndex]->ActivateObject(IID_PPV_ARGS(&mediaSource_)); + } else if (anyClaimed) { + std::cerr << "WARNING: Every matching webcam is already recording in this take; trying DirectShow" + << std::endl; + } else { std::cerr << "WARNING: Requested webcam device was not found by Media Foundation; trying DirectShow" << std::endl; } - selectedMatchScore_ = bestScore; - selectedDeviceName_ = readAllocatedString(devices[selectedIndex], MF_DEVSOURCE_ATTRIBUTE_FRIENDLY_NAME); - hr = devices[selectedIndex]->ActivateObject(IID_PPV_ARGS(&mediaSource_)); - for (UINT32 index = 0; index < deviceCount; index += 1) { devices[index]->Release(); } CoTaskMemFree(devices); - return succeeded(hr, "ActivateObject(webcam)"); + return selectedIndex >= 0 && succeeded(hr, "ActivateObject(webcam)"); +} + +namespace { + +/** + * Is this subtype something the source reader must DECODE before it can convert? + * + * Listed positively -- an unknown subtype counts as compressed -- because the + * cost of guessing wrong the other way is a camera that silently records + * nothing, while guessing wrong this way only costs resolution. + */ +bool isCompressedVideoSubtype(const GUID& subtype) { + static const GUID* const kUncompressed[] = { + &MFVideoFormat_NV12, + &MFVideoFormat_YUY2, + &MFVideoFormat_RGB32, + &MFVideoFormat_RGB24, + &MFVideoFormat_ARGB32, + &MFVideoFormat_UYVY, + &MFVideoFormat_YV12, + &MFVideoFormat_I420, + &MFVideoFormat_IYUV, + }; + for (const GUID* candidate : kUncompressed) { + if (subtype == *candidate) { + return false; + } + } + return true; +} + +/** + * Every video format the camera advertises, read off the source reader. + * + * MF_MT_FRAME_RATE is a ratio; a 30000/1001 entry is NTSC 29.97 and rounds to + * 30, which is what we want to compare against the requested rate. A type + * missing either attribute is skipped rather than guessed at -- it cannot be + * requested by size anyway. + */ +std::vector readNativeFormats(IMFSourceReader* reader) { + std::vector formats; + for (DWORD index = 0;; ++index) { + Microsoft::WRL::ComPtr nativeType; + const HRESULT hr = + reader->GetNativeMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, index, &nativeType); + if (hr == MF_E_NO_MORE_TYPES || FAILED(hr)) { + break; + } + + UINT32 width = 0; + UINT32 height = 0; + if (FAILED(MFGetAttributeSize(nativeType.Get(), MF_MT_FRAME_SIZE, &width, &height))) { + continue; + } + + UINT32 numerator = 0; + UINT32 denominator = 0; + int fps = 0; + if (SUCCEEDED(MFGetAttributeRatio(nativeType.Get(), MF_MT_FRAME_RATE, &numerator, &denominator)) && + denominator > 0) { + fps = static_cast((numerator + denominator / 2) / denominator); + } + + GUID subtype{}; + const bool compressed = SUCCEEDED(nativeType->GetGUID(MF_MT_SUBTYPE, &subtype)) + ? isCompressedVideoSubtype(subtype) + : true; + + formats.push_back( + WebcamFormat{static_cast(width), static_cast(height), fps, compressed}); + } + return formats; } -bool WebcamCapture::configureReader(int requestedWidth, int requestedHeight, int requestedFps) { + +/** + * Puts the camera itself on `wanted` by selecting the matching native type. + * + * Best-effort: a driver that refuses leaves the reader on whatever it had, and + * the RGB32 request that follows still produces a usable -- if smaller -- frame. + */ +void selectNativeFormat(IMFSourceReader* reader, const WebcamFormat& wanted) { + for (DWORD index = 0;; ++index) { + Microsoft::WRL::ComPtr nativeType; + const HRESULT hr = + reader->GetNativeMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, index, &nativeType); + if (hr == MF_E_NO_MORE_TYPES || FAILED(hr)) { + return; + } + + UINT32 width = 0; + UINT32 height = 0; + if (FAILED(MFGetAttributeSize(nativeType.Get(), MF_MT_FRAME_SIZE, &width, &height))) { + continue; + } + if (static_cast(width) != wanted.width || static_cast(height) != wanted.height) { + continue; + } + + GUID subtype{}; + if (FAILED(nativeType->GetGUID(MF_MT_SUBTYPE, &subtype)) || + isCompressedVideoSubtype(subtype) != wanted.compressed) { + continue; + } + + UINT32 numerator = 0; + UINT32 denominator = 0; + if (wanted.fps > 0 && + SUCCEEDED(MFGetAttributeRatio(nativeType.Get(), MF_MT_FRAME_RATE, &numerator, &denominator)) && + denominator > 0) { + const int fps = static_cast((numerator + denominator / 2) / denominator); + if (fps != wanted.fps) { + continue; + } + } + + if (SUCCEEDED(reader->SetCurrentMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, nullptr, nativeType.Get()))) { + return; + } + } +} + +} // namespace + +bool WebcamCapture::configureReader( + int requestedWidth, + int requestedHeight, + int requestedFps, + bool preferNv12) { Microsoft::WRL::ComPtr attributes; if (!succeeded(MFCreateAttributes(&attributes, 3), "MFCreateAttributes(webcam reader)")) { return false; @@ -274,15 +298,57 @@ bool WebcamCapture::configureReader(int requestedWidth, int requestedHeight, int return false; } mediaType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video); - mediaType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_RGB32); - if (requestedWidth > 0 && requestedHeight > 0) { - MFSetAttributeSize(mediaType.Get(), MF_MT_FRAME_SIZE, static_cast(requestedWidth), static_cast(requestedHeight)); - } + // NV12 when the frame only has to reach the webcam encoder, which wants + // NV12 anyway. Asking for RGB32 instead makes the source reader decode AND + // convert every frame: measured at 92ms per 3840x2160 frame against 32ms + // for NV12, which is the difference between 11 fps and the camera's full + // 30. BGRA is still used when the frame is composited into the screen + // recording inline, which needs it in that layout. + deliversNv12_ = preferNv12; + mediaType->SetGUID(MF_MT_SUBTYPE, preferNv12 ? MFVideoFormat_NV12 : MFVideoFormat_RGB32); + + // Pick the capture format, then SELECT it on the source before asking for + // RGB32 output. + // + // Both halves matter. Handing Media Foundation an output type with no + // MF_MT_FRAME_SIZE leaves the device on its default format -- 640x480 on a + // BRIO that offers 3840x2160 -- and that default is what the overlay was + // upscaling. But naming the size on the *output* type alone is not enough + // either: with MF_SOURCE_READER_ENABLE_VIDEO_PROCESSING the reader answers + // S_OK by inserting a converter in front of whichever native type is + // already selected, so the camera keeps running at 640x480 and the size we + // asked for is quietly dropped. Selecting the native type first is what + // actually reconfigures the camera. + const WebcamFormat chosen = chooseWebcamFormat( + readNativeFormats(sourceReader_.Get()), + requestedWidth > 0 ? requestedWidth : 1920, + requestedHeight > 0 ? requestedHeight : 1080, + std::max(1, requestedFps)); + std::cerr << "INFO: Native webcam format " << chosen.width << "x" << chosen.height << "@" + << chosen.fps << (chosen.compressed ? " (compressed)" : " (uncompressed)") << std::endl; + selectNativeFormat(sourceReader_.Get(), chosen); + + MFSetAttributeSize( + mediaType.Get(), MF_MT_FRAME_SIZE, static_cast(chosen.width), static_cast(chosen.height)); MFSetAttributeRatio(mediaType.Get(), MF_MT_FRAME_RATE, static_cast(std::max(1, requestedFps)), 1); - if (!succeeded(sourceReader_->SetCurrentMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, nullptr, mediaType.Get()), - "SetCurrentMediaType(webcam RGB32)")) { - return false; + if (FAILED(sourceReader_->SetCurrentMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, nullptr, mediaType.Get()))) { + // Some drivers refuse a size they nonetheless enumerate. A slightly soft + // camera beats no camera, so drop the size and let the driver choose -- + // exactly what this code did before it asked for anything. + std::cerr << "WARNING: Webcam rejected " << chosen.width << "x" << chosen.height + << "; falling back to the device default format" << std::endl; + Microsoft::WRL::ComPtr fallbackType; + if (!succeeded(MFCreateMediaType(&fallbackType), "MFCreateMediaType(webcam fallback)")) { + return false; + } + fallbackType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video); + fallbackType->SetGUID(MF_MT_SUBTYPE, preferNv12 ? MFVideoFormat_NV12 : MFVideoFormat_RGB32); + if (!succeeded( + sourceReader_->SetCurrentMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, nullptr, fallbackType.Get()), + "SetCurrentMediaType(webcam RGB32)")) { + return false; + } } sourceReader_->SetStreamSelection(MF_SOURCE_READER_ALL_STREAMS, FALSE); sourceReader_->SetStreamSelection(MF_SOURCE_READER_FIRST_VIDEO_STREAM, TRUE); @@ -335,13 +401,19 @@ void WebcamCapture::stop() { } void WebcamCapture::captureLoop() { + const MmcssThread mmcss(L"Capture"); CoInitializeEx(nullptr, COINIT_MULTITHREADED); + const auto loopStartedAt = std::chrono::steady_clock::now(); + + // Start of the current run of failed reads; empty while reads succeed. + std::optional failureRunStartedAt; while (!stopRequested_) { DWORD streamIndex = 0; DWORD flags = 0; LONGLONG timestamp = 0; Microsoft::WRL::ComPtr sample; + const auto readStartedAt = std::chrono::steady_clock::now(); HRESULT hr = sourceReader_->ReadSample( MF_SOURCE_READER_FIRST_VIDEO_STREAM, 0, @@ -349,24 +421,61 @@ void WebcamCapture::captureLoop() { &flags, ×tamp, &sample); + readSampleUs_ += static_cast( + std::chrono::duration_cast( + std::chrono::steady_clock::now() - readStartedAt) + .count()); (void)streamIndex; (void)timestamp; if (FAILED(hr)) { - std::cerr << "WARNING: Failed to read webcam sample (hr=0x" << std::hex << hr << std::dec << ")" - << std::endl; + // Counted, not printed: a reader that fails every call would other- + // wise write thousands of identical lines over a long take. + readFailures_ += 1; + lastReadFailure_ = hr; + const auto now = std::chrono::steady_clock::now(); + if (!failureRunStartedAt) { + failureRunStartedAt = now; + } + const int64_t failingForMs = + std::chrono::duration_cast(now - *failureRunStartedAt).count(); + if (isWebcamLossResult(hr, false, failingForMs)) { + markLost(hr); + break; + } std::this_thread::sleep_for(std::chrono::milliseconds(20)); continue; } + failureRunStartedAt.reset(); if ((flags & MF_SOURCE_READERF_ENDOFSTREAM) != 0) { + sawEndOfStream_ = true; + // Stop has its own way out of this loop; end of stream before it + // is the camera leaving. + if (!stopRequested_ && isWebcamLossResult(hr, true, 0)) { + markLost(hr); + } break; } if (!sample) { + // Normal in small numbers (MF_SOURCE_READERF_STREAMTICK); a run of + // nothing else is a camera that never started producing. + emptySamples_ += 1; continue; } + // `Lock()` below is enough; row padding cannot reach `latestFrame_`. + // + // A 2-D media buffer may well pad each row out to a stride wider than + // the frame, but that padding is never what `Lock()` hands back: + // "The Lock method returns a buffer that is guaranteed to be + // contiguous. If the underlying buffer is not contiguous, the method + // copies the data into a new buffer" (IMF2DBuffer, Remarks). Reading + // the native pitch through IMF2DBuffer::Lock2D would only be worth it + // as a measured optimisation -- it saves that copy -- and mixing the + // two interfaces on one buffer is advised against in the same page. Microsoft::WRL::ComPtr buffer; if (FAILED(sample->ConvertToContiguousBuffer(&buffer)) || !buffer) { + bufferFailures_ += 1; continue; } @@ -374,36 +483,82 @@ void WebcamCapture::captureLoop() { DWORD maxLength = 0; DWORD currentLength = 0; if (FAILED(buffer->Lock(&data, &maxLength, ¤tLength)) || !data) { + bufferFailures_ += 1; continue; } - const DWORD expectedLength = static_cast(std::max(0, width_) * std::max(0, height_) * 4); + const int pixels = std::max(0, width_) * std::max(0, height_); + const DWORD expectedLength = + static_cast(deliversNv12_ ? pixels * 3 / 2 : pixels * 4); if (currentLength >= expectedLength && expectedLength > 0) { - std::scoped_lock lock(frameMutex_); - latestFrame_.assign(data, data + expectedLength); - latestFrameSequence_ += 1; + if (framesDelivered_ == 0) { + const auto waitedMs = std::chrono::duration_cast( + std::chrono::steady_clock::now() - loopStartedAt) + .count(); + std::cerr << "INFO: First webcam frame after " << waitedMs << "ms" << std::endl; + } + framesDelivered_ += 1; + const auto storeStartedAt = std::chrono::steady_clock::now(); + { + std::scoped_lock lock(frameMutex_); + latestFrame_.assign(data, data + expectedLength); + latestFrameSequence_ += 1; + } + storeUs_ += static_cast( + std::chrono::duration_cast( + std::chrono::steady_clock::now() - storeStartedAt) + .count()); + } else if (shortBuffers_++, !reportedShortBuffer_) { + // Every frame arriving short means the reader is handing us something + // other than the RGB32 we asked for -- a compressed frame it never + // converted, most often. Silently dropped, that reads downstream as a + // camera that produced nothing at all, with no clue why. Said once, + // because it is true for every frame that follows. + reportedShortBuffer_ = true; + std::cerr << "WARNING: Webcam frame is " << currentLength << " bytes, expected " + << expectedLength << " for " << width_ << "x" << height_ << " " + << (deliversNv12_ ? "NV12" : "RGB32") << "; dropping frames" << std::endl; } buffer->Unlock(); } + std::cerr << "INFO: Webcam capture loop ended: delivered=" << framesDelivered_ + << " emptySamples=" << emptySamples_ << " shortBuffers=" << shortBuffers_ + << " bufferFailures=" << bufferFailures_ << " readFailures=" << readFailures_ + << " lastReadHr=0x" << std::hex << lastReadFailure_ << std::dec + << " endOfStream=" << (sawEndOfStream_ ? "yes" : "no") + << " readSampleMs=" << (readSampleUs_ / 1000) << " storeMs=" << (storeUs_ / 1000) + << std::endl; + CoUninitialize(); } -bool WebcamCapture::copyLatestFrame(WebcamFrameSnapshot& destination) { +void WebcamCapture::markLost(HRESULT hr) { + if (lost_.exchange(true)) { + return; + } + reportWebcamLost(selectedDeviceName_, hr); +} + +bool WebcamCapture::isLost() const { if (usingDirectShow_) { - return directShowCapture_.copyLatestFrame(destination); + return directShowCapture_.isLost(); } - std::scoped_lock lock(frameMutex_); - if (latestFrame_.empty() || width_ <= 0 || height_ <= 0) { - return false; + return lost_; +} + +bool WebcamCapture::copyLatestFrame(WebcamFrameSnapshot& destination, uint64_t lastSeenSequence) { + if (usingDirectShow_) { + return directShowCapture_.copyLatestFrame(destination, lastSeenSequence); } + std::scoped_lock lock(frameMutex_); + return snapshotWebcamFrame( + latestFrame_, width_, height_, latestFrameSequence_, lastSeenSequence, destination); +} - destination.data = latestFrame_; - destination.width = width_; - destination.height = height_; - destination.sequence = latestFrameSequence_; - return true; +bool WebcamCapture::deliversNv12() const { + return !usingDirectShow_ && deliversNv12_; } int WebcamCapture::width() const { diff --git a/electron/native/wgc-capture/src/webcam_capture.h b/electron/native/wgc-capture/src/webcam_capture.h index 5b61aa6b98..5888e5ed99 100644 --- a/electron/native/wgc-capture/src/webcam_capture.h +++ b/electron/native/wgc-capture/src/webcam_capture.h @@ -1,5 +1,6 @@ #pragma once +#include "device_selection.h" #include "dshow_webcam_capture.h" #include @@ -22,32 +23,63 @@ class WebcamCapture { WebcamCapture(const WebcamCapture&) = delete; WebcamCapture& operator=(const WebcamCapture&) = delete; + /** + * Opens the requested camera, skipping devices in `claims`. + * + * `claims` belongs to the take: every camera of it is initialized against + * the same set, in config order, and one that opens adds its device. That + * is what sends a second camera of the same model to the second device + * instead of the busy first one. An empty set selects exactly as before. + */ bool initialize( const std::wstring& deviceId, const std::wstring& deviceName, const std::wstring& directShowClsid, int requestedWidth, int requestedHeight, - int requestedFps); + int requestedFps, + bool preferNv12, + DeviceClaims& claims); bool start(); void stop(); - bool copyLatestFrame(WebcamFrameSnapshot& destination); + bool copyLatestFrame(WebcamFrameSnapshot& destination, uint64_t lastSeenSequence); int width() const; int height() const; int fps() const; + /** + * Do the frames from `copyLatestFrame` carry NV12 rather than BGRA? + * + * Always false on the DirectShow fallback, which unpacks whatever the + * camera speaks into BGRA itself. Callers must ask rather than assume: + * the two layouts differ in size as well as in meaning. + */ + bool deliversNv12() const; const std::wstring& selectedDeviceName() const; + /** + * Has the camera left the take -- unplugged, or its driver given up? + * + * Latched: once true it stays true, and the camera delivers no more + * frames. Safe to ask from any thread. + */ + bool isLost() const; private: - bool selectDevice(const std::wstring& deviceId, const std::wstring& deviceName); - bool configureReader(int requestedWidth, int requestedHeight, int requestedFps); + bool selectDevice( + const std::wstring& deviceId, + const std::wstring& deviceName, + const DeviceClaims& claims); + bool configureReader(int requestedWidth, int requestedHeight, int requestedFps, bool preferNv12); void captureLoop(); + /** Latches `lost_` and says so once; see `isWebcamLossResult`. */ + void markLost(HRESULT hr); Microsoft::WRL::ComPtr mediaSource_; Microsoft::WRL::ComPtr sourceReader_; DirectShowWebcamCapture directShowCapture_; std::thread thread_; std::atomic stopRequested_ = false; + std::atomic lost_ = false; std::mutex frameMutex_; std::vector latestFrame_; uint64_t latestFrameSequence_ = 0; @@ -56,6 +88,28 @@ class WebcamCapture { int fps_ = 30; bool mfStarted_ = false; bool usingDirectShow_ = false; - int selectedMatchScore_ = 0; + bool deliversNv12_ = false; + /** Latches the short-buffer warning so it is said once, not 30x a second. */ + bool reportedShortBuffer_ = false; + /** + * Why the capture loop did or did not produce frames. + * + * A camera that delivers nothing used to surface only as + * MF_E_SINK_NO_SAMPLES_PROCESSED from the encoder's Finalize -- the one + * place that cannot say which of the loop's five silent `continue` paths + * swallowed the frames. These are reported once when the loop ends. + */ + uint64_t framesDelivered_ = 0; + uint64_t emptySamples_ = 0; + uint64_t readFailures_ = 0; + uint64_t shortBuffers_ = 0; + uint64_t bufferFailures_ = 0; + bool sawEndOfStream_ = false; + HRESULT lastReadFailure_ = S_OK; + /** Where the loop's wall time goes, in microseconds. */ + uint64_t readSampleUs_ = 0; + uint64_t storeUs_ = 0; std::wstring selectedDeviceName_; + /** The symbolic link of the device this capture opened, for the take's claims. */ + std::wstring selectedIdentity_; }; diff --git a/electron/native/wgc-capture/src/webcam_config.cpp b/electron/native/wgc-capture/src/webcam_config.cpp new file mode 100644 index 0000000000..d437b9d26f --- /dev/null +++ b/electron/native/wgc-capture/src/webcam_config.cpp @@ -0,0 +1,92 @@ +#include "webcam_config.h" + +#include "json_fields.h" + +#include + +namespace { + +// The find* helpers search the whole document, so each array entry is sliced +// out first to keep its fields from being confused with the top-level ones. +std::vector splitTopLevelObjects(const std::string& json, const std::string& arrayKey) { + std::vector objects; + size_t pos = json.find("\"" + arrayKey + "\""); + if (pos == std::string::npos) { + return objects; + } + pos = json.find('[', pos); + if (pos == std::string::npos) { + return objects; + } + ++pos; + + int depth = 0; + bool inString = false; + size_t objectStart = 0; + for (; pos < json.size(); ++pos) { + const char c = json[pos]; + if (inString) { + if (c == '\\') { + ++pos; + } else if (c == '"') { + inString = false; + } + continue; + } + if (c == '"') { + inString = true; + } else if (c == '{') { + if (depth == 0) { + objectStart = pos; + } + ++depth; + } else if (c == '}') { + if (depth > 0 && --depth == 0) { + objects.push_back(json.substr(objectStart, pos - objectStart + 1)); + } + } else if (c == ']' && depth == 0) { + break; + } + } + return objects; +} + +} // namespace + +std::vector parseWebcamConfigs(const std::string& json) { + std::vector configs; + for (const std::string& entry : splitTopLevelObjects(json, "webcams")) { + if (configs.size() >= kMaxWebcams) { + break; + } + WebcamConfig config; + config.outputPath = findString(entry, "camPath"); + if (config.outputPath.empty()) { + continue; + } + config.deviceId = findString(entry, "camDeviceId"); + config.deviceName = findString(entry, "camDeviceName"); + config.directShowClsid = findString(entry, "camClsid"); + config.width = findInt(entry, "camWidth", 0); + config.height = findInt(entry, "camHeight", 0); + config.fps = findInt(entry, "camFps", 0); + configs.push_back(std::move(config)); + } + if (!configs.empty() || !findBool(json, "webcamEnabled", false)) { + return configs; + } + + WebcamConfig legacy; + legacy.outputPath = findString(json, "webcamPath"); + if (legacy.outputPath.empty()) { + return configs; + } + legacy.deviceId = findString(json, "webcamDeviceId"); + legacy.deviceName = findString(json, "webcamDeviceName"); + legacy.directShowClsid = findString(json, "webcamDirectShowClsid"); + legacy.width = findInt(json, "webcamWidth", 0); + legacy.height = findInt(json, "webcamHeight", 0); + legacy.fps = findInt(json, "webcamFps", 0); + configs.push_back(std::move(legacy)); + return configs; +} diff --git a/electron/native/wgc-capture/src/webcam_config.h b/electron/native/wgc-capture/src/webcam_config.h new file mode 100644 index 0000000000..b0c0998eaa --- /dev/null +++ b/electron/native/wgc-capture/src/webcam_config.h @@ -0,0 +1,17 @@ +#pragma once + +#include +#include +#include + +struct WebcamConfig { + std::string deviceId, deviceName, directShowClsid, outputPath; + int width = 0, height = 0, fps = 0; +}; + +constexpr size_t kMaxWebcams = 4; + +// The cameras to record, in order. Reads the `webcams` list when present and +// non-empty; otherwise the legacy single-camera fields (webcamEnabled, webcamDeviceId, +// webcamDeviceName, webcamDirectShowClsid, webcamWidth/Height/Fps, webcamPath). +std::vector parseWebcamConfigs(const std::string& json); diff --git a/electron/native/wgc-capture/src/webcam_config_test.cpp b/electron/native/wgc-capture/src/webcam_config_test.cpp new file mode 100644 index 0000000000..1696698bfe --- /dev/null +++ b/electron/native/wgc-capture/src/webcam_config_test.cpp @@ -0,0 +1,63 @@ +#include "webcam_config.h" + +#include +#include + +namespace { +int failures = 0; +void expect(bool ok, const char* label) { + if (!ok) { + std::printf("FAIL %s\n", label); + ++failures; + } +} +} // namespace + +int main() { + const std::string legacy = + R"({"fps":60,"width":2560,"height":1440,"webcamEnabled":true,"webcamDeviceId":"id1",)" + R"("webcamDeviceName":"Cam One","webcamDirectShowClsid":"{A}","webcamWidth":1920,)" + R"("webcamHeight":1080,"webcamFps":30,"outputs":{"screenPath":"s.mp4","webcamPath":"w.mp4"}})"; + auto l = parseWebcamConfigs(legacy); + expect(l.size() == 1, "legacy: one camera"); + expect(l.size() == 1 && l[0].deviceName == "Cam One" && l[0].outputPath == "w.mp4" && + l[0].width == 1920 && l[0].fps == 30, + "legacy: fields"); + + expect(parseWebcamConfigs(R"({"webcamEnabled":false,"webcamPath":"w.mp4"})").empty(), + "legacy disabled: none"); + + const std::string list = + R"({"fps":60,"width":2560,"webcamEnabled":true,"webcamDeviceName":"Cam One","webcamPath":"w.mp4",)" + R"("webcams":[{"camDeviceId":"id1","camDeviceName":"Cam One","camClsid":"{A}","camWidth":1920,)" + R"("camHeight":1080,"camFps":30,"camPath":"w.mp4"},)" + R"({"camDeviceId":"id2","camDeviceName":"Desk \"Cam\"","camClsid":"","camWidth":1280,)" + R"("camHeight":720,"camFps":30,"camPath":"w-2.mp4"}]})"; + auto m = parseWebcamConfigs(list); + expect(m.size() == 2, "list: two cameras"); + expect(m.size() == 2 && m[1].deviceName == "Desk \"Cam\"" && m[1].outputPath == "w-2.mp4" && + m[1].width == 1280 && m[1].height == 720, + "list: second camera fields, escaped quote"); + expect(m.size() == 2 && m[0].fps == 30, "list: entry fps not the top-level fps"); + + expect(parseWebcamConfigs(R"({"webcamEnabled":true,"webcamPath":"w.mp4","webcams":[]})").size() == 1, + "empty list falls back to legacy"); + + std::string five = R"({"webcams":[)"; + for (int i = 0; i < 5; ++i) { + five += (i ? "," : ""); + five += R"({"camDeviceName":"C","camPath":"p)" + std::to_string(i) + R"(.mp4"})"; + } + five += "]}"; + expect(parseWebcamConfigs(five).size() == 4, "capped at four"); + + expect(parseWebcamConfigs(R"({"webcams":[{"camDeviceName":"No path"}]})").empty(), + "entry without camPath is skipped"); + + if (failures == 0) { + std::printf("webcam_config_test: all assertions passed\n"); + return 0; + } + std::printf("webcam_config_test: %d assertion(s) failed\n", failures); + return 1; +} diff --git a/electron/native/wgc-capture/src/webcam_format.cpp b/electron/native/wgc-capture/src/webcam_format.cpp new file mode 100644 index 0000000000..32e2020ede --- /dev/null +++ b/electron/native/wgc-capture/src/webcam_format.cpp @@ -0,0 +1,82 @@ +#include "webcam_format.h" + +#include + +namespace { + +bool isUsable(const WebcamFormat& format) { + return format.width > 0 && format.height > 0; +} + +bool fitsTarget(const WebcamFormat& format, int targetWidth, int targetHeight) { + return format.width <= targetWidth && format.height <= targetHeight; +} + +long long pixels(const WebcamFormat& format) { + return static_cast(format.width) * static_cast(format.height); +} + +// True when `candidate` is a better frame rate than `best` for a target of +// `targetFps`: the lowest rate that still reaches the target, or -- when no +// mode reaches it -- the highest one on offer. +bool betterFps(int candidate, int best, int targetFps) { + const bool candidateReaches = candidate >= targetFps; + const bool bestReaches = best >= targetFps; + if (candidateReaches != bestReaches) { + return candidateReaches; + } + return candidateReaches ? candidate < best : candidate > best; +} + +} // namespace + +WebcamFormat chooseWebcamFormat( + const std::vector& available, + int targetWidth, + int targetHeight, + int targetFps) { + const WebcamFormat target{targetWidth, targetHeight, targetFps, false}; + + // Compressed modes are a last resort, so they are only considered when the + // camera offers nothing else at all. + const bool anyUncompressed = std::any_of(available.begin(), available.end(), [](const WebcamFormat& format) { + return isUsable(format) && !format.compressed; + }); + + const auto eligible = [&](const WebcamFormat& format) { + return isUsable(format) && !(anyUncompressed && format.compressed); + }; + + const bool anyFits = std::any_of(available.begin(), available.end(), [&](const WebcamFormat& format) { + return eligible(format) && fitsTarget(format, targetWidth, targetHeight); + }); + + const WebcamFormat* best = nullptr; + for (const WebcamFormat& format : available) { + if (!eligible(format)) { + continue; + } + // Once anything fits, oversized modes are out of the running entirely. + // With nothing fitting we rank the oversized modes instead, smallest + // first, so the scaler has the least work to do. + if (anyFits != fitsTarget(format, targetWidth, targetHeight)) { + continue; + } + if (best == nullptr) { + best = &format; + continue; + } + if (pixels(format) != pixels(*best)) { + const bool wins = anyFits ? pixels(format) > pixels(*best) : pixels(format) < pixels(*best); + if (wins) { + best = &format; + } + continue; + } + if (betterFps(format.fps, best->fps, targetFps)) { + best = &format; + } + } + + return best ? *best : target; +} diff --git a/electron/native/wgc-capture/src/webcam_format.h b/electron/native/wgc-capture/src/webcam_format.h new file mode 100644 index 0000000000..7cb329af75 --- /dev/null +++ b/electron/native/wgc-capture/src/webcam_format.h @@ -0,0 +1,44 @@ +#pragma once + +#include + +// One capture format a camera advertises. +struct WebcamFormat { + int width = 0; + int height = 0; + int fps = 0; + /** + * True for MJPEG/H264 modes, which reach higher resolutions than a camera's + * uncompressed ones but need a decoder MFT in front of the RGB32 conversion. + */ + bool compressed = false; +}; + +// Picks the format to drive the camera at, given everything it advertises. +// +// Both capture backends used to leave this to the driver: Media Foundation was +// handed a media type with no MF_MT_FRAME_SIZE, and the DirectShow graph was +// rendered without touching IAMStreamConfig. In both cases the device answers +// with its *default* type, and for a UVC camera that is the first format it +// enumerates -- 640x480 even on a BRIO that offers 3840x2160. The recording was +// then upscaled into the overlay, which is the pixelation users reported. +// +// Rules, in order: +// * uncompressed modes beat compressed ones outright, whatever their size. +// Pinning a camera's MJPEG mode and asking the source reader for RGB32 +// leaves it delivering no samples at all on some hosts -- a recording with +// no camera in it, which is far worse than a smaller one that works; +// * only formats that fit inside the target in BOTH dimensions are eligible, +// so an ultrawide mode cannot sneak in on pixel count alone; +// * among those, the most pixels wins; +// * ties go to the lowest frame rate that still reaches the target, else the +// highest available -- a 1080p60 mode is no help when we encode at 30; +// * if nothing fits (a camera that only does 4K), the smallest mode wins, so +// we scale down rather than push 1 GB/s of RGB32 through the encoder; +// * an empty list returns the target unchanged, letting the caller fall back +// to whatever the driver would have picked on its own. +WebcamFormat chooseWebcamFormat( + const std::vector& available, + int targetWidth, + int targetHeight, + int targetFps); diff --git a/electron/native/wgc-capture/src/webcam_format_test.cpp b/electron/native/wgc-capture/src/webcam_format_test.cpp new file mode 100644 index 0000000000..9e40f5cbfb --- /dev/null +++ b/electron/native/wgc-capture/src/webcam_format_test.cpp @@ -0,0 +1,131 @@ +#include "webcam_format.h" + +#include +#include + +namespace { + +int failures = 0; + +void expectFormat(const std::string& label, WebcamFormat actual, int width, int height, int fps) { + if (actual.width == width && actual.height == height && actual.fps == fps) { + return; + } + std::printf( + "FAIL %s: expected %dx%d@%d, got %dx%d@%d\n", + label.c_str(), + width, + height, + fps, + actual.width, + actual.height, + actual.fps); + ++failures; +} + +// Every mode a Logitech BRIO advertises that this helper could drive, as read +// off the device with `ffmpeg -f dshow -list_options true`. +const std::vector kBrio = { + {640, 480, 30, false}, {640, 360, 30, false}, {1280, 720, 30, false}, + {1920, 1080, 30, false}, {1280, 720, 90, true}, {1920, 1080, 60, true}, + {2560, 1440, 30, true}, {3840, 2160, 30, true}, {160, 120, 30, false}, + {340, 340, 30, false}, {1600, 896, 30, false}, {1024, 576, 30, false}, +}; + +} // namespace + +int main() { + // The regression this file exists for: a BRIO asked for 1080p must not come + // back with the 640x480 default it enumerates first. + expectFormat("brio targets 1080p", chooseWebcamFormat(kBrio, 1920, 1080, 30), 1920, 1080, 30); + + // Asked for more than the camera can do uncompressed, it must stop at the + // best uncompressed mode rather than pin an MJPEG one. Pinning MJPEG and + // asking the source reader for RGB32 produced a reader that delivered no + // samples at all, so the take had no camera in it. + expectFormat( + "4K request settles for the best uncompressed mode", + chooseWebcamFormat(kBrio, 3840, 2160, 30), + 1920, + 1080, + 30); + if (chooseWebcamFormat(kBrio, 3840, 2160, 30).compressed) { + std::printf("FAIL 4K request must not pin a compressed mode\n"); + ++failures; + } + + // A camera with nothing but compressed modes still has to be usable. + expectFormat( + "compressed-only camera is still driven", + chooseWebcamFormat({{1920, 1080, 30, true}, {1280, 720, 30, true}}, 1920, 1080, 30), + 1920, + 1080, + 30); + + // Uncompressed wins even when it is much smaller than a compressed option. + expectFormat( + "uncompressed beats a larger compressed mode", + chooseWebcamFormat({{3840, 2160, 30, true}, {640, 480, 30, false}}, 3840, 2160, 30), + 640, + 480, + 30); + + // 1080p60 exists too; at a 30 fps encode the extra frames only cost bandwidth. + expectFormat( + "prefers the frame rate we encode at", + chooseWebcamFormat({{1920, 1080, 60}, {1920, 1080, 30}}, 1920, 1080, 30), + 1920, + 1080, + 30); + + // Nothing reaches 30, so take the best the camera can do rather than fail. + expectFormat( + "falls back to the highest rate offered", + chooseWebcamFormat({{1920, 1080, 15}, {1920, 1080, 24}}, 1920, 1080, 30), + 1920, + 1080, + 24); + + // A 720p-only camera keeps working, just smaller. + expectFormat( + "720p camera stays 720p", + chooseWebcamFormat({{1280, 720, 30}, {640, 480, 30}}, 1920, 1080, 30), + 1280, + 720, + 30); + + // 2560x800 has fewer pixels than 1920x1080 but is far wider; picking it + // would letterbox the overlay and cost a needless downscale. + expectFormat( + "rejects an oversized dimension even when pixel count fits", + chooseWebcamFormat({{2560, 800, 30}, {1280, 720, 30}}, 1920, 1080, 30), + 1280, + 720, + 30); + + // Only modes above the target: scale down from the cheapest of them. + expectFormat( + "oversized-only camera takes the smallest mode", + chooseWebcamFormat({{3840, 2160, 30}, {2560, 1440, 30}}, 1920, 1080, 30), + 2560, + 1440, + 30); + + // Nothing to choose from: hand the target back and let the driver decide. + expectFormat("empty list returns the target", chooseWebcamFormat({}, 1920, 1080, 30), 1920, 1080, 30); + + // Drivers do report junk; it must not win. + expectFormat( + "ignores degenerate entries", + chooseWebcamFormat({{0, 0, 30}, {-1920, 1080, 30}, {1280, 720, 30}}, 1920, 1080, 30), + 1280, + 720, + 30); + + if (failures == 0) { + std::printf("webcam_format_test: all assertions passed\n"); + return 0; + } + std::printf("webcam_format_test: %d assertion(s) failed\n", failures); + return 1; +} diff --git a/electron/native/wgc-capture/src/webcam_loss.cpp b/electron/native/wgc-capture/src/webcam_loss.cpp new file mode 100644 index 0000000000..7548f84c2b --- /dev/null +++ b/electron/native/wgc-capture/src/webcam_loss.cpp @@ -0,0 +1,28 @@ +#include "webcam_loss.h" + +#include + +#include + +bool isWebcamLossResult(HRESULT hr, bool endOfStream, int64_t consecutiveFailureMs) { + if (SUCCEEDED(hr)) { + return endOfStream; + } + if (hr == MF_E_VIDEO_RECORDING_DEVICE_INVALIDATED || hr == MF_E_HW_MFT_FAILED_START_STREAMING) { + return true; + } + return consecutiveFailureMs >= kWebcamLossFailureRunMs; +} + +void reportWebcamLost(const std::wstring& deviceName, HRESULT hr) { + std::string name; + const int size = WideCharToMultiByte( + CP_UTF8, 0, deviceName.data(), static_cast(deviceName.size()), nullptr, 0, nullptr, nullptr); + if (size > 0) { + name.resize(static_cast(size)); + WideCharToMultiByte( + CP_UTF8, 0, deviceName.data(), static_cast(deviceName.size()), name.data(), size, nullptr, nullptr); + } + std::cerr << "WARNING: Webcam lost during the take: " << name << " hr=0x" << std::hex + << static_cast(hr) << std::dec << std::endl; +} diff --git a/electron/native/wgc-capture/src/webcam_loss.h b/electron/native/wgc-capture/src/webcam_loss.h new file mode 100644 index 0000000000..64af1f987b --- /dev/null +++ b/electron/native/wgc-capture/src/webcam_loss.h @@ -0,0 +1,34 @@ +#pragma once + +#include + +#include +#include + +/** How long reads may keep failing, back to back, before the camera counts as gone. */ +constexpr int64_t kWebcamLossFailureRunMs = 1000; + +/** + * Does this read result mean the camera has left the take? + * + * Media Foundation reports an unplugged camera as + * MF_E_VIDEO_RECORDING_DEVICE_INVALIDATED on every following ReadSample, and a + * camera whose driver gave up as MF_E_HW_MFT_FAILED_START_STREAMING; both are + * final. End of stream while the take is still running is the same thing said + * differently. Any other failure is forgiven once -- a camera can drop a read -- + * but not for `kWebcamLossFailureRunMs` in a row: a reader that fails that long + * delivers nothing, and treating it as alive is what froze a file on its last + * picture for the rest of the take. + * + * `consecutiveFailureMs` is the time since the first failure of the current + * run; a successful read ends the run. Ignored when `hr` succeeded. + */ +bool isWebcamLossResult(HRESULT hr, bool endOfStream, int64_t consecutiveFailureMs); + +/** + * The one line both capture backends print when a camera leaves the take. + * + * `hr` is the read result that decided it, or the DirectShow event code when + * the graph reported the loss. + */ +void reportWebcamLost(const std::wstring& deviceName, HRESULT hr); diff --git a/electron/native/wgc-capture/src/webcam_loss_test.cpp b/electron/native/wgc-capture/src/webcam_loss_test.cpp new file mode 100644 index 0000000000..2a8462f8d9 --- /dev/null +++ b/electron/native/wgc-capture/src/webcam_loss_test.cpp @@ -0,0 +1,41 @@ +#include "webcam_loss.h" + +#include + +#include + +namespace { +int failures = 0; +void expect(bool ok, const char* label) { + if (!ok) { + std::printf("FAIL %s\n", label); + ++failures; + } +} +} // namespace + +int main() { + // The code the on-site test logged after the C920 was unplugged. + expect(isWebcamLossResult(MF_E_VIDEO_RECORDING_DEVICE_INVALIDATED, false, 0), + "device invalidated: lost on the first read"); + expect(isWebcamLossResult(MF_E_HW_MFT_FAILED_START_STREAMING, false, 0), + "hardware failed to start streaming: lost"); + expect(isWebcamLossResult(S_OK, true, 0), "end of stream during the take: lost"); + + expect(!isWebcamLossResult(E_FAIL, false, 0), "one other failure: not lost"); + expect(!isWebcamLossResult(E_FAIL, false, kWebcamLossFailureRunMs - 1), + "other failures for just under a second: not lost"); + expect(isWebcamLossResult(E_FAIL, false, kWebcamLossFailureRunMs), + "other failures for a second: lost"); + expect(isWebcamLossResult(MF_E_NOTACCEPTING, false, 5000), "any failure code, long run: lost"); + + expect(!isWebcamLossResult(S_OK, false, 0), "S_OK: not lost"); + expect(!isWebcamLossResult(S_OK, false, 5000), "S_OK ends a run of failures: not lost"); + + if (failures == 0) { + std::printf("webcam_loss_test: all assertions passed\n"); + return 0; + } + std::printf("webcam_loss_test: %d failure(s)\n", failures); + return 1; +} diff --git a/electron/native/wgc-capture/src/webcam_snapshot_test.cpp b/electron/native/wgc-capture/src/webcam_snapshot_test.cpp new file mode 100644 index 0000000000..0dd555cb40 --- /dev/null +++ b/electron/native/wgc-capture/src/webcam_snapshot_test.cpp @@ -0,0 +1,46 @@ +#include "dshow_webcam_capture.h" + +#include + +namespace { + +int failures = 0; + +void expect(const char* label, bool condition) { + if (condition) { + return; + } + std::printf("FAIL %s\n", label); + ++failures; +} + +} // namespace + +int main() { + const std::vector frame(4 * 2 * 2, 0x80); + + // A new sequence is copied, with the size and sequence it was stored with. + WebcamFrameSnapshot fresh; + expect("new sequence is copied", snapshotWebcamFrame(frame, 2, 2, 7, 6, fresh)); + expect("new sequence carries its data", fresh.data == frame); + expect("new sequence carries its size", fresh.width == 2 && fresh.height == 2); + expect("new sequence carries its number", fresh.sequence == 7); + + // The regression this file exists for: the writer polls at the screen's + // rate, twice the camera's, and a frame it already holds must cost nothing. + WebcamFrameSnapshot unchanged; + expect("unchanged sequence reports nothing new", !snapshotWebcamFrame(frame, 2, 2, 7, 7, unchanged)); + expect("unchanged sequence copies nothing", unchanged.data.empty() && unchanged.sequence == 0); + + // No frame stored yet: nothing to hand out, whatever the caller last saw. + WebcamFrameSnapshot empty; + expect("empty frame is not copied", !snapshotWebcamFrame({}, 2, 2, 1, 0, empty)); + expect("unknown size is not copied", !snapshotWebcamFrame(frame, 0, 2, 1, 0, empty)); + + if (failures > 0) { + std::printf("%d webcam snapshot check(s) failed\n", failures); + return 1; + } + std::printf("webcam snapshot checks passed\n"); + return 0; +} diff --git a/electron/native/wgc-capture/src/wgc_session.cpp b/electron/native/wgc-capture/src/wgc_session.cpp index b75d0819ef..726d7effd1 100644 --- a/electron/native/wgc-capture/src/wgc_session.cpp +++ b/electron/native/wgc-capture/src/wgc_session.cpp @@ -214,6 +214,27 @@ bool WgcSession::applySessionOptions(bool captureCursor) { // IsBorderRequired is Windows 11-only. Ignore it on older builds. } + // Best effort, never a reason to fail the recording: a menu missing from the + // video is today's behavior, a recording that does not start is not. + if (includeSecondaryWindows_ && !secondaryWindowsReported_) { + bool applied = false; +#if defined(NTDDI_WIN11_GE) + // IGraphicsCaptureSession6 (Windows 11 24H2, SDK 26100). try_as returns + // null on an older runtime, so it degrades to the window alone. + try { + if (auto session6 = session_.try_as()) { + session6.IncludeSecondaryWindows(true); + applied = session6.IncludeSecondaryWindows(); + } + } catch (...) { + // Left at applied=false; the event below says so. + } +#endif + secondaryWindowsReported_ = true; + std::cout << "{\"event\":\"secondary-windows\",\"applied\":" << (applied ? "true" : "false") + << "}" << std::endl; + } + return true; } @@ -240,8 +261,9 @@ bool WgcSession::initialize(HMONITOR monitor, int fps, bool captureCursor) { return true; } -bool WgcSession::initialize(HWND window, int fps, bool captureCursor) { +bool WgcSession::initialize(HWND window, int fps, bool captureCursor, bool includeSecondaryWindows) { fps_ = fps > 0 ? fps : 60; + includeSecondaryWindows_ = includeSecondaryWindows; if (!createD3DDevice()) { return false; } diff --git a/electron/native/wgc-capture/src/wgc_session.h b/electron/native/wgc-capture/src/wgc_session.h index 35eb25a547..4dbb33e43c 100644 --- a/electron/native/wgc-capture/src/wgc_session.h +++ b/electron/native/wgc-capture/src/wgc_session.h @@ -53,7 +53,11 @@ class WgcSession { WgcSession& operator=(const WgcSession&) = delete; bool initialize(HMONITOR monitor, int fps, bool captureCursor); - bool initialize(HWND window, int fps, bool captureCursor); + // includeSecondaryWindows asks DWM to draw the popups and tool windows that + // overlap the captured window (context menus, dropdowns) into the same + // texture. Windows 11 24H2+ only; older runtimes silently keep capturing the + // window alone, and a `secondary-windows` event reports which one happened. + bool initialize(HWND window, int fps, bool captureCursor, bool includeSecondaryWindows = false); bool start(); // Returns the most recently arrived frame's texture and timestamp, or // false if none is available since the last call. The returned pointer @@ -113,5 +117,7 @@ class WgcSession { int height_ = 0; int fps_ = 60; bool captureCursor_ = false; + bool includeSecondaryWindows_ = false; + bool secondaryWindowsReported_ = false; bool started_ = false; }; diff --git a/electron/native/whisper-stt/CMakeLists.txt b/electron/native/whisper-stt/CMakeLists.txt index b95baf7d18..a22d02eed5 100644 --- a/electron/native/whisper-stt/CMakeLists.txt +++ b/electron/native/whisper-stt/CMakeLists.txt @@ -144,9 +144,45 @@ endif() FetchContent_MakeAvailable(whisper httplib json) -add_executable(whisper-stt-server src/main.cpp) +# Character-level DTW (issue #948): whisper.cpp's word-timing pass, +# whisper_exp_compute_token_level_timestamps_dtw, and the median filter only it +# used, are replaced by whisper-patches/char-dtw.cpp. The whisper target +# compiles that copy of whisper.cpp from the build tree, so the fetched source +# is never modified: a shared FETCHCONTENT_SOURCE_DIR_WHISPER and the Nix build +# get the same result. Bumping WHISPER_REF fails here, loudly, if the code moved. +set(OSC_DTW_FRAGMENT "${CMAKE_CURRENT_SOURCE_DIR}/whisper-patches/char-dtw.cpp") +set(OSC_WHISPER_CPP "${whisper_SOURCE_DIR}/src/whisper.cpp") +set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${OSC_DTW_FRAGMENT}" "${OSC_WHISPER_CPP}") +file(READ "${OSC_WHISPER_CPP}" osc_src) +string(REPLACE "\r\n" "\n" osc_src "${osc_src}") +string(FIND "${osc_src}" "\nstruct median_filter_user_data {" osc_begin) +string(FIND "${osc_src}" "\nvoid whisper_log_set(" osc_end) +if(osc_begin GREATER -1 AND osc_end GREATER osc_begin) + math(EXPR osc_len "${osc_end} - ${osc_begin}") + string(SUBSTRING "${osc_src}" ${osc_begin} ${osc_len} osc_old) + string(FIND "${osc_old}" "static void whisper_exp_compute_token_level_timestamps_dtw(" osc_fn) +endif() +if(osc_begin EQUAL -1 OR NOT osc_end GREATER osc_begin OR osc_fn EQUAL -1) + message(FATAL_ERROR "char-dtw: the DTW pass was not found where expected in ${OSC_WHISPER_CPP}") +endif() +string(SUBSTRING "${osc_src}" 0 ${osc_begin} osc_head) +string(SUBSTRING "${osc_src}" ${osc_end} -1 osc_tail) +file(READ "${OSC_DTW_FRAGMENT}" osc_fragment) +file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/whisper-patched/whisper.cpp.tmp" + "${osc_head}\n${osc_fragment}${osc_tail}") +configure_file("${CMAKE_CURRENT_BINARY_DIR}/whisper-patched/whisper.cpp.tmp" + "${CMAKE_CURRENT_BINARY_DIR}/whisper-patched/whisper.cpp" COPYONLY) +get_target_property(osc_whisper_srcs whisper SOURCES) +list(TRANSFORM osc_whisper_srcs REPLACE "^whisper\\.cpp$" "${CMAKE_CURRENT_BINARY_DIR}/whisper-patched/whisper.cpp") +if(NOT "${CMAKE_CURRENT_BINARY_DIR}/whisper-patched/whisper.cpp" IN_LIST osc_whisper_srcs) + message(FATAL_ERROR "char-dtw: the whisper target does not list whisper.cpp as expected (${osc_whisper_srcs})") +endif() +set_property(TARGET whisper PROPERTY SOURCES ${osc_whisper_srcs}) + +add_executable(whisper-stt-server src/main.cpp src/ctc_aligner.cpp) target_link_libraries(whisper-stt-server PRIVATE whisper + ggml httplib::httplib nlohmann_json::nlohmann_json ) diff --git a/electron/native/whisper-stt/src/ctc_aligner.cpp b/electron/native/whisper-stt/src/ctc_aligner.cpp new file mode 100644 index 0000000000..18d5145965 --- /dev/null +++ b/electron/native/whisper-stt/src/ctc_aligner.cpp @@ -0,0 +1,433 @@ +// wav2vec2-for-CTC forward pass on ggml (HuggingFace Wav2Vec2ForCTC, both the +// "group norm, post-LN" base layout and the "layer norm, pre-LN" large one). +// Weights come from scripts/convert-wav2vec2-gguf.mjs. +// +// Layout: activations are [channels, time] (ne0 = channels) for the linear +// layers, and transposed to [time, channels] only where a convolution or a +// per-channel norm needs time contiguous. + +#include "ctc_aligner.h" + +#include +#include +#include +#include + +#include +#include +#include + +namespace { + +constexpr int kGraphSize = 8192; +// Longest input one graph sees; longer ones run in windows. Attention is +// quadratic in time and the first convolutions are wide, so this bounds memory +// (about 265 MB of compute buffers at 20 s, for either model: the first +// convolutions dominate). +constexpr int kWindowFrames = 1000; // 20 s +constexpr int kContextFrames = 50; // 1 s of context kept on each side of a window + +struct Layer { + ggml_tensor *q_w, *q_b, *k_w, *k_b, *v_w, *v_b, *o_w, *o_b; + ggml_tensor *ln1_w, *ln1_b, *ff1_w, *ff1_b, *ff2_w, *ff2_b, *ln2_w, *ln2_b; +}; + +} // namespace + +struct CtcModel { + CtcModelInfo info; + int hidden = 0, n_heads = 0, pos_k = 0, pos_groups = 0; + float eps = 1e-5f; + bool group_norm = false, stable = false; + std::vector conv_k, conv_s; + + std::vector conv_w, conv_b, conv_ln_w, conv_ln_b; + ggml_tensor *fp_ln_w = nullptr, *fp_ln_b = nullptr, *fp_w = nullptr, *fp_b = nullptr; + ggml_tensor *pos_w = nullptr, *pos_b = nullptr, *enc_ln_w = nullptr, *enc_ln_b = nullptr; + ggml_tensor *head_w = nullptr, *head_b = nullptr; + std::vector layers; + + ggml_context* wctx = nullptr; + gguf_context* gctx = nullptr; + ggml_backend_buffer_t wbuf = nullptr; + ggml_backend_t gpu = nullptr, cpu = nullptr; + ggml_backend_sched_t sched = nullptr; + std::string device = "CPU"; + + ~CtcModel() { + if (sched) ggml_backend_sched_free(sched); + if (wbuf) ggml_backend_buffer_free(wbuf); + if (gpu) ggml_backend_free(gpu); + if (cpu) ggml_backend_free(cpu); + if (wctx) ggml_free(wctx); + if (gctx) gguf_free(gctx); + } +}; + +void CtcModelDeleter::operator()(CtcModel* m) const { delete m; } + +const CtcModelInfo& ctc_info(const CtcModel& model) { return model.info; } +std::string ctc_device(const CtcModel& model) { return model.device; } + +namespace { + +int64_t key(gguf_context* g, const char* k, std::string& err) { + const int64_t id = gguf_find_key(g, k); + if (id < 0 && err.empty()) err = std::string("missing metadata ") + k; + return id; +} + +std::vector str_array(gguf_context* g, int64_t id) { + std::vector out; + if (id < 0) return out; + const size_t n = gguf_get_arr_n(g, id); + for (size_t i = 0; i < n; ++i) out.emplace_back(gguf_get_arr_str(g, id, i)); + return out; +} + +std::vector u32_array(gguf_context* g, int64_t id) { + std::vector out; + if (id < 0) return out; + const auto* p = static_cast(gguf_get_arr_data(g, id)); + out.assign(p, p + gguf_get_arr_n(g, id)); + return out; +} + +ggml_tensor* layer_norm(ggml_context* ctx, ggml_tensor* x, ggml_tensor* w, ggml_tensor* b, float eps) { + return ggml_add(ctx, ggml_mul(ctx, ggml_norm(ctx, x, eps), w), b); +} + +ggml_tensor* linear(ggml_context* ctx, ggml_tensor* x, ggml_tensor* w, ggml_tensor* b) { + return ggml_add(ctx, ggml_mul_mat(ctx, w, x), b); +} + +// 1-D convolution of `x` [time, in] with `w` [k, in, out] -> [out, time']. The +// columns are F16, as in whisper.cpp's encoder: half the memory traffic of F32, +// which made the CPU path 1.7x slower, and `w` must be F16 to match. +ggml_tensor* conv1d(ggml_context* ctx, ggml_tensor* w, ggml_tensor* x, int stride, int pad) { + ggml_tensor* cols = ggml_im2col(ctx, w, x, stride, 0, pad, 0, 1, 0, false, GGML_TYPE_F16); + // cols: [k * in, time'] + cols = ggml_reshape_2d(ctx, cols, cols->ne[0], cols->ne[1]); + ggml_tensor* w2 = ggml_reshape_2d(ctx, w, w->ne[0] * w->ne[1], w->ne[2]); + return ggml_mul_mat(ctx, w2, cols); +} + +ggml_tensor* build(CtcModel& m, ggml_context* ctx, ggml_tensor* wave) { + // ---- feature encoder ---- + ggml_tensor* x = wave; // [time, 1] + ggml_tensor* y = nullptr; + for (size_t i = 0; i < m.conv_w.size(); ++i) { + y = conv1d(ctx, m.conv_w[i], x, m.conv_s[i], 0); // [512, t] + if (m.conv_b[i]) y = ggml_add(ctx, y, m.conv_b[i]); + if (m.group_norm) { + // Time-major for the next convolution. Only the first layer is normed: + // GroupNorm with one group per channel, i.e. each channel over time. + ggml_tensor* t = ggml_cont(ctx, ggml_transpose(ctx, y)); // [t, 512] + if (i == 0) { + t = ggml_norm(ctx, t, m.eps); + t = ggml_add(ctx, ggml_mul(ctx, t, ggml_reshape_2d(ctx, m.conv_ln_w[0], 1, t->ne[1])), + ggml_reshape_2d(ctx, m.conv_ln_b[0], 1, t->ne[1])); + } + x = ggml_gelu_erf(ctx, t); + } else { + y = layer_norm(ctx, y, m.conv_ln_w[i], m.conv_ln_b[i], m.eps); + x = ggml_cont(ctx, ggml_transpose(ctx, ggml_gelu_erf(ctx, y))); // [t, 512] + } + } + // x: [t, 512] -> [512, t] + ggml_tensor* h = ggml_cont(ctx, ggml_transpose(ctx, x)); + const int64_t T = h->ne[1]; + + // ---- feature projection ---- + h = layer_norm(ctx, h, m.fp_ln_w, m.fp_ln_b, m.eps); + h = linear(ctx, h, m.fp_w, m.fp_b); // [H, T] + + // ---- positional convolution (grouped, same padding, drop the last frame) ---- + { + ggml_tensor* ht = ggml_cont(ctx, ggml_transpose(ctx, h)); // [T, H] + const int64_t cg = m.hidden / m.pos_groups; + ggml_tensor* pos = nullptr; + for (int g = 0; g < m.pos_groups; ++g) { + ggml_tensor* xin = ggml_view_2d(ctx, ht, T, cg, ht->nb[1], g * cg * ht->nb[1]); + ggml_tensor* wg = ggml_view_3d(ctx, m.pos_w, m.pos_k, cg, cg, m.pos_w->nb[1], + m.pos_w->nb[2], g * cg * m.pos_w->nb[2]); + ggml_tensor* part = conv1d(ctx, wg, ggml_cont(ctx, xin), 1, m.pos_k / 2); // [cg, T+1] + pos = pos ? ggml_concat(ctx, pos, part, 0) : part; + } + pos = ggml_view_2d(ctx, pos, m.hidden, T, pos->nb[1], 0); + pos = ggml_gelu_erf(ctx, ggml_add(ctx, pos, m.pos_b)); + h = ggml_add(ctx, h, pos); + } + if (!m.stable) h = layer_norm(ctx, h, m.enc_ln_w, m.enc_ln_b, m.eps); + + // ---- transformer ---- + const int64_t dh = m.hidden / m.n_heads; + const float scale = 1.0f / std::sqrt(static_cast(dh)); + for (const Layer& L : m.layers) { + ggml_tensor* res = h; + ggml_tensor* a = m.stable ? layer_norm(ctx, h, L.ln1_w, L.ln1_b, m.eps) : h; + ggml_tensor* q = ggml_reshape_3d(ctx, linear(ctx, a, L.q_w, L.q_b), dh, m.n_heads, T); + ggml_tensor* k = ggml_reshape_3d(ctx, linear(ctx, a, L.k_w, L.k_b), dh, m.n_heads, T); + ggml_tensor* v = ggml_reshape_3d(ctx, linear(ctx, a, L.v_w, L.v_b), dh, m.n_heads, T); + q = ggml_permute(ctx, q, 0, 2, 1, 3); // [dh, T, nh] + k = ggml_permute(ctx, k, 0, 2, 1, 3); // [dh, T, nh] + v = ggml_cont(ctx, ggml_permute(ctx, v, 1, 2, 0, 3)); // [T, dh, nh] + ggml_tensor* kq = ggml_mul_mat(ctx, k, q); // [Tk, Tq, nh] + kq = ggml_soft_max_ext(ctx, kq, nullptr, scale, 0.0f); + ggml_tensor* o = ggml_mul_mat(ctx, v, kq); // [dh, Tq, nh] + o = ggml_cont(ctx, ggml_permute(ctx, o, 0, 2, 1, 3)); // [dh, nh, T] + o = linear(ctx, ggml_reshape_2d(ctx, o, m.hidden, T), L.o_w, L.o_b); + h = ggml_add(ctx, res, o); + if (m.stable) { + ggml_tensor* f = layer_norm(ctx, h, L.ln2_w, L.ln2_b, m.eps); + f = linear(ctx, ggml_gelu_erf(ctx, linear(ctx, f, L.ff1_w, L.ff1_b)), L.ff2_w, L.ff2_b); + h = ggml_add(ctx, h, f); + } else { + h = layer_norm(ctx, h, L.ln1_w, L.ln1_b, m.eps); + ggml_tensor* f = linear(ctx, ggml_gelu_erf(ctx, linear(ctx, h, L.ff1_w, L.ff1_b)), L.ff2_w, L.ff2_b); + h = layer_norm(ctx, ggml_add(ctx, h, f), L.ln2_w, L.ln2_b, m.eps); + } + } + if (m.stable) h = layer_norm(ctx, h, m.enc_ln_w, m.enc_ln_b, m.eps); + return linear(ctx, h, m.head_w, m.head_b); // [V, T] +} + +int frames_for(const CtcModel& m, int64_t n) { + for (size_t i = 0; i < m.conv_k.size(); ++i) { + if (n < m.conv_k[i]) return 0; + n = (n - m.conv_k[i]) / m.conv_s[i] + 1; + } + return static_cast(n); +} + +// Logits for one window of already-normalized samples. +bool run_window(CtcModel& m, const float* pcm, int64_t n, std::vector& logits, int& frames, + std::string& err) { + frames = frames_for(m, n); + if (frames <= 0) return true; + ggml_init_params p = {ggml_tensor_overhead() * kGraphSize + ggml_graph_overhead_custom(kGraphSize, false), + nullptr, true}; + ggml_context* ctx = ggml_init(p); + ggml_tensor* wave = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n, 1); + ggml_set_input(wave); + ggml_tensor* out = build(m, ctx, wave); + ggml_set_output(out); + ggml_cgraph* gf = ggml_new_graph_custom(ctx, kGraphSize, false); + ggml_build_forward_expand(gf, out); + ggml_backend_sched_reset(m.sched); + bool ok = ggml_backend_sched_alloc_graph(m.sched, gf); + if (ok) { + ggml_backend_tensor_set(wave, pcm, 0, n * sizeof(float)); + ok = ggml_backend_sched_graph_compute(m.sched, gf) == GGML_STATUS_SUCCESS; + } + if (ok) { + if (out->ne[1] != frames) { + err = "unexpected frame count"; + ok = false; + } else { + logits.resize(static_cast(out->ne[0]) * frames); + ggml_backend_tensor_get(out, logits.data(), 0, logits.size() * sizeof(float)); + } + } else if (err.empty()) { + err = "ggml graph allocation or compute failed"; + } + ggml_free(ctx); + return ok; +} + +} // namespace + +CtcModelPtr ctc_load(const std::string& path, bool use_gpu, int threads, std::string& err) { + CtcModelPtr m(new CtcModel()); + gguf_init_params gp = {true, &m->wctx}; + m->gctx = gguf_init_from_file(path.c_str(), gp); + if (!m->gctx) { + err = "cannot read " + path; + return nullptr; + } + gguf_context* g = m->gctx; + CtcModelInfo& info = m->info; + info.vocab = str_array(g, key(g, "w2v.vocab", err)); + info.languages = str_array(g, key(g, "w2v.languages", err)); + int64_t id; + if ((id = key(g, "w2v.blank_id", err)) >= 0) info.blank = gguf_get_val_u32(g, id); + if ((id = key(g, "w2v.hidden_size", err)) >= 0) m->hidden = gguf_get_val_u32(g, id); + if ((id = key(g, "w2v.num_attention_heads", err)) >= 0) m->n_heads = gguf_get_val_u32(g, id); + if ((id = key(g, "w2v.layer_norm_eps", err)) >= 0) m->eps = gguf_get_val_f32(g, id); + if ((id = key(g, "w2v.num_conv_pos_embeddings", err)) >= 0) m->pos_k = gguf_get_val_u32(g, id); + if ((id = key(g, "w2v.num_conv_pos_embedding_groups", err)) >= 0) m->pos_groups = gguf_get_val_u32(g, id); + if ((id = key(g, "w2v.feat_extract_norm", err)) >= 0) m->group_norm = std::string(gguf_get_val_str(g, id)) == "group"; + if ((id = key(g, "w2v.stable_layer_norm", err)) >= 0) m->stable = gguf_get_val_bool(g, id); + m->conv_k = u32_array(g, key(g, "w2v.conv_kernel", err)); + m->conv_s = u32_array(g, key(g, "w2v.conv_stride", err)); + int n_layers = 0; + if ((id = key(g, "w2v.num_hidden_layers", err)) >= 0) n_layers = gguf_get_val_u32(g, id); + if (!err.empty()) return nullptr; + info.stride = 1; + for (int s : m->conv_s) info.stride *= s; + info.receptive_field = 1; + for (int i = static_cast(m->conv_k.size()) - 1; i >= 0; --i) + info.receptive_field = (info.receptive_field - 1) * m->conv_s[i] + m->conv_k[i]; + + auto T = [&](const std::string& name, bool required = true) -> ggml_tensor* { + ggml_tensor* t = ggml_get_tensor(m->wctx, name.c_str()); + if (!t && required && err.empty()) err = "missing tensor " + name; + return t; + }; + for (size_t i = 0; i < m->conv_k.size(); ++i) { + const std::string p = "feature_extractor.conv_layers." + std::to_string(i) + "."; + m->conv_w.push_back(T(p + "conv.weight")); + m->conv_b.push_back(T(p + "conv.bias", false)); + const bool has_ln = !m->group_norm || i == 0; + m->conv_ln_w.push_back(has_ln ? T(p + "layer_norm.weight") : nullptr); + m->conv_ln_b.push_back(has_ln ? T(p + "layer_norm.bias") : nullptr); + } + m->fp_ln_w = T("feature_projection.layer_norm.weight"); + m->fp_ln_b = T("feature_projection.layer_norm.bias"); + m->fp_w = T("feature_projection.projection.weight"); + m->fp_b = T("feature_projection.projection.bias"); + m->pos_w = T("encoder.pos_conv_embed.conv.weight"); + m->pos_b = T("encoder.pos_conv_embed.conv.bias"); + m->enc_ln_w = T("encoder.layer_norm.weight"); + m->enc_ln_b = T("encoder.layer_norm.bias"); + m->head_w = T("lm_head.weight"); + m->head_b = T("lm_head.bias"); + for (int l = 0; l < n_layers; ++l) { + const std::string p = "encoder.layers." + std::to_string(l) + "."; + m->layers.push_back({ + T(p + "attention.q_proj.weight"), T(p + "attention.q_proj.bias"), + T(p + "attention.k_proj.weight"), T(p + "attention.k_proj.bias"), + T(p + "attention.v_proj.weight"), T(p + "attention.v_proj.bias"), + T(p + "attention.out_proj.weight"), T(p + "attention.out_proj.bias"), + T(p + "layer_norm.weight"), T(p + "layer_norm.bias"), + T(p + "feed_forward.intermediate_dense.weight"), T(p + "feed_forward.intermediate_dense.bias"), + T(p + "feed_forward.output_dense.weight"), T(p + "feed_forward.output_dense.bias"), + T(p + "final_layer_norm.weight"), T(p + "final_layer_norm.bias"), + }); + } + if (!err.empty()) return nullptr; + // conv1d() multiplies them with F16 columns; anything else aborts inside ggml. + for (const ggml_tensor* w : m->conv_w) { + if (w->type != GGML_TYPE_F16) err = "convolution weights must be F16"; + } + if (m->pos_w->type != GGML_TYPE_F16) err = "convolution weights must be F16"; + if (!err.empty()) return nullptr; + if (static_cast(info.vocab.size()) != m->head_w->ne[1]) { + err = "vocabulary does not match lm_head"; + return nullptr; + } + + // ---- backends ---- + if (use_gpu) { + for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { + ggml_backend_dev_t dev = ggml_backend_dev_get(i); + const auto type = ggml_backend_dev_type(dev); + if (type != GGML_BACKEND_DEVICE_TYPE_GPU && type != GGML_BACKEND_DEVICE_TYPE_IGPU) continue; + m->gpu = ggml_backend_dev_init(dev, nullptr); + if (m->gpu) { + m->device = ggml_backend_dev_name(dev); + break; + } + } + } + m->cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr); + if (!m->cpu) { + err = "no CPU backend"; + return nullptr; + } + { + ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(m->cpu)); + using set_threads_t = void (*)(ggml_backend_t, int); + auto set_threads = reinterpret_cast( + ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads")); + if (set_threads) set_threads(m->cpu, threads); + } + ggml_backend_t weights_on = m->gpu ? m->gpu : m->cpu; + m->wbuf = ggml_backend_alloc_ctx_tensors(m->wctx, weights_on); + if (!m->wbuf) { + err = "cannot allocate the model's weights"; + return nullptr; + } + { + FILE* f = std::fopen(path.c_str(), "rb"); + if (!f) { + err = "cannot open " + path; + return nullptr; + } + std::vector buf; + const size_t data_off = gguf_get_data_offset(g); + for (int64_t i = 0; i < gguf_get_n_tensors(g); ++i) { + ggml_tensor* t = ggml_get_tensor(m->wctx, gguf_get_tensor_name(g, i)); + const size_t nbytes = ggml_nbytes(t); + buf.resize(nbytes); +#ifdef _WIN32 + _fseeki64(f, static_cast(data_off + gguf_get_tensor_offset(g, i)), SEEK_SET); +#else + fseeko(f, static_cast(data_off + gguf_get_tensor_offset(g, i)), SEEK_SET); +#endif + if (std::fread(buf.data(), 1, nbytes, f) != nbytes) { + std::fclose(f); + err = "truncated model file " + path; + return nullptr; + } + ggml_backend_tensor_set(t, buf.data(), 0, nbytes); + } + std::fclose(f); + } + std::vector backends; + if (m->gpu) backends.push_back(m->gpu); + backends.push_back(m->cpu); + m->sched = ggml_backend_sched_new(backends.data(), nullptr, static_cast(backends.size()), + kGraphSize, false, true); + if (!m->sched) { + err = "cannot create the ggml scheduler"; + return nullptr; + } + return m; +} + +bool ctc_emissions(CtcModel& m, const float* pcm, size_t n, std::vector& out, int& frames, + std::string& err) { + const int V = static_cast(m.info.vocab.size()); + const int64_t stride = m.info.stride; + frames = frames_for(m, static_cast(n)); + out.assign(static_cast(std::max(frames, 0)) * V, 0.0f); + if (frames <= 0) return true; + + // Wav2Vec2FeatureExtractor(do_normalize=True): zero mean, unit variance, per input. + double mean = 0.0, var = 0.0; + for (size_t i = 0; i < n; ++i) mean += pcm[i]; + mean /= static_cast(n); + for (size_t i = 0; i < n; ++i) var += (pcm[i] - mean) * (pcm[i] - mean); + var /= static_cast(n); + const double inv = 1.0 / std::sqrt(var + 1e-7); + std::vector x(n); + for (size_t i = 0; i < n; ++i) x[i] = static_cast((pcm[i] - mean) * inv); + + // Windows of kWindowFrames, stepping by kWindowFrames - 2 * kContextFrames; each + // keeps its frames away from the edges it shares with a neighbour. + const int64_t win_samples = kWindowFrames * stride + (m.info.receptive_field - stride); + const int hop = kWindowFrames - 2 * kContextFrames; + std::vector logits; + for (int f0 = 0;; f0 += hop) { + const int64_t s0 = static_cast(f0) * stride; + const int64_t len = std::min(win_samples, static_cast(n) - s0); + int wf = 0; + if (!run_window(m, x.data() + s0, len, logits, wf, err)) return false; + const bool last = s0 + len >= static_cast(n); + const int keep_from = f0 == 0 ? 0 : kContextFrames; + const int keep_to = last ? wf : std::min(wf, kWindowFrames - kContextFrames); + for (int i = keep_from; i < keep_to && f0 + i < frames; ++i) { + const float* row = logits.data() + static_cast(i) * V; + float mx = row[0]; + for (int v = 1; v < V; ++v) mx = std::max(mx, row[v]); + double sum = 0.0; + for (int v = 0; v < V; ++v) sum += std::exp(row[v] - mx); + const float lse = mx + static_cast(std::log(sum)); + float* dst = out.data() + static_cast(f0 + i) * V; + for (int v = 0; v < V; ++v) dst[v] = row[v] - lse; + } + if (last) break; + } + return true; +} diff --git a/electron/native/whisper-stt/src/ctc_aligner.h b/electron/native/whisper-stt/src/ctc_aligner.h new file mode 100644 index 0000000000..80997505f8 --- /dev/null +++ b/electron/native/whisper-stt/src/ctc_aligner.h @@ -0,0 +1,37 @@ +// CTC acoustic model (wav2vec2 fine-tuned for CTC) run on ggml, for the word +// aligner's second pass. The helper only computes the per-frame log-probs; the +// forced alignment of whisper's words over them lives on the Node side +// (electron/stt/ctcAlign.ts). See transcription-and-captions.md § Word-level alignment. +#pragma once + +#include +#include +#include + +struct CtcModel; + +struct CtcModelInfo { + std::vector vocab; // token id -> text; "|" is the word delimiter + std::vector languages; // what the model was fine-tuned on + int blank = 0; + int stride = 320; // samples per output frame (20 ms at 16 kHz) + int receptive_field = 400; // samples the first frame sees +}; + +struct CtcModelDeleter { void operator()(CtcModel* m) const; }; +using CtcModelPtr = std::unique_ptr; + +// Loads a GGUF written by scripts/convert-wav2vec2-gguf.mjs. On the GPU when +// `use_gpu` and a GPU device is registered, else on the CPU. Null + `err` on failure. +CtcModelPtr ctc_load(const std::string& path, bool use_gpu, int threads, std::string& err); + +const CtcModelInfo& ctc_info(const CtcModel& model); + +// Name of the device the weights live on ("Vulkan0", "MTL0", "CPU"). +std::string ctc_device(const CtcModel& model); + +// Log-softmax emissions for 16 kHz mono `pcm`: `frames` rows of vocab-size +// floats. Frame i sees samples [i * stride, i * stride + receptive_field). +// Long inputs run in overlapping windows, so memory stays bounded. +bool ctc_emissions(CtcModel& model, const float* pcm, size_t n, std::vector& out, + int& frames, std::string& err); diff --git a/electron/native/whisper-stt/src/main.cpp b/electron/native/whisper-stt/src/main.cpp index f287eed2ef..77440e54d3 100644 --- a/electron/native/whisper-stt/src/main.cpp +++ b/electron/native/whisper-stt/src/main.cpp @@ -27,7 +27,8 @@ // "start": 0.0, "end": 11.0, // "words": [ { "word": "...", "start": 0.5, "end": 0.9, "probability": 0.9 }, ... ] // }, ... -// ] +// ], +// "speech": [ { "start": 0.4, "end": 2.1 }, ... ] // only when the VAD model loaded // } // // Concurrency: whisper contexts are not thread-safe. /inference is serialized @@ -35,6 +36,7 @@ // this is a belt-and-braces guarantee against a future bug or parallel invoker). #include "whisper.h" +#include "ctc_aligner.h" #include #include @@ -222,12 +224,57 @@ std::string detect_active_backend() { } struct Word { - double start = 0.0; - double end = 0.0; - double prob = 0.0; + double start = 0.0; + double end = 0.0; + // t_dtw of the word's first token: the end of that token, so always inside + // the word. Only used to tell which stretch of speech a word belongs to. + double anchor = 0.0; + double prob = 0.0; std::string text; }; +// One stretch of speech copied from the upload into the buffer whisper decodes. +struct Kept { + int64_t at; // first sample in that buffer + int64_t from; // first sample in the upload + int64_t len; +}; + +// A whisper time (centiseconds on the speech-only buffer) back on the upload's +// clock, in seconds. Inside a kept stretch it moves with the stretch; in the +// silence inserted between two stretches it snaps to the nearer edge, so a +// word can never land in audio that was cut out. No stretches: VAD was off. +double to_original_sec(int64_t cs, const std::vector& kept) { + if (kept.empty()) return cs / 100.0; + const int64_t x = cs * 160; // 16 kHz: 160 samples per centisecond + for (size_t i = 0; i < kept.size(); ++i) { + const Kept& k = kept[i]; + if (x < k.at) { + if (i == 0) return k.from / 16000.0; + const Kept& p = kept[i - 1]; + const bool nearer_prev = x - (p.at + p.len) <= k.at - x; + return (nearer_prev ? p.from + p.len : k.from) / 16000.0; + } + if (x < k.at + k.len) return (k.from + x - k.at) / 16000.0; + } + return (kept.back().from + kept.back().len) / 16000.0; +} + +std::string base64(const void* data, size_t n) { + static const char* abc = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + const auto* p = static_cast(data); + std::string out; + out.reserve((n + 2) / 3 * 4); + for (size_t i = 0; i < n; i += 3) { + const uint32_t v = (p[i] << 16) | (i + 1 < n ? p[i + 1] << 8 : 0) | (i + 2 < n ? p[i + 2] : 0); + out += abc[(v >> 18) & 63]; + out += abc[(v >> 12) & 63]; + out += i + 1 < n ? abc[(v >> 6) & 63] : '='; + out += i + 2 < n ? abc[v & 63] : '='; + } + return out; +} + } // namespace int main(int argc, char** argv) { @@ -312,17 +359,17 @@ int main(int argc, char** argv) { log("model loaded; backend=" + active_backend); // ---- Init VAD context (Silero VAD v6.2.0) ---- + // On the CPU, always. Asked for the GPU on Vulkan, whisper.cpp 1.9.1 puts the + // VAD weights in a Vulkan buffer, then finds no GPU for the VAD's own backend + // and aborts inside ggml (0xC0000409) — so the app relaunched the helper with + // --cpu and every transcription lost its GPU. Silero is 0.9 MB: the CPU is + // not where the time goes. struct whisper_vad_context* vctx = nullptr; if (!vad_model_path.empty()) { struct whisper_vad_context_params vad_ctx_params = whisper_vad_default_context_params(); vad_ctx_params.n_threads = threads; - vad_ctx_params.use_gpu = !force_cpu; + vad_ctx_params.use_gpu = false; vctx = whisper_vad_init_from_file_with_params(vad_model_path.c_str(), vad_ctx_params); - if (!vctx && vad_ctx_params.use_gpu) { - log("GPU VAD initialization failed; retrying with CPU inference"); - vad_ctx_params.use_gpu = false; - vctx = whisper_vad_init_from_file_with_params(vad_model_path.c_str(), vad_ctx_params); - } if (vctx) { log("VAD model loaded; path=" + vad_model_path); } else { @@ -345,13 +392,15 @@ int main(int argc, char** argv) { res.set_content(probe.dump(), "application/json"); }); - // POST /inference — multipart form with `file` (WAV) + `language` + `response_format`. - svr.Post("/inference", [&](const httplib::Request& req, httplib::Response& res) { + // The upload of /inference and /emissions: a 16 kHz mono PCM16 WAV in the + // multipart field `file`. False after answering 400. + const auto read_upload = [](const httplib::Request& req, httplib::Response& res, + std::vector& pcm) -> bool { auto it = req.files.find("file"); if (it == req.files.end()) { res.status = 400; res.set_content(R"({"error":"missing 'file' form field"})", "application/json"); - return; + return false; } const auto& file_entry = it->second; @@ -373,7 +422,6 @@ int main(int argc, char** argv) { out.write(file_entry.content.data(), static_cast(file_entry.content.size())); } - std::vector pcm; int sample_rate = 0, channels = 0; const bool ok = read_wav_pcm16(tmp_wav, pcm, sample_rate, channels); std::error_code ec; @@ -381,15 +429,22 @@ int main(int argc, char** argv) { if (!ok) { res.status = 400; res.set_content(R"({"error":"failed to parse WAV"})", "application/json"); - return; + return false; } if (sample_rate != 16000 || channels != 1) { res.status = 400; res.set_content( R"({"error":"expected 16 kHz mono PCM16 WAV"})", "application/json"); - return; + return false; } + return true; + }; + + // POST /inference — multipart form with `file` (WAV) + `language` + `response_format`. + svr.Post("/inference", [&](const httplib::Request& req, httplib::Response& res) { + std::vector pcm; + if (!read_upload(req, res, pcm)) return; // language param std::string language = "auto"; @@ -407,14 +462,52 @@ int main(int argc, char** argv) { wparams.print_realtime = false; wparams.print_timestamps = false; wparams.n_threads = threads; - if (vctx && !vad_model_path.empty()) { - wparams.vad = true; - wparams.vad_model_path = vad_model_path.c_str(); - wparams.vad_params = whisper_vad_default_params(); + + // ---- Speech only (Silero VAD), cut here rather than by whisper_full ---- + // whisper_full's own `vad` param maps SEGMENT times back onto the upload + // but not token times, so every word (t_dtw included) came back early by + // all the silence removed before it: 13 s into a 25 s clip. Cutting here + // keeps the map. The stretches also go out as `speech`, which the Node + // side anchors phrase edges on. + const auto t0 = std::chrono::steady_clock::now(); // VAD time is part of the run + std::vector speech; + std::vector kept; + nlohmann::json speech_json = nlohmann::json::array(); + if (vctx) { + whisper_vad_segments* vs = whisper_vad_segments_from_samples( + vctx, whisper_vad_default_params(), pcm.data(), static_cast(pcm.size())); + if (!vs) { + res.status = 500; + res.set_content(R"({"error":"VAD failed"})", "application/json"); + return; + } + const int64_t n_pcm = static_cast(pcm.size()); + const auto to_sample = [&](float cs) { + return std::clamp(std::llround(cs * 160.0), 0, n_pcm); + }; + const int n_vs = whisper_vad_segments_n_segments(vs); + for (int i = 0; i < n_vs; ++i) { + const int64_t from = to_sample(whisper_vad_segments_get_segment_t0(vs, i)); + const int64_t end = to_sample(whisper_vad_segments_get_segment_t1(vs, i)); + if (end <= from) continue; + speech_json.push_back({{"start", from / 16000.0}, {"end", end / 16000.0}}); + // As whisper.cpp does: 0.1 s past the detected end so a soft ending + // survives, and 0.1 s of silence between stretches. Unlike it, the + // tail never runs into the next stretch and decodes it twice. + const int64_t next = i + 1 < n_vs + ? to_sample(whisper_vad_segments_get_segment_t0(vs, i + 1)) + : n_pcm; + const int64_t to = std::max(end, std::min(end + 1600, next)); + if (!kept.empty()) speech.insert(speech.end(), 1600, 0.0f); + kept.push_back({static_cast(speech.size()), from, to - from}); + speech.insert(speech.end(), pcm.begin() + from, pcm.begin() + to); + } + whisper_vad_free_segments(vs); } + // VAD on and no speech found: nothing to decode, so nothing to report. + const std::vector& input = vctx ? speech : pcm; - const auto t0 = std::chrono::steady_clock::now(); - const int rc = whisper_full(ctx, wparams, pcm.data(), static_cast(pcm.size())); + const int rc = input.empty() ? 0 : whisper_full(ctx, wparams, input.data(), static_cast(input.size())); const auto t1 = std::chrono::steady_clock::now(); if (rc != 0) { log("whisper_full returned " + std::to_string(rc)); @@ -451,18 +544,28 @@ int main(int argc, char** argv) { }; std::vector segments; - const int n_segments = whisper_full_n_segments(ctx); + // whisper.cpp writes a token's t_dtw when the DTW path enters the decoder + // row that PREDICTS the next token (whisper_exp_compute_token_level_ + // timestamps_dtw, v1.9.1): it marks the END of the token, not its start. + // So a word runs from the t_dtw of the text token before it to the t_dtw + // of its own last token. Taking its first token's t_dtw as the start put + // every word one token late (+175 ms median, tools/stt-eval/word-timing). + // The previous token carries across segments; the request's very first + // word has none and starts on its first token's time (the Node side + // anchors it on the speech onset anyway). + double prev_tok_end = -1.0; + const int n_segments = input.empty() ? 0 : whisper_full_n_segments(ctx); for (int si = 0; si < n_segments; ++si) { Segment seg; - seg.start = whisper_full_get_segment_t0(ctx, si) / 100.0; - seg.end = whisper_full_get_segment_t1(ctx, si) / 100.0; + seg.start = to_original_sec(whisper_full_get_segment_t0(ctx, si), kept); + seg.end = to_original_sec(whisper_full_get_segment_t1(ctx, si), kept); if (const char* t = whisper_full_get_segment_text(ctx, si)) seg.text = t; - struct W { double t_dtw_first; double p_sum; int p_n; std::string text; }; + struct W { double start; double end; double anchor; double p_sum; int p_n; std::string text; }; std::vector word_buf; std::string cur_text; bool in_word = false; - double w_first_t_dtw = 0; + double w_start = 0, w_end = 0, w_anchor = 0; double w_p_sum = 0; int w_p_n = 0; const int n_tokens = whisper_full_n_tokens(ctx, si); @@ -486,33 +589,35 @@ int main(int argc, char** argv) { ++non_special_tokens; const bool starts_word = (!in_word) || (!raw.empty() && raw[0] == ' '); - const double td_dtw = (td.t_dtw >= 0 ? td.t_dtw : 0) / 100.0; + const double td_dtw = to_original_sec(td.t_dtw >= 0 ? td.t_dtw : 0, kept); if (starts_word && in_word) { - word_buf.push_back({ w_first_t_dtw, w_p_sum, w_p_n, cur_text }); + word_buf.push_back({ w_start, w_end, w_anchor, w_p_sum, w_p_n, cur_text }); w_p_sum = 0; w_p_n = 0; cur_text.clear(); } if (starts_word) { in_word = true; - w_first_t_dtw = td_dtw; + w_start = prev_tok_end >= 0 ? prev_tok_end : td_dtw; + w_anchor = td_dtw; cur_text = (!raw.empty() && raw[0] == ' ') ? raw.substr(1) : raw; } else { cur_text += raw; } + w_end = td_dtw; + prev_tok_end = td_dtw; w_p_sum += td.p; w_p_n += 1; } if (in_word) { - word_buf.push_back({ w_first_t_dtw, w_p_sum, w_p_n, cur_text }); + word_buf.push_back({ w_start, w_end, w_anchor, w_p_sum, w_p_n, cur_text }); } - for (size_t wi = 0; wi < word_buf.size(); ++wi) { + for (const W& b : word_buf) { Word w; - w.start = word_buf[wi].t_dtw_first; - w.end = (wi + 1 < word_buf.size()) - ? word_buf[wi + 1].t_dtw_first - : seg.end; - w.prob = word_buf[wi].p_sum / std::max(1, word_buf[wi].p_n); - w.text = word_buf[wi].text; + w.start = b.start; + w.end = b.end; + w.anchor = b.anchor; + w.prob = b.p_sum / std::max(1, b.p_n); + w.text = b.text; seg.words.push_back(w); } segments.push_back(std::move(seg)); @@ -547,7 +652,7 @@ int main(int argc, char** argv) { // whisper resolved — the detected one under "auto", and the forced one // otherwise, which is correct for both paths. std::string resolved_language = language; - const int lang_id = whisper_full_lang_id(ctx); + const int lang_id = input.empty() ? -1 : whisper_full_lang_id(ctx); if (lang_id >= 0) { if (const char* lang_str = whisper_lang_str(lang_id)) { resolved_language = lang_str; @@ -576,6 +681,7 @@ int main(int argc, char** argv) { {"word", w.text}, {"start", w.start}, {"end", w.end}, + {"anchor", w.anchor}, {"probability", w.prob}, }); } @@ -583,73 +689,78 @@ int main(int argc, char** argv) { segs.push_back(std::move(seg)); } reply["segments"] = std::move(segs); + if (vctx) reply["speech"] = std::move(speech_json); res.set_content(reply.dump(), "application/json"); }); - // POST /vad — multipart form with `file` (WAV: 16 kHz mono PCM16). - // Runs Silero VAD segmentation and returns speech intervals [start, end] in seconds. - svr.Post("/vad", [&](const httplib::Request& req, httplib::Response& res) { - if (!vctx) { - res.status = 503; - res.set_content(R"({"error":"VAD model was not loaded on server startup"})", "application/json"); - return; - } - auto it = req.files.find("file"); - if (it == req.files.end()) { - res.status = 400; - res.set_content(R"({"error":"missing 'file' form field"})", "application/json"); - return; - } - const auto& file_entry = it->second; - - static std::atomic tmp_vad_counter{0}; - const auto tmp_vad_id = tmp_vad_counter.fetch_add(1, std::memory_order_relaxed); - const auto tmp_vad_ts = std::chrono::high_resolution_clock::now().time_since_epoch().count(); - const std::string tmp_wav = (std::filesystem::temp_directory_path() / - ("openscreen-vad-" + std::to_string(tmp_vad_ts) + - "-" + std::to_string(tmp_vad_id) + ".wav")).string(); - { - std::ofstream out(tmp_wav, std::ios::binary); - out.write(file_entry.content.data(), - static_cast(file_entry.content.size())); - } + // POST /emissions — the CTC aligner's acoustic pass (issue #948, phase 3). + // Multipart form: `file` (the same WAV as /inference), `model` (path of a + // wav2vec2 GGUF, see ctc_aligner.h) and `regions` (JSON [[start_s, end_s], ...]). + // Answers the model's vocabulary and, per region, base64 float32 log-probs + // [frames x vocab]; frame i of a region sees the audio from + // `start + i * stride_s` for `receptive_s`. The forced alignment itself runs + // on the Node side (electron/stt/ctcAlign.ts). The model stays loaded until a + // request names another one. + CtcModelPtr aligner; + std::string aligner_path; + svr.Post("/emissions", [&](const httplib::Request& req, httplib::Response& res) { std::vector pcm; - int sample_rate = 0, channels = 0; - const bool ok = read_wav_pcm16(tmp_wav, pcm, sample_rate, channels); - std::error_code ec; - std::filesystem::remove(tmp_wav, ec); - if (!ok) { + if (!read_upload(req, res, pcm)) return; + const std::string model = req.get_file_value("model").content; + nlohmann::json regions = nlohmann::json::parse(req.get_file_value("regions").content, nullptr, false); + if (model.empty() || !regions.is_array()) { res.status = 400; - res.set_content(R"({"error":"failed to parse WAV"})", "application/json"); + res.set_content(R"({"error":"need 'model' and a JSON 'regions' array"})", "application/json"); return; } - if (sample_rate != 16000 || channels != 1) { - res.status = 400; - res.set_content( - R"({"error":"expected 16 kHz mono PCM16 WAV"})", - "application/json"); - return; - } - const std::lock_guard lk(infer_mu); - struct whisper_vad_params vad_params = whisper_vad_default_params(); - struct whisper_vad_segments* vad_segments = - whisper_vad_segments_from_samples(vctx, vad_params, pcm.data(), static_cast(pcm.size())); - - nlohmann::json reply; - reply["segments"] = nlohmann::json::array(); - if (vad_segments) { - const int n_segs = whisper_vad_segments_n_segments(vad_segments); - for (int i = 0; i < n_segs; ++i) { - const float t0 = whisper_vad_segments_get_segment_t0(vad_segments, i); - const float t1 = whisper_vad_segments_get_segment_t1(vad_segments, i); - reply["segments"].push_back({ - {"start", static_cast(t0) / 100.0}, - {"end", static_cast(t1) / 100.0} - }); + const auto t0 = std::chrono::steady_clock::now(); + if (!aligner || aligner_path != model) { + aligner.reset(); + std::string err; + aligner = ctc_load(model, cparams.use_gpu, threads, err); + if (!aligner) { + log("aligner: " + err); + res.status = 500; + res.set_content(nlohmann::json{{"error", "aligner: " + err}}.dump(), "application/json"); + return; } - whisper_vad_free_segments(vad_segments); + aligner_path = model; + log("aligner loaded on " + ctc_device(*aligner) + ": " + model); } + const CtcModelInfo& info = ctc_info(*aligner); + nlohmann::json out_regions = nlohmann::json::array(); + const int64_t n_pcm = static_cast(pcm.size()); + for (const auto& r : regions) { + if (!r.is_array() || r.size() != 2 || !r[0].is_number() || !r[1].is_number()) continue; + const int64_t from = std::clamp(std::llround(r[0].get() * 16000.0), 0, n_pcm); + const int64_t to = std::clamp(std::llround(r[1].get() * 16000.0), from, n_pcm); + std::vector lp; + int frames = 0; + std::string err; + if (!ctc_emissions(*aligner, pcm.data() + from, static_cast(to - from), lp, frames, err)) { + log("aligner: " + err); + res.status = 500; + res.set_content(nlohmann::json{{"error", "aligner: " + err}}.dump(), "application/json"); + return; + } + out_regions.push_back({ + {"start", from / 16000.0}, + {"frames", frames}, + {"logprobs", base64(lp.data(), lp.size() * sizeof(float))}, + }); + } + const double elapsed_s = std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); + nlohmann::json reply = { + {"vocab", info.vocab}, + {"blank", info.blank}, + {"languages", info.languages}, + {"stride_s", info.stride / 16000.0}, + {"receptive_s", info.receptive_field / 16000.0}, + {"device", ctc_device(*aligner)}, + {"elapsed_s", elapsed_s}, + {"regions", std::move(out_regions)}, + }; res.set_content(reply.dump(), "application/json"); }); diff --git a/electron/native/whisper-stt/whisper-patches/char-dtw.cpp b/electron/native/whisper-stt/whisper-patches/char-dtw.cpp new file mode 100644 index 0000000000..241ac5abdf --- /dev/null +++ b/electron/native/whisper-stt/whisper-patches/char-dtw.cpp @@ -0,0 +1,227 @@ +// OpenScreen (issue #948): character-level teacher-forced DTW. +// +// Spliced into whisper.cpp v1.9.1 by ../CMakeLists.txt in place of everything +// from `struct median_filter_user_data` to the end of +// whisper_exp_compute_token_level_timestamps_dtw. Same signature, same output: +// a `t_dtw` on every text token of the window, in centiseconds. +// +// Upstream teacher-forces the decoded BPE tokens once more and runs DTW on the +// cross-attention of the alignment heads. This does the same with the text cut +// into single characters ("Whisper Has an Internal Word Aligner", +// arXiv 2509.09987): one decoder row per character, so a word boundary is +// where the path reaches the row that predicts the space or the letter after +// it, not wherever a multi-letter token happens to start. On the word-timing +// harness (tools/stt-eval/word-timing) it takes the inner-boundary median from +// 31 to 16 ms. +// +// t_dtw keeps its upstream meaning: the time the path enters the row that +// predicts whatever follows the token, i.e. the END of the token. +// +// Departures from upstream, all measured on the harness: +// - the attention of the heads is averaged and each audio frame's column is +// scaled to unit L2 norm, as in the paper. Upstream's z-score + median filter +// does worse on characters (+7 ms median); +// - French drops one silent final consonant per word before aligning (vais, +// vous, plaît): its row otherwise eats the start of the next word, +60 to +// +140 ms on the words after it; +// - the decoder has n_text_ctx positions. A window too long as characters is +// aligned in several runs: one stretch of it as characters, the rest of the +// window around it as BPE tokens; +// - runs never step back in time: each t_dtw is at least the previous one. +static void whisper_exp_compute_token_level_timestamps_dtw( + struct whisper_context * ctx, + struct whisper_state * state, + struct whisper_full_params params, + int i_segment, + size_t n_segments, + int seek, + int n_frames, + int /*medfilt_width*/, + int n_threads) +{ + const int n_audio_ctx = state->exp_n_audio_ctx > 0 ? state->exp_n_audio_ctx : ctx->model.hparams.n_audio_ctx; + WHISPER_ASSERT(n_frames <= n_audio_ctx * 2); + WHISPER_ASSERT(ctx->params.dtw_aheads_preset != WHISPER_AHEADS_NONE); + + const whisper_token eot = whisper_token_eot(ctx); + const auto & vocab = ctx->vocab; + + std::vector text; + for (size_t i = i_segment; i < i_segment + n_segments; ++i) { + for (auto & t : state->result_all[i].tokens) { + if (t.id < eot) { + text.push_back(&t); + } + } + } + if (text.empty()) { + return; + } + + std::vector sot_seq = { whisper_token_sot(ctx) }; + if (whisper_is_multilingual(ctx)) { + const int lang_id = whisper_lang_id(params.language); + state->lang_id = lang_id; + sot_seq.push_back(whisper_token_lang(ctx, lang_id)); + } + sot_seq.push_back(whisper_token_not(ctx)); + + // Each text token as one token per character. A character with no token of + // its own goes as its bytes; a token that cannot be split stays whole. + const auto find_id = [&](const std::string & s, whisper_token & id) { + const auto it = vocab.token_to_id.find(s); + if (it == vocab.token_to_id.end()) return false; + id = it->second; + return true; + }; + std::vector> chars(text.size()); + for (size_t k = 0; k < text.size(); ++k) { + const std::string & s = vocab.id_to_token.at(text[k]->id); + std::vector out; + bool ok = true; + for (size_t i = 0; ok && i < s.size();) { + const unsigned char c = s[i]; + const size_t len = std::min(c < 0x80 ? 1 : (c >> 5) == 0x6 ? 2 : (c >> 4) == 0xE ? 3 : (c >> 3) == 0x1E ? 4 : 1, s.size() - i); + const std::string piece = s.substr(i, len); + i += len; + whisper_token id; + if (find_id(piece, id)) { + out.push_back(id); + continue; + } + for (size_t b = 0; ok && b < piece.size(); ++b) { + ok = find_id(piece.substr(b, 1), id); + if (ok) out.push_back(id); + } + } + chars[k] = ok ? std::move(out) : std::vector{ text[k]->id }; + } + + if (state->lang_id == whisper_lang_id("fr")) { + // One silent final consonant per word of two letters or more. A word is a + // run of letters, across tokens; a byte >= 0x80 counts as a letter. + const auto ch = [&](whisper_token id) { + const std::string & s = vocab.id_to_token.at(id); + return s.size() == 1 ? (unsigned char) s[0] : (unsigned char) 0x80; + }; + std::vector> word; // (token, character) + const auto end_word = [&]() { + if (word.size() >= 2 && strchr("stxdz", tolower(ch(chars[word.back().first][word.back().second]))) != nullptr) { + chars[word.back().first][word.back().second] = -1; + } + word.clear(); + }; + for (size_t k = 0; k < chars.size(); ++k) { + for (size_t i = 0; i < chars[k].size(); ++i) { + const unsigned char c = ch(chars[k][i]); + if (c >= 0x80 || isalpha(c)) { + word.push_back({ k, i }); + } else { + end_word(); + } + } + } + end_word(); + for (auto & cs : chars) { + cs.erase(std::remove(cs.begin(), cs.end(), -1), cs.end()); + } + } + + struct ggml_init_params gparams = { + /*.mem_size =*/ ctx->params.dtw_mem_size, + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ false, + }; + + int64_t t_floor = 0; + for (int i = i_segment - 1; i >= 0 && t_floor == 0; --i) { + for (const auto & t : state->result_all[i].tokens) { + if (t.id < eot && t.t_dtw > t_floor) t_floor = t.t_dtw; + } + } + + const int n_ctx = ctx->model.hparams.n_text_ctx; + const int n_text = (int) text.size(); + for (int k0 = 0; k0 < n_text;) { + // Tokens [k0, k1) as characters, as many as fit. + int len = (int) sot_seq.size() + n_text + 1; + int k1 = k0; + while (k1 < n_text && len - 1 + (int) chars[k1].size() <= n_ctx) { + len += (int) chars[k1].size() - 1; + ++k1; + } + if (k1 == k0) { + chars[k0] = { text[k0]->id }; + continue; + } + + std::vector tokens = sot_seq; + std::vector first(n_text + 1); // each text token's first row after `not` + for (int k = 0; k < n_text; ++k) { + first[k] = (int) (tokens.size() - sot_seq.size()); + if (k >= k0 && k < k1) { + tokens.insert(tokens.end(), chars[k].begin(), chars[k].end()); + } else { + tokens.push_back(text[k]->id); + } + } + first[n_text] = (int) (tokens.size() - sot_seq.size()); + tokens.push_back(eot); + + whisper_kv_cache_clear(state->kv_self); + whisper_batch_prep_legacy(state->batch, tokens.data(), tokens.size(), 0, 0); + whisper_kv_cache_seq_rm(state->kv_self, 0, 0, -1); + if (!whisper_decode_internal(*ctx, *state, state->batch, n_threads, true, nullptr, nullptr)) { + WHISPER_LOG_INFO("DECODER FAILED\n"); + WHISPER_ASSERT(0); + } + const ggml_tensor * qk = state->aheads_cross_QKs; + WHISPER_ASSERT(qk != nullptr && qk->type == GGML_TYPE_F32 && ggml_is_contiguous(qk)); + const int64_t n_tok = qk->ne[0]; + const int64_t n_actx = qk->ne[1]; + const int64_t n_head = qk->ne[2]; + const int64_t F = n_frames / 2; // 20 ms per frame + WHISPER_ASSERT(F <= n_actx); + auto & data = state->aheads_cross_QKs_data; + data.resize(n_tok * n_actx * n_head); + ggml_backend_tensor_get(qk, data.data(), 0, sizeof(float) * data.size()); + + // Rows: `not`, then every token after it but eot. Row r predicts the + // token after it, so entering row r is where that token starts. + const int64_t row0 = (int64_t) sot_seq.size() - 1; + const int64_t R = (int64_t) tokens.size() - 1 - row0; + + struct ggml_context * gctx = ggml_init(gparams); + ggml_tensor * x = ggml_new_tensor_2d(gctx, GGML_TYPE_F32, R, F); + float * xd = (float *) x->data; + for (int64_t f = 0; f < F; ++f) { + double norm = 0.0; + for (int64_t r = 0; r < R; ++r) { + float a = 0.0f; + for (int64_t h = 0; h < n_head; ++h) { + a += data[(row0 + r) + f * n_tok + h * n_tok * n_actx]; + } + xd[r + f * R] = a; + norm += (double) a * a; + } + norm = std::sqrt(norm) + 1e-9; + for (int64_t r = 0; r < R; ++r) { + xd[r + f * R] = (float) (-xd[r + f * R] / norm); + } + } + + const ggml_tensor * path = dtw_and_backtrace(gctx, x); + std::vector entry(R, 0); + for (int64_t i = path->ne[1] - 1; i >= 0; --i) { + const int32_t r = whisper_get_i32_nd(path, 0, i, 0, 0); + if (r >= 0 && r < R) entry[r] = whisper_get_i32_nd(path, 1, i, 0, 0); + } + ggml_free(gctx); + + for (int k = k0; k < k1; ++k) { + t_floor = std::max(t_floor, (int64_t) entry[first[k + 1]] * 2 + seek); + text[k]->t_dtw = t_floor; + } + k0 = k1; + } +} diff --git a/electron/preload.ts b/electron/preload.ts index 80e583f241..feb3942dfb 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -1,11 +1,22 @@ -import { contextBridge, ipcRenderer, webUtils } from "electron"; +import { contextBridge, ipcRenderer, sharedTexture, webUtils } from "electron"; import type { NativeLinuxRecordingRequest } from "../src/lib/nativeLinuxRecording"; import type { NativeMacRecordingRequest } from "../src/lib/nativeMacRecording"; import type { NativeWindowsRecordingRequest } from "../src/lib/nativeWindowsRecording"; import type { RecordingSession, StoreRecordedSessionInput } from "../src/lib/recordingSession"; import type { ShortcutBinding } from "../src/lib/shortcuts"; -import type { AiEditionChatEvent } from "../src/native/contracts"; -import { NATIVE_BRIDGE_CHANNEL, type NativeBridgeRequest } from "../src/native/contracts"; +import type { + AiEditionChatEvent, + AiEditionMcpHostRequest, + AiEditionMcpHostResponse, + CompositorSharedFrameMeta, +} from "../src/native/contracts"; +import { + AI_EDITION_MCP_HOST_CHANNEL, + AI_EDITION_MCP_REQUEST_CHANNEL, + AI_EDITION_MCP_RESPONSE_CHANNEL, + NATIVE_BRIDGE_CHANNEL, + type NativeBridgeRequest, +} from "../src/native/contracts"; import type { RecordingPrefs } from "./ipc/handlers"; import type { SttStatusEvent, @@ -24,6 +35,23 @@ const assetBaseUrl = assetBaseUrlArg ? assetBaseUrlArg.slice(ASSET_BASE_URL_ARG_ // so a synchronous read here saves the renderer's every-call IPC round-trip. const PLATFORM = process.platform; +// Preview frames the main process hands over as shared GPU textures (see +// `compositorViewService`). Electron gives up on a send after 1 s without a receiver, so it is +// registered here, at load, and forwards to whichever listener the page has set. +type CompositorFrameListener = (frame: VideoFrame, meta: CompositorSharedFrameMeta) => void; +let compositorFrameListener: CompositorFrameListener | null = null; +sharedTexture?.setSharedTextureReceiver(async ({ importedSharedTexture }, meta) => { + const frame = importedSharedTexture.getVideoFrame(); + try { + compositorFrameListener?.(frame, meta as CompositorSharedFrameMeta); + } finally { + // The page drew from its own copy (contextBridge clones the frame) and closed it. Closing + // ours and releasing the import is what lets the native ring write the slot again. + frame.close(); + importedSharedTexture.release(); + } +}); + contextBridge.exposeInMainWorld("electronAPI", { assetBaseUrl, @@ -75,6 +103,14 @@ contextBridge.exposeInMainWorld("electronAPI", { ipcRenderer.on("export:native-progress", handler); return () => ipcRenderer.off("export:native-progress", handler); }, + onCompositorFrame: (listener: CompositorFrameListener) => { + compositorFrameListener = listener; + return () => { + if (compositorFrameListener === listener) { + compositorFrameListener = null; + } + }; + }, invokeNativeBridge: (request: NativeBridgeRequest) => { return ipcRenderer.invoke(NATIVE_BRIDGE_CHANNEL, request) as Promise; }, @@ -129,6 +165,9 @@ contextBridge.exposeInMainWorld("electronAPI", { usesSystemSourcePicker: (): Promise => { return ipcRenderer.invoke("uses-system-source-picker"); }, + getLastPickedSource: (): Promise => { + return ipcRenderer.invoke("get-last-picked-source"); + }, openNotes: () => { return ipcRenderer.invoke("open-notes"); }, @@ -255,6 +294,11 @@ contextBridge.exposeInMainWorld("electronAPI", { startNativeMacRecording: (request: NativeMacRecordingRequest) => { return ipcRenderer.invoke("start-native-mac-recording", request); }, + onNativeMacSystemAudioUnavailable: (callback: () => void) => { + const listener = () => callback(); + ipcRenderer.on("native-mac-system-audio-unavailable", listener); + return () => ipcRenderer.removeListener("native-mac-system-audio-unavailable", listener); + }, pauseNativeMacRecording: () => { return ipcRenderer.invoke("pause-native-mac-recording"); }, @@ -517,6 +561,33 @@ contextBridge.exposeInMainWorld("electronAPI", { ipcRenderer.on("ai-edition.chat-event", listener); return () => ipcRenderer.removeListener("ai-edition.chat-event", listener); }, + // The editor answers the MCP server's reads and writes of the live document. + // Subscribing is what makes this window the one the server asks; the + // returned unsubscribe withdraws it. See electron/mcp/editor-document-host.ts. + onAiEditionMcpRequest: ( + callback: (request: AiEditionMcpHostRequest) => Promise, + ) => { + const listener = (_e: unknown, request: AiEditionMcpHostRequest) => { + void callback(request).then( + (result) => + ipcRenderer.send(AI_EDITION_MCP_RESPONSE_CHANNEL, { + requestId: request.requestId, + result, + }), + () => + ipcRenderer.send(AI_EDITION_MCP_RESPONSE_CHANNEL, { + requestId: request.requestId, + result: null, + }), + ); + }; + ipcRenderer.on(AI_EDITION_MCP_REQUEST_CHANNEL, listener); + ipcRenderer.send(AI_EDITION_MCP_HOST_CHANNEL, true); + return () => { + ipcRenderer.removeListener(AI_EDITION_MCP_REQUEST_CHANNEL, listener); + ipcRenderer.send(AI_EDITION_MCP_HOST_CHANNEL, false); + }; + }, stt: { transcribe: (request: SttTranscribeRequest): Promise => { return ipcRenderer.invoke("stt:transcribe", request) as Promise; diff --git a/electron/recording-source-settings.test.ts b/electron/recording-source-settings.test.ts index f56a684fa2..40aefc588a 100644 --- a/electron/recording-source-settings.test.ts +++ b/electron/recording-source-settings.test.ts @@ -1,8 +1,10 @@ import { describe, expect, it } from "vitest"; import { + describeMacPickerSource, describeRecordingSource, enumerationIncludesSourceKind, mergeEnumeratedSources, + rememberedPickerSourceName, resolveCurrentRecordingSource, resolveRecordingSource, restoreRecordingSourceAfterEnumeration, @@ -139,3 +141,32 @@ describe("recording source settings", () => { ).toEqual(display); }); }); + +describe("the display picked in Apple's picker, kept only to be named", () => { + const picked = { id: "mac-picker:display:5", name: "Studio Display", display_id: "5" }; + + it("describes a display pick as a screen and a window pick as a window", () => { + expect(describeMacPickerSource(picked, "display")).toEqual({ + platform: "darwin", + kind: "screen", + id: "mac-picker:display:5", + name: "Studio Display", + displayId: "5", + }); + // Not derived from the id: `recordingSourceKindFromId` only knows desktopCapturer's + // `window:` prefix and would call a picked window a screen. + expect( + describeMacPickerSource( + { id: "mac-picker:window:42", name: "Notes", display_id: "" }, + "window", + ), + ).toMatchObject({ kind: "window", displayId: null }); + }); + + it("names the last pick only where Apple's picker owns the choice", () => { + const last = describeMacPickerSource(picked, "display"); + expect(rememberedPickerSourceName(last, true)).toBe("Studio Display"); + expect(rememberedPickerSourceName(last, false)).toBeNull(); + expect(rememberedPickerSourceName(null, true)).toBeNull(); + }); +}); diff --git a/electron/recording-source-settings.ts b/electron/recording-source-settings.ts index d8d89425bc..2480d24a88 100644 --- a/electron/recording-source-settings.ts +++ b/electron/recording-source-settings.ts @@ -23,6 +23,36 @@ export function describeRecordingSource( }; } +/** + * What Apple's picker returned, kept only to name it after a relaunch. It is not a + * restorable descriptor: the filter behind the pick cannot leave the helper, and rebuilding + * one needs the Screen Recording grant the picker exists to avoid. + */ +export function describeMacPickerSource( + source: { id: string; name: string; display_id: string }, + kind: "display" | "window", +): RecordingSourceDescriptor { + return { + platform: "darwin", + kind: kind === "window" ? "window" : "screen", + id: source.id, + name: source.name, + displayId: source.display_id || null, + }; +} + +/** + * The name to show for the last pick while nothing is live. Only where Apple's picker owns + * the choice: everywhere else the descriptor restores a real source, so there is nothing to + * show in its place. + */ +export function rememberedPickerSourceName( + lastSource: RecordingSourceDescriptor | null | undefined, + pickerOwnsSources: boolean, +): string | null { + return pickerOwnsSources && lastSource ? lastSource.name : null; +} + /** * A screen-only (or window-only) enumeration cannot prove a source of the * other kind is gone. Clearing the live pick in that case is a false negative. diff --git a/electron/recording/displaySleepBlocker.test.ts b/electron/recording/displaySleepBlocker.test.ts new file mode 100644 index 0000000000..ff420d52ce --- /dev/null +++ b/electron/recording/displaySleepBlocker.test.ts @@ -0,0 +1,52 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const { start, stop } = vi.hoisted(() => { + let nextId = 1; + return { start: vi.fn(() => nextId++), stop: vi.fn() }; +}); + +vi.mock("electron", () => ({ powerSaveBlocker: { start, stop } })); + +describe("setDisplaySleepBlocked", () => { + let setDisplaySleepBlocked: (recording: boolean) => void; + + beforeEach(async () => { + vi.resetModules(); + start.mockClear(); + stop.mockClear(); + ({ setDisplaySleepBlocked } = await import("./displaySleepBlocker")); + }); + + it("starts one display-sleep blocker per recording and stops that same one", () => { + setDisplaySleepBlocked(true); + expect(start).toHaveBeenCalledOnce(); + expect(start).toHaveBeenCalledWith("prevent-display-sleep"); + + setDisplaySleepBlocked(false); + expect(stop).toHaveBeenCalledOnce(); + expect(stop).toHaveBeenCalledWith(start.mock.results[0].value); + }); + + it("never stacks a second blocker when a start is reported twice", () => { + setDisplaySleepBlocked(true); + setDisplaySleepBlocked(true); + setDisplaySleepBlocked(false); + expect(start).toHaveBeenCalledOnce(); + expect(stop).toHaveBeenCalledOnce(); + }); + + it("ignores a stop with nothing started, as a failed start then cancel reports", () => { + setDisplaySleepBlocked(false); + expect(stop).not.toHaveBeenCalled(); + }); + + it("leaves nothing running across consecutive takes", () => { + for (let take = 0; take < 3; take++) { + setDisplaySleepBlocked(true); + setDisplaySleepBlocked(false); + setDisplaySleepBlocked(false); + } + expect(start).toHaveBeenCalledTimes(3); + expect(stop.mock.calls.map(([id]) => id)).toEqual(start.mock.results.map((r) => r.value)); + }); +}); diff --git a/electron/recording/displaySleepBlocker.ts b/electron/recording/displaySleepBlocker.ts new file mode 100644 index 0000000000..c584074939 --- /dev/null +++ b/electron/recording/displaySleepBlocker.ts @@ -0,0 +1,18 @@ +import { powerSaveBlocker } from "electron"; + +let blockerId: number | null = null; + +/** + * Keeps the display awake for the whole take, pauses included (#935). Driven by the + * same recording flag as the tray, which every backend's start and every stop, + * cancel and failure path already report, so it is idempotent in both directions: + * a repeated `true` never stacks a second blocker, a stray `false` is a no-op. + */ +export function setDisplaySleepBlocked(recording: boolean) { + if (recording && blockerId === null) { + blockerId = powerSaveBlocker.start("prevent-display-sleep"); + } else if (!recording && blockerId !== null) { + powerSaveBlocker.stop(blockerId); + blockerId = null; + } +} diff --git a/electron/recording/nativeWindowsCaptureStop.test.ts b/electron/recording/nativeWindowsCaptureStop.test.ts index 68ebf01c95..0def7936f5 100644 --- a/electron/recording/nativeWindowsCaptureStop.test.ts +++ b/electron/recording/nativeWindowsCaptureStop.test.ts @@ -7,8 +7,13 @@ import { NATIVE_WINDOWS_SALVAGEABLE_OUTPUT_BYTES, readMicrophoneDefaulted, readMicrophoneUnavailable, + readReportedWebcamPaths, + readSecondaryWindowsApplied, readStoppedPath, + readStoppedWebcamPaths, + readUnavailableWebcamIndices, readWebcamFormat, + readWebcamFormatAt, readWebcamUnavailable, terminateNativeWindowsCapture, waitForNativeWindowsCaptureStop, @@ -125,6 +130,27 @@ describe("readMicrophoneDefaulted", () => { }); }); +describe("readSecondaryWindowsApplied", () => { + it("is true when the helper drew menus and popups into the capture", () => { + const output = + '{"event":"cursor-capture","schemaVersion":2,"requested":false,"applied":false}\n' + + '{"event":"secondary-windows","applied":true}\n' + + "Recording started\n"; + expect(readSecondaryWindowsApplied(output)).toBe(true); + }); + + it("is false on a runtime older than Windows 11 24H2", () => { + expect(readSecondaryWindowsApplied('{"event":"secondary-windows","applied":false}\n')).toBe( + false, + ); + }); + + it("is null when the helper said nothing, as for a monitor source", () => { + const output = '{"event":"cursor-capture","applied":true}\nRecording started\n'; + expect(readSecondaryWindowsApplied(output)).toBe(null); + }); +}); + describe("readMicrophoneUnavailable", () => { // The line ScreenCaptureRecorder.swift emits on macOS 13 and 14, which have no // `captureMicrophone`, before recording the take without it (#700). @@ -517,3 +543,75 @@ describe("terminateNativeWindowsCapture", () => { expect(helper.killCalls).toBe(1); }); }); + +describe("per-camera helper events", () => { + const unavailable = (i?: number) => + `{"event":"warning","code":"webcam-unavailable"${i === undefined ? "" : `,"index":${i}`},"message":"x"}`; + + it("collects unavailable cameras by index, an old event counting as camera 0", () => { + expect(readUnavailableWebcamIndices(`${unavailable(1)}\n${unavailable(3)}`)).toEqual([1, 3]); + expect(readUnavailableWebcamIndices(unavailable())).toEqual([0]); + expect(readUnavailableWebcamIndices("nothing")).toEqual([]); + }); + + it("ignores other warnings and reads a prefixed unavailable event", () => { + const other = '{"event":"warning","code":"microphone-defaulted","index":2}'; + const prefixed = `INFO: camera lost ${unavailable(2)}`; + expect(readUnavailableWebcamIndices(`${other}\n${prefixed}`)).toEqual([2]); + }); + + it("reads the format of a given camera", () => { + const out = [ + '{"event":"webcam-format","schemaVersion":2,"index":0,"width":1920,"height":1080,"fps":30,"deviceName":"A"}', + '{"event":"webcam-format","schemaVersion":2,"index":1,"width":1280,"height":720,"fps":30,"deviceName":"B"}', + ].join("\n"); + expect(readWebcamFormatAt(out, 1)?.width).toBe(1280); + expect(readWebcamFormatAt(out, 0)?.width).toBe(1920); + expect(readWebcamFormatAt(out, 2)).toBeNull(); + }); + + it("reads a format event glued to a diagnostic prefix, the last one winning", () => { + const out = [ + 'INFO: DirectShow webcam connected subtype NV12 {"event":"webcam-format","index":1,"width":640,"height":480}', + 'INFO: again {"event":"webcam-format","index":1,"width":1280,"height":720}', + ].join("\n"); + expect(readWebcamFormatAt(out, 1)?.width).toBe(1280); + }); + + it("treats a format without index as camera 0", () => { + expect(readWebcamFormatAt('{"event":"webcam-format","width":800}', 0)?.width).toBe(800); + }); + + it("reads every stopped camera path, falling back to the single legacy path", () => { + expect( + readStoppedWebcamPaths( + '{"event":"recording-stopped","webcamPath":"a.mp4","webcamPaths":["a.mp4","b.mp4"]}', + ), + ).toEqual(["a.mp4", "b.mp4"]); + expect(readStoppedWebcamPaths('{"event":"recording-stopped","webcamPath":"a.mp4"}')).toEqual([ + "a.mp4", + ]); + expect(readStoppedWebcamPaths('{"event":"recording-stopped"}')).toEqual([]); + }); + + it("tells a reported webcamPaths list apart from its absence", () => { + expect( + readReportedWebcamPaths('{"event":"recording-stopped","webcamPaths":["a.mp4","b.mp4"]}'), + ).toEqual(["a.mp4", "b.mp4"]); + expect(readReportedWebcamPaths('{"event":"recording-stopped","webcamPaths":[]}')).toEqual([]); + // An old helper sends only the legacy path; that is "unknown", not "none". + expect( + readReportedWebcamPaths('{"event":"recording-stopped","webcamPath":"a.mp4"}'), + ).toBeNull(); + expect(readReportedWebcamPaths("Recording stopped. Output path: s.mp4")).toBeNull(); + }); + + it("reads JSON-escaped Windows paths behind a prefix", () => { + const line = + 'INFO: x {"event":"recording-stopped","schemaVersion":2,"screenPath":"C:\\\\Users\\\\me\\\\s.mp4","webcamPath":"C:\\\\Users\\\\me\\\\s-webcam.mp4","webcamPaths":["C:\\\\Users\\\\me\\\\s-webcam.mp4","C:\\\\Users\\\\me\\\\s-webcam-2.mp4"]}'; + expect(readStoppedWebcamPaths(line)).toEqual([ + "C:\\Users\\me\\s-webcam.mp4", + "C:\\Users\\me\\s-webcam-2.mp4", + ]); + }); +}); diff --git a/electron/recording/nativeWindowsCaptureStop.ts b/electron/recording/nativeWindowsCaptureStop.ts index 9f2e2db33d..05c2a0f1f4 100644 --- a/electron/recording/nativeWindowsCaptureStop.ts +++ b/electron/recording/nativeWindowsCaptureStop.ts @@ -122,6 +122,20 @@ export function readMicrophoneDefaulted(output: string) { return output.includes('"code":"microphone-defaulted"'); } +/** + * Did the helper draw menus and popups into a window recording? + * + * `true` on Windows 11 24H2+, `false` where the runtime cannot (the recording is + * the window alone, as before), `null` when the helper never said: a monitor + * source, the kill switch, or an older helper (getopenscreen/openscreen#894). + */ +export function readSecondaryWindowsApplied(output: string) { + if (output.includes('"event":"secondary-windows","applied":true')) { + return true; + } + return output.includes('"event":"secondary-windows","applied":false') ? false : null; +} + /** * Did the macOS helper record without the microphone it was asked for? * @@ -204,6 +218,93 @@ export function readWebcamFormat(output: string) { } } +type HelperEvent = Record; + +/** + * Every `{"event":""…}` object in the output, in order. + * + * Locates each object start and slices it with `findObjectEnd` instead of + * parsing whole lines, because diagnostics can be glued in front of an event + * (see `readWebcamFormat`). Slices that do not parse are skipped. + */ +function readHelperEvents(output: string, name: string): HelperEvent[] { + const needle = `{"event":"${name}"`; + const events: HelperEvent[] = []; + let from = 0; + for (;;) { + const start = output.indexOf(needle, from); + if (start === -1) { + return events; + } + const end = findObjectEnd(output, start); + if (end === -1) { + return events; + } + from = end + 1; + try { + const parsed: unknown = JSON.parse(output.slice(start, end + 1)); + if (parsed !== null && typeof parsed === "object" && !Array.isArray(parsed)) { + events.push(parsed as HelperEvent); + } + } catch { + // Not a complete JSON object; keep looking. + } + } +} + +/** A helper event's camera index; an event without one comes from an old helper (camera 0). */ +function eventCameraIndex(event: HelperEvent) { + return typeof event.index === "number" ? event.index : 0; +} + +/** Indices of every camera the helper reported as unavailable (no index = camera 0). */ +export function readUnavailableWebcamIndices(output: string): number[] { + return readHelperEvents(output, "warning") + .filter((event) => event.code === "webcam-unavailable") + .map(eventCameraIndex); +} + +/** The last `webcam-format` event of the given camera (no index = camera 0), or null. */ +export function readWebcamFormatAt( + output: string, + index: number, +): ReturnType { + const match = readHelperEvents(output, "webcam-format") + .filter((event) => eventCameraIndex(event) === index) + .at(-1); + return (match as ReturnType) ?? null; +} + +/** + * The `webcamPaths` list of the last `recording-stopped` event, or null when + * there is no such event or it carried no `webcamPaths` key. + * + * Null is the signal that matters: only a helper that sends the key says which + * cameras were still recording at stop, so only then may a camera missing from + * it be called one that stopped early. An old helper, or a stop without the + * event, must read as "unknown", never as "every camera stopped". + */ +export function readReportedWebcamPaths(output: string): string[] | null { + const event = readHelperEvents(output, "recording-stopped").at(-1); + if (!event || !Array.isArray(event.webcamPaths)) { + return null; + } + return event.webcamPaths.filter((entry): entry is string => typeof entry === "string"); +} + +/** + * Camera files the helper reported at stop: `webcamPaths`, else the single + * legacy `webcamPath`, else none. + */ +export function readStoppedWebcamPaths(output: string): string[] { + const reported = readReportedWebcamPaths(output); + if (reported) { + return reported; + } + const event = readHelperEvents(output, "recording-stopped").at(-1); + return typeof event?.webcamPath === "string" && event.webcamPath ? [event.webcamPath] : []; +} + /** * The most useful line of a failed helper run, for a toast. * diff --git a/electron/recording/nativeWindowsWebcams.test.ts b/electron/recording/nativeWindowsWebcams.test.ts new file mode 100644 index 0000000000..04ce24811d --- /dev/null +++ b/electron/recording/nativeWindowsWebcams.test.ts @@ -0,0 +1,319 @@ +import { describe, expect, it } from "vitest"; +import { + additionalWebcamLabels, + buildHelperWebcamConfig, + collectStoppedWebcams, + dedupeAdditionalWebcams, + deskWebcamPath, + isWebcamSidecarFile, + labelsOfUnavailableAdditionalWebcams, + labelsOfWebcamsStoppedEarly, + recordedDeskCamera, + remapDeskCamera, + stripWebcamSuffix, + webcamOutputPath, +} from "./nativeWindowsWebcams"; + +describe("nativeWindowsWebcams", () => { + it("names camera files", () => { + expect(webcamOutputPath("C:\\r", "rec-", 7, 1)).toMatch(/rec-7-webcam\.mp4$/); + expect(webcamOutputPath("C:\\r", "rec-", 7, 3)).toMatch(/rec-7-webcam-3\.mp4$/); + }); + + it("recognizes every camera file of a recording and nothing else", () => { + for (const f of ["rec-7-webcam.mp4", "rec-7-webcam-2.mp4", "rec-7-webcam-4.webm"]) { + expect(isWebcamSidecarFile(f)).toBe(true); + } + for (const f of ["rec-7.mp4", "rec-7-webcamera.mp4", "rec-7-webcam-x.mp4"]) { + expect(isWebcamSidecarFile(f)).toBe(false); + } + expect(stripWebcamSuffix("rec-7-webcam-2")).toBe("rec-7"); + expect(stripWebcamSuffix("rec-7-webcam")).toBe("rec-7"); + expect(stripWebcamSuffix("rec-7")).toBe("rec-7"); + }); + + it("dedupes a device that equals camera 1, duplicates, and caps at three", () => { + const extras = [ + { deviceId: "a", deviceName: "Front" }, + { deviceId: "b", deviceName: "Desk" }, + { deviceId: "b", deviceName: "Desk" }, + { deviceName: "Side" }, + { deviceName: "Top" }, + { deviceName: "Fifth" }, + ]; + expect( + dedupeAdditionalWebcams({ deviceId: "a", deviceName: "Front" }, extras).map( + (e) => e.deviceName, + ), + ).toEqual(["Desk", "Side", "Top"]); + }); + + it("drops extras that name no device at all", () => { + expect(dedupeAdditionalWebcams(null, [{ deviceName: " " }, { deviceName: "Desk" }])).toEqual([ + { deviceName: "Desk" }, + ]); + }); + + it("skips malformed entries from IPC", () => { + const junk = [ + null, + { deviceName: 3 }, + { deviceId: 4, deviceName: "X" }, + { deviceName: "Desk" }, + ]; + expect(dedupeAdditionalWebcams(null, junk as unknown as Array<{ deviceName: string }>)).toEqual( + [{ deviceName: "Desk" }], + ); + }); + + it("never matches an extra with an id to an id-less camera 1 by name", () => { + // Two cameras of the same model: same name, and camera 1 came without an id. + expect( + dedupeAdditionalWebcams({ deviceName: "USB Camera" }, [ + { deviceId: "b", deviceName: "USB Camera" }, + ]), + ).toEqual([{ deviceId: "b", deviceName: "USB Camera" }]); + // Without an id on either side the name is all there is, and it still matches. + expect( + dedupeAdditionalWebcams({ deviceName: "USB Camera" }, [{ deviceName: "USB Camera" }]), + ).toEqual([]); + }); + + describe("labelsOfWebcamsStoppedEarly", () => { + const front = String.raw`C:\Rec\r-webcam.mp4`; + const desk = String.raw`C:\Rec\r-webcam-2.mp4`; + const side = String.raw`C:\Rec\r-webcam-3.mp4`; + const requested = [ + { path: front, label: "Front" }, + { path: desk, label: "Desk" }, + { path: side, label: "Side" }, + ]; + const sizes = new Map([ + [front, 100], + [desk, 100], + [side, 0], + ]); + + it("names a kept camera the helper no longer listed, camera 1 included", () => { + expect( + labelsOfWebcamsStoppedEarly({ + requested, + sizes, + helperWebcamPaths: [desk], + }), + ).toEqual(["Front"]); + }); + + it("leaves out a camera whose file was not kept: that one was not recorded", () => { + expect(labelsOfWebcamsStoppedEarly({ requested, sizes, helperWebcamPaths: [] })).toEqual([ + "Front", + "Desk", + ]); + }); + + it("flags nothing without a webcamPaths key", () => { + expect(labelsOfWebcamsStoppedEarly({ requested, sizes, helperWebcamPaths: null })).toEqual( + [], + ); + }); + + it("matches paths regardless of case and separator", () => { + expect( + labelsOfWebcamsStoppedEarly({ + requested, + sizes, + helperWebcamPaths: ["c:/rec/R-WEBCAM.mp4", String.raw`c:\REC//r-webcam-2.MP4`], + }), + ).toEqual([]); + }); + }); + + it("drops an empty additional camera file and names it", () => { + const r = collectStoppedWebcams({ + camera1Enabled: true, + requested: [ + { path: "w.mp4", label: "Front" }, + { path: "w-2.mp4", label: "Desk" }, + { path: "w-3.mp4", label: "Side" }, + ], + sizes: new Map([ + ["w.mp4", 100], + ["w-2.mp4", 0], + ["w-3.mp4", 50], + ]), + }); + expect(r).toEqual({ + camera1: "w.mp4", + additional: [{ path: "w-3.mp4", label: "Side" }], + dropped: ["Desk"], + }); + }); + + it("keeps camera 1 and reports a camera whose file never appeared as not recorded", () => { + const r = collectStoppedWebcams({ + camera1Enabled: true, + requested: [ + { path: "w.mp4", label: "Front" }, + { path: "w-2.mp4", label: "Desk" }, + ], + sizes: new Map([["w.mp4", 100]]), + }); + expect(r).toEqual({ camera1: "w.mp4", additional: [], dropped: ["Desk"] }); + }); + + it("keeps extras when camera 1 is lost, leaving camera 1 out of dropped", () => { + const r = collectStoppedWebcams({ + camera1Enabled: true, + requested: [ + { path: "w.mp4", label: "Front" }, + { path: "w-2.mp4", label: "Desk" }, + ], + sizes: new Map([ + ["w.mp4", 0], + ["w-2.mp4", 10], + ]), + }); + expect(r).toEqual({ additional: [{ path: "w-2.mp4", label: "Desk" }], dropped: [] }); + }); + + it("maps unavailable helper indices to the labels of the extras", () => { + const requested = [ + { path: "w.mp4", label: "Front" }, + { path: "w-2.mp4", label: "Desk" }, + { path: "w-3.mp4", label: "Side" }, + ]; + expect(labelsOfUnavailableAdditionalWebcams(requested, [0, 2, 2, 9])).toEqual(["Side"]); + }); + + it("labels extras by device name, else Camera ", () => { + expect(additionalWebcamLabels("Front", [{ deviceName: "Desk" }, { deviceName: " " }])).toEqual( + ["Desk", "Camera 3"], + ); + }); + + it("tells cameras with the same name apart by occurrence, counting camera 1", () => { + expect( + additionalWebcamLabels("USB Camera", [ + { deviceName: "USB Camera" }, + { deviceName: "Desk" }, + { deviceName: " USB Camera " }, + ]), + ).toEqual(["USB Camera (2)", "Desk", "USB Camera (3)"]); + expect( + additionalWebcamLabels(undefined, [ + { deviceName: "USB Camera" }, + { deviceName: "USB Camera" }, + ]), + ).toEqual(["USB Camera", "USB Camera (2)"]); + }); + + it("builds a start config with the legacy fields and a list of every camera", () => { + const config = buildHelperWebcamConfig({ + camera1: { + enabled: true, + deviceId: "id-1", + deviceName: "Front", + width: 1280, + height: 720, + fps: 30, + }, + camera1Clsid: "{c1}", + camera1Path: "C:\\r\\rec-7-webcam.mp4", + extras: [ + { deviceId: "id-2", deviceName: "Desk", clsid: null, path: "C:\\r\\rec-7-webcam-2.mp4" }, + ], + }); + const parsed = JSON.parse(JSON.stringify(config)); + expect(parsed).toMatchObject({ + webcamEnabled: true, + webcamDeviceId: "id-1", + webcamDeviceName: "Front", + webcamDirectShowClsid: "{c1}", + webcamWidth: 1280, + webcamHeight: 720, + webcamFps: 30, + }); + expect(parsed.webcams).toEqual([ + { + camDeviceId: "id-1", + camDeviceName: "Front", + camClsid: "{c1}", + camWidth: 1280, + camHeight: 720, + camFps: 30, + camPath: "C:\\r\\rec-7-webcam.mp4", + }, + { + camDeviceId: "id-2", + camDeviceName: "Desk", + camClsid: null, + camWidth: 1280, + camHeight: 720, + camFps: 30, + camPath: "C:\\r\\rec-7-webcam-2.mp4", + }, + ]); + }); + + it("sends no camera list while camera 1 is off", () => { + const config = buildHelperWebcamConfig({ + camera1: { enabled: false, width: 1280, height: 720, fps: 30 }, + camera1Clsid: null, + camera1Path: "C:\\r\\rec-7-webcam.mp4", + extras: [{ deviceName: "Desk", clsid: null, path: "C:\\r\\rec-7-webcam-2.mp4" }], + }); + expect(config.webcamEnabled).toBe(false); + expect(config.webcams).toEqual([]); + }); +}); + +describe("desk camera of a native take", () => { + const requested = [ + { deviceId: "b", deviceName: "B" }, + { deviceId: "a", deviceName: "A" }, + { deviceId: "c", deviceName: "C" }, + ]; + // "a" is camera 1 and is dropped by the dedupe: the files go to B and C. + const kept = [requested[0], requested[2]]; + const keptPaths = ["/r-webcam-2.mp4", "/r-webcam-3.mp4"]; + + it("finds the file the desk camera records into", () => { + const input = { camera1Path: "/r-webcam.mp4", requested, kept, keptPaths }; + expect(deskWebcamPath({ ...input, deskCamera: 0 })).toBe("/r-webcam.mp4"); + expect(deskWebcamPath({ ...input, deskCamera: 1 })).toBe("/r-webcam-2.mp4"); + expect(deskWebcamPath({ ...input, deskCamera: 3 })).toBe("/r-webcam-3.mp4"); + }); + + it("has no file for a desk camera that is not recorded or not an index", () => { + const input = { camera1Path: "/r-webcam.mp4", requested, kept, keptPaths }; + for (const deskCamera of [2, 4, -1, 0.5, "1", undefined]) { + expect(deskWebcamPath({ ...input, deskCamera })).toBeNull(); + } + expect(deskWebcamPath({ ...input, camera1Path: null, deskCamera: 0 })).toBeNull(); + }); + + it("indexes the desk camera in recorded order", () => { + const additional = [ + { path: "/r-webcam-2.mp4", label: "B" }, + { path: "/r-webcam-3.mp4", label: "C" }, + ]; + expect(recordedDeskCamera("/r-webcam.mp4", "/r-webcam.mp4", additional)).toBe(0); + expect(recordedDeskCamera("/r-webcam-3.mp4", "/r-webcam.mp4", additional)).toBe(2); + // Camera 2 came out empty: camera 3 moves up to index 1. + expect(recordedDeskCamera("/r-webcam-3.mp4", "/r-webcam.mp4", [additional[1]])).toBe(1); + }); + + it("has no recorded index for a desk camera whose file was lost", () => { + expect(recordedDeskCamera("/r-webcam-2.mp4", "/r-webcam.mp4", [])).toBeUndefined(); + expect(recordedDeskCamera("/r-webcam.mp4", undefined, [])).toBeUndefined(); + expect(recordedDeskCamera(null, "/r-webcam.mp4", [])).toBeUndefined(); + }); + + it("follows the desk camera when restored cameras are dropped", () => { + expect(remapDeskCamera(0, true, [0, 2])).toBe(0); + expect(remapDeskCamera(3, true, [0, 2])).toBe(2); + expect(remapDeskCamera(2, true, [0, 2])).toBeUndefined(); + expect(remapDeskCamera(0, false, [0, 1])).toBeUndefined(); + expect(remapDeskCamera(undefined, true, [0])).toBeUndefined(); + }); +}); diff --git a/electron/recording/nativeWindowsWebcams.ts b/electron/recording/nativeWindowsWebcams.ts new file mode 100644 index 0000000000..a913bfeba3 --- /dev/null +++ b/electron/recording/nativeWindowsWebcams.ts @@ -0,0 +1,333 @@ +/** + * Pure helpers for recording several cameras with the native Windows helper. + * + * Camera 1 stays exactly what it always was: the helper's legacy `webcam*` + * fields, the file `-webcam.mp4` and the session's + * `webcamVideoPath`. Cameras 2-4 ride along in the helper's `webcams` list and + * in the session's `additionalWebcams`. Kept out of `electron/ipc/handlers.ts` + * because that module cannot be loaded from a test. + */ +import path from "node:path"; +import { type AdditionalWebcam, MAX_ADDITIONAL_WEBCAMS } from "../../src/lib/recordingSession"; + +/** `-webcam` (camera 1) or `-webcam-<2..9>` at the end of a file's base name. */ +const WEBCAM_SUFFIX = /-webcam(?:-[2-9])?$/; + +/** Camera 1 → `-webcam.mp4`, camera n ≥ 2 → `-webcam-.mp4`. */ +export function webcamOutputPath( + dir: string, + prefix: string, + recordingId: number, + cameraNumber: number, +): string { + const suffix = cameraNumber <= 1 ? "-webcam" : `-webcam-${cameraNumber}`; + return path.join(dir, `${prefix}${recordingId}${suffix}.mp4`); +} + +/** Whether a file name is one of a recording's camera files (any extension). */ +export function isWebcamSidecarFile(fileName: string): boolean { + return WEBCAM_SUFFIX.test(path.parse(fileName).name); +} + +/** The recording's base name for a camera file's base name; other names pass through. */ +export function stripWebcamSuffix(baseName: string): string { + return baseName.replace(WEBCAM_SUFFIX, ""); +} + +/** One camera in the helper config's `webcams` list (keys are the helper's). */ +export interface HelperWebcamEntry { + camDeviceId: string | null; + camDeviceName: string; + camClsid: string | null; + camWidth: number; + camHeight: number; + camFps: number; + camPath: string; +} + +type DeviceRef = { deviceId?: string; deviceName?: string }; + +/** + * Same device: by id when both sides carry one, otherwise by name — but only + * when neither carries an id. Two webcams of the same model share a name, so an + * id on one side and none on the other says nothing about whether they are the + * same device, and matching by name there would silently drop the second one. + */ +function isSameDevice(a: DeviceRef, b: DeviceRef) { + if (a.deviceId && b.deviceId) { + return a.deviceId === b.deviceId; + } + if (a.deviceId || b.deviceId) { + return false; + } + const name = a.deviceName?.trim(); + return Boolean(name) && name === b.deviceName?.trim(); +} + +/** + * The additional cameras worth asking the helper for: no malformed entry, no + * entry that names no device, none equal to camera 1, no duplicates, and at most + * {@link MAX_ADDITIONAL_WEBCAMS}. + */ +export function dedupeAdditionalWebcams( + camera1: DeviceRef | null, + extras: T[], +): T[] { + const kept: T[] = []; + for (const extra of extras) { + if (kept.length >= MAX_ADDITIONAL_WEBCAMS) { + break; + } + // The list crosses IPC, so a malformed entry is skipped rather than trusted. + if ( + !extra || + typeof extra.deviceName !== "string" || + (extra.deviceId !== undefined && typeof extra.deviceId !== "string") + ) { + continue; + } + if (!extra.deviceId && !extra.deviceName.trim()) { + continue; + } + if (camera1 && isSameDevice(camera1, extra)) { + continue; + } + if (kept.some((other) => isSameDevice(other, extra))) { + continue; + } + kept.push(extra); + } + return kept; +} + +/** + * Labels of the additional cameras, in order: the device name, else + * `Camera ` (n counts camera 1, so the first extra is camera 2). + * + * Two webcams of the same model report the same name, and a label is all the + * user is told about a camera that could not be opened or was dropped. So a + * label already used — by camera 1 or an earlier extra — gets " (2)", " (3)" + * by occurrence. Camera 1's own label is never changed. + */ +export function additionalWebcamLabels( + camera1Name: string | undefined, + extras: Array<{ deviceName: string }>, +): string[] { + const seen = new Map(); + const camera1Label = camera1Name?.trim(); + if (camera1Label) { + seen.set(camera1Label, 1); + } + return extras.map((extra, i) => { + const label = extra.deviceName.trim() || `Camera ${i + 2}`; + const occurrence = (seen.get(label) ?? 0) + 1; + seen.set(label, occurrence); + return occurrence > 1 ? `${label} (${occurrence})` : label; + }); +} + +/** + * The camera part of the helper config: the unchanged legacy `webcam*` fields + * for camera 1 plus the `webcams` list (camera 1 first, then the extras). + * + * Extras are recorded only while camera 1 is on (the HUD's camera toggle + * governs every camera), so with camera 1 off the list is empty. Every extra + * uses camera 1's requested size and rate: the quality setting is one for all. + */ +export function buildHelperWebcamConfig(input: { + camera1: { + enabled: boolean; + deviceId?: string; + deviceName?: string; + width: number; + height: number; + fps: number; + }; + camera1Clsid: string | null; + camera1Path: string; + extras: Array<{ deviceId?: string; deviceName: string; clsid: string | null; path: string }>; +}) { + const { camera1 } = input; + const entry = ( + device: { deviceId?: string; deviceName?: string }, + clsid: string | null, + camPath: string, + ): HelperWebcamEntry => ({ + camDeviceId: device.deviceId ?? null, + camDeviceName: device.deviceName ?? "", + camClsid: clsid, + camWidth: camera1.width, + camHeight: camera1.height, + camFps: camera1.fps, + camPath, + }); + const webcams: HelperWebcamEntry[] = camera1.enabled + ? [ + entry(camera1, input.camera1Clsid, input.camera1Path), + ...input.extras.map((extra) => entry(extra, extra.clsid, extra.path)), + ] + : []; + return { + webcamEnabled: camera1.enabled, + webcamDeviceId: camera1.deviceId ?? null, + webcamDeviceName: camera1.deviceName ?? null, + webcamDirectShowClsid: input.camera1Clsid, + webcamWidth: camera1.width, + webcamHeight: camera1.height, + webcamFps: camera1.fps, + webcams, + }; +} + +/** + * Labels of the additional cameras the helper reported unavailable. The + * helper's index is the position in the `webcams` list, which is the position + * in `requested` (camera 1 first). Index 0 is camera 1 and is left out: the + * caller reports camera 1 through its own `webcamUnavailable` flag. The labels + * come from our own list because the helper's `deviceName` can be empty. + */ +export function labelsOfUnavailableAdditionalWebcams( + requested: Array<{ label: string }>, + indices: number[], +): string[] { + const labels: string[] = []; + for (const index of new Set(indices)) { + const camera = index > 0 ? requested[index] : undefined; + if (camera) { + labels.push(camera.label); + } + } + return labels; +} + +/** + * Which requested cameras made it into the take. + * + * A camera is kept when its file exists with size > 0. The helper's own list + * at stop is deliberately not consulted: it names the cameras still recording + * at that moment, so a camera that died mid-take with a playable partial file + * would be missing from it, and an old helper names only camera 1. A file that + * never appeared is simply absent from `sizes`. + * + * `camera1Enabled` says whether `requested[0]` is camera 1. Camera 1 comes back + * as `camera1` (undefined when lost) and is never in `dropped`, which names + * only the additional cameras not kept — the caller reports camera 1 through + * its own `webcamDropped` flag. + */ +export function collectStoppedWebcams(input: { + camera1Enabled: boolean; + requested: Array<{ path: string; label: string }>; + sizes: Map; +}): { camera1?: string; additional: AdditionalWebcam[]; dropped: string[] } { + const isKept = (filePath: string) => (input.sizes.get(filePath) ?? 0) > 0; + const [first, ...rest] = input.requested; + const camera1 = input.camera1Enabled && first && isKept(first.path) ? first.path : undefined; + const extras = input.camera1Enabled ? rest : input.requested; + const additional: AdditionalWebcam[] = []; + const dropped: string[] = []; + for (const camera of extras) { + if (isKept(camera.path)) { + additional.push({ path: camera.path, label: camera.label }); + } else { + dropped.push(camera.label); + } + } + return { ...(camera1 ? { camera1 } : {}), additional, dropped }; +} + +/** A path compared the way Windows does: case-insensitive, either separator. */ +function comparablePath(filePath: string) { + return filePath.replace(/[\\/]+/g, "\\").toLowerCase(); +} + +/** + * Labels of the cameras that stopped early: their file was kept (size > 0, the + * same test as {@link collectStoppedWebcams}) but the helper did not list it in + * `recording-stopped.webcamPaths`, the cameras still recording at stop. That is + * a camera the helper disabled mid-take; its partial file stays in the take, + * and the user is told which camera it was. + * + * `helperWebcamPaths` is null when the event carried no `webcamPaths` key (an + * old helper, or no event at all) — then nothing is known and nothing is + * flagged. Camera 1 counts like any other, so its entry needs a real label. + */ +export function labelsOfWebcamsStoppedEarly(input: { + requested: Array<{ path: string; label: string }>; + sizes: Map; + helperWebcamPaths: string[] | null; +}): string[] { + if (!input.helperWebcamPaths) { + return []; + } + const stillRecording = new Set(input.helperWebcamPaths.map(comparablePath)); + return input.requested + .filter( + (camera) => + (input.sizes.get(camera.path) ?? 0) > 0 && !stillRecording.has(comparablePath(camera.path)), + ) + .map((camera) => camera.label); +} + +/** + * The file the desk camera of a native take records into, or null. `deskCamera` is the + * request's index (0 camera 1, k the k-th requested extra) and crosses IPC, so anything that is + * not an index is ignored. `kept` holds the requested extras that survived + * {@link dedupeAdditionalWebcams} (the same objects), `keptPaths` their files in that order. + */ +export function deskWebcamPath(input: { + deskCamera: unknown; + camera1Path: string | null; + requested: readonly T[]; + kept: readonly T[]; + keptPaths: readonly string[]; +}): string | null { + const { deskCamera } = input; + if (typeof deskCamera !== "number" || !Number.isInteger(deskCamera) || deskCamera < 0) { + return null; + } + if (deskCamera === 0) { + return input.camera1Path; + } + const extra = input.requested[deskCamera - 1]; + const index = extra === undefined ? -1 : input.kept.indexOf(extra); + return index >= 0 ? (input.keptPaths[index] ?? null) : null; +} + +/** + * The session's `deskCamera` once a take has stopped: the desk camera's index among the cameras + * that were actually kept (0 camera 1, k the k-th of `additional`). Undefined when its file was + * lost, or when camera 1 was, since every index is relative to it. + */ +export function recordedDeskCamera( + deskPath: string | null, + camera1Path: string | undefined, + additional: readonly AdditionalWebcam[], +): number | undefined { + if (!deskPath || !camera1Path) { + return undefined; + } + if (deskPath === camera1Path) { + return 0; + } + const index = additional.findIndex((camera) => camera.path === deskPath); + return index >= 0 ? index + 1 : undefined; +} + +/** + * A restored session's `deskCamera` after some of its cameras were dropped. `keptExtraIndices` + * are the positions, in the old `additionalWebcams`, of the extras that were kept, in order. + */ +export function remapDeskCamera( + deskCamera: number | undefined, + camera1Kept: boolean, + keptExtraIndices: readonly number[], +): number | undefined { + if (deskCamera === undefined || !camera1Kept) { + return undefined; + } + if (deskCamera === 0) { + return 0; + } + const index = keptExtraIndices.indexOf(deskCamera - 1); + return index >= 0 ? index + 1 : undefined; +} diff --git a/electron/stt/ctcAlign.test.ts b/electron/stt/ctcAlign.test.ts new file mode 100644 index 0000000000..579f97a37c --- /dev/null +++ b/electron/stt/ctcAlign.test.ts @@ -0,0 +1,239 @@ +import { describe, expect, it } from "vitest"; +import { + type AlignableWord, + alignWordsOnEmissions, + type CtcEmissions, + ctcViterbi, + END_OFFSET_SEC, + emissionRegions, + PAUSE_END_OFFSET_SEC, + PAUSE_START_OFFSET_SEC, + parseEmissions, + START_OFFSET_SEC, + wordTokens, +} from "./ctcAlign"; + +const VOCAB = ["", "|", "a", "b", "c", "é", "'"]; +const index = new Map(VOCAB.map((t, i) => [t, i])); +const id = (t: string) => index.get(t) as number; + +/** Log-probs where frame t is sure of `tokens[t]` (a vocab entry, "" for blank). */ +function logprobs(tokens: string[]): Float32Array { + const V = VOCAB.length; + const out = new Float32Array(tokens.length * V).fill(-12); + tokens.forEach((t, f) => { + out[f * V + (t ? id(t) : 0)] = -0.01; + }); + return out; +} + +const STRIDE = 0.02; +const emissions = (tokens: string[], startSec = 0): CtcEmissions => ({ + vocab: VOCAB, + blank: 0, + strideSec: STRIDE, + regions: [{ startSec, frames: tokens.length, logprobs: logprobs(tokens) }], +}); +/** `n` blank frames. */ +const gap = (n: number) => Array(n).fill(""); +const word = ( + text: string, + startSec: number, + endSec: number, + anchorSec = startSec, +): AlignableWord => ({ + word: text, + startSec, + endSec, + anchorSec, +}); +const r = (x: number) => Math.round(x * 1000) / 1000; + +describe("wordTokens", () => { + it("spells a word in the vocabulary's case, dropping silent punctuation", () => { + const upper = new Map(["", "|", "A", "B", "C", "'"].map((t, i) => [t, i])); + expect(wordTokens("Cab,", upper)).toEqual([4, 2, 3]); + expect(wordTokens("ab’c", upper)).toEqual([2, 3, 5, 4]); + expect(wordTokens("cab", index)).toEqual([id("c"), id("a"), id("b")]); + }); + + it("keeps an accented letter the vocabulary has and strips the accent otherwise", () => { + expect(wordTokens("É", index)).toEqual([id("é")]); + expect(wordTokens("àbc", index)).toEqual([id("a"), id("b"), id("c")]); + }); + + it("gives nothing to say for punctuation, and no spelling for digits and symbols", () => { + expect(wordTokens("?", index)).toEqual([]); + expect(wordTokens("...", index)).toEqual([]); + expect(wordTokens("1080p", index)).toBeNull(); + expect(wordTokens("€", index)).toBeNull(); + expect(wordTokens("日本", index)).toBeNull(); + }); +}); + +describe("ctcViterbi", () => { + it("finds the frames each label is spoken on", () => { + const tokens = ["", "a", "a", "", "b", ""]; + expect(ctcViterbi(logprobs(tokens), 6, VOCAB.length, 0, [id("a"), id("b")])).toEqual([ + [1, 2], + [4, 4], + ]); + }); + + it("needs a blank between two repeats of a label, and fails when the frames are too few", () => { + const aba = logprobs(["a", "", "a"]); + expect(ctcViterbi(aba, 3, VOCAB.length, 0, [id("a"), id("a")])).toEqual([ + [0, 0], + [2, 2], + ]); + expect(ctcViterbi(logprobs(["a", "a"]), 2, VOCAB.length, 0, [id("a"), id("a")])).toBeNull(); + }); + + it("scores a wildcard as whatever is spoken", () => { + const spans = ctcViterbi(logprobs(["a", "", "c", "c", "", "b"]), 6, VOCAB.length, 0, [ + id("a"), + -1, + id("b"), + ]); + expect(spans?.[1]).toEqual([2, 3]); + }); +}); + +describe("emissionRegions", () => { + it("pads each stretch of speech and merges the ones that touch", () => { + expect( + emissionRegions( + [ + { startSec: 0.1, endSec: 1 }, + { startSec: 1.4, endSec: 2 }, + { startSec: 5, endSec: 6 }, + ], + 6.1, + ).map(([a, b]) => [r(a), r(b)]), + ).toEqual([ + [0, 2.3], + [4.7, 6.1], + ]); + }); +}); + +describe("alignWordsOnEmissions", () => { + const speech = [{ startSec: 0.2, endSec: 1 }]; + + it("times words on their letters, calibrated outward, meeting mid-way inside speech", () => { + // "ab" on frames 20-21, the delimiter on 22, "c" on 24: continuous speech. + const tokens = [...gap(20), "a", "b", "|", "", "c", ...gap(40)]; + const out = alignWordsOnEmissions( + [word("ab", 0.3, 0.5, 0.45), word("c", 0.5, 0.7, 0.6)], + speech, + emissions(tokens), + ); + const abEnd = 22 * STRIDE + END_OFFSET_SEC; + const cStart = 24 * STRIDE + START_OFFSET_SEC; + const boundary = abEnd > cStart ? (abEnd + cStart) / 2 : null; + expect(r(out[0].startSec)).toBe(r(20 * STRIDE + PAUSE_START_OFFSET_SEC)); + expect(r(out[0].endSec)).toBe(r(boundary ?? abEnd)); + expect(r(out[1].startSec)).toBe(r(boundary ?? cStart)); + expect(r(out[1].endSec)).toBe(r(25 * STRIDE + PAUSE_END_OFFSET_SEC)); + expect(out[0].anchorSec).toBe(0.45); + }); + + it("keeps a pause between two words as a pause", () => { + // "a" on frame 12, "c" on frame 40: 0.54 s apart. + const tokens = [...gap(12), "a", ...gap(27), "c", ...gap(40)]; + const out = alignWordsOnEmissions( + [word("a", 0.2, 0.4, 0.3), word("c", 0.4, 0.9, 0.85)], + speech, + emissions(tokens), + ); + expect(r(out[0].endSec)).toBe(r(13 * STRIDE + PAUSE_END_OFFSET_SEC)); + expect(r(out[1].startSec)).toBe(r(40 * STRIDE + PAUSE_START_OFFSET_SEC)); + }); + + it("gives a word it cannot spell the speech between its neighbours", () => { + const tokens = [...gap(15), "a", "|", "c", "b", "c", "|", "b", ...gap(40)]; + const out = alignWordsOnEmissions( + [word("a", 0.3, 0.32, 0.31), word("42", 0.32, 0.4, 0.35), word("b", 0.4, 0.5, 0.42)], + speech, + emissions(tokens), + ); + expect(out[1].startSec).toBeGreaterThanOrEqual(out[0].endSec); + expect(out[1].endSec).toBeLessThanOrEqual(out[2].startSec); + expect(r(out[1].startSec)).toBeLessThan(0.34); + expect(r(out[2].startSec)).toBeLessThan(0.42); + }); + + it("puts punctuation on the end of the word before it", () => { + const tokens = [...gap(15), "a", "b", ...gap(40)]; + const out = alignWordsOnEmissions( + [word("ab", 0.3, 0.5, 0.35), word("!", 0.5, 0.6, 0.55)], + speech, + emissions(tokens), + ); + expect(out[1].startSec).toBe(out[0].endSec); + expect(out[1].endSec).toBe(out[0].endSec); + }); + + it("aligns a stretch that runs to the end of the upload, past the last frame", () => { + // 1 s of audio gives 49 frames (0.98 s): the stretch ends after them. + const tokens = [...gap(30), "a", "b", ...gap(17)]; + const out = alignWordsOnEmissions( + [word("ab", 0.65, 1, 0.7)], + [{ startSec: 0.5, endSec: 1 }], + emissions(tokens), + ); + expect(r(out[0].startSec)).toBe(r(30 * STRIDE + PAUSE_START_OFFSET_SEC)); + }); + + it("aligns a sentence's last word in the stretch it closes, even anchored in the pause", () => { + // "b." is said on frame 25 (0.5 s), but DTW anchored it at 0.75 s, past the + // first stretch's tail (issue #948). It is aligned there, not in "c"'s audio. + const tokens = [...gap(15), "a", "|", ...gap(8), "b", ...gap(49), "c", ...gap(24)]; + const out = alignWordsOnEmissions( + [word("a", 0.25, 0.35, 0.3), word("b.", 0.35, 0.75, 0.75), word("c", 0.75, 1.6, 1.55)], + [ + { startSec: 0.2, endSec: 0.6 }, + { startSec: 1.4, endSec: 1.8 }, + ], + emissions(tokens), + ); + expect(r(out[1].startSec)).toBe(r(25 * STRIDE + PAUSE_START_OFFSET_SEC)); + expect(r(out[2].startSec)).toBe(r(75 * STRIDE + PAUSE_START_OFFSET_SEC)); + }); + + it("keeps the helper's times where it cannot align", () => { + const words = [word("ab", 0.3, 0.5, 0.35), word("c", 1.5, 1.7, 1.6)]; + // The second word is outside every stretch; the first one's letters do not fit. + const out = alignWordsOnEmissions(words, speech, emissions(gap(2))); + expect(out).toEqual(words); + // No region covers the stretch at all. + expect(alignWordsOnEmissions(words, speech, emissions(gap(60), 5))).toEqual(words); + }); +}); + +describe("parseEmissions", () => { + const floats = new Float32Array([-0.1, -2, -3, -4, -5, -6, -7]); + const logprobs = Buffer.from(floats.buffer).toString("base64"); + + it("decodes the helper's base64 log-probs", () => { + const parsed = parseEmissions({ + vocab: VOCAB, + blank: 0, + stride_s: 0.02, + regions: [{ start: 1.5, frames: 1, logprobs }], + }); + expect(parsed?.regions[0].startSec).toBe(1.5); + expect([...(parsed?.regions[0].logprobs ?? [])]).toEqual([...floats]); + }); + + it("refuses a reply whose sizes do not add up", () => { + expect( + parseEmissions({ + vocab: VOCAB, + blank: 0, + stride_s: 0.02, + regions: [{ start: 0, frames: 2, logprobs }], + }), + ).toBeNull(); + }); +}); diff --git a/electron/stt/ctcAlign.ts b/electron/stt/ctcAlign.ts new file mode 100644 index 0000000000..4e142671ec --- /dev/null +++ b/electron/stt/ctcAlign.ts @@ -0,0 +1,294 @@ +// Second alignment pass: re-times whisper's words on a CTC acoustic model +// (issue #948, phase 3). +// +// whisper's DTW times words within ~30 ms median inside continuous speech, but +// its 20 ms frames and diffuse cross-attention cap it there (P90 ~110 ms). A +// wav2vec2 model fine-tuned for CTC scores every 20 ms frame against every +// letter, so a Viterbi pass that forces whisper's own words through those +// scores puts each letter where the audio has it. The helper computes the +// scores (`/emissions`, electron/native/whisper-stt/src/ctc_aligner.cpp); this +// file spells whisper's words in the model's letters, aligns them, and maps the +// letters back to word times. +// +// Words the model cannot spell (digits, symbols, another script) become one +// wildcard token that matches any speech, so they hold their place and their +// neighbours stay aligned. A stretch the model cannot fit keeps the times it +// came with, and so does every word when there is no aligner for the language. +// +// Self-contained on purpose: tools/stt-eval/word-timing imports this file +// straight into Node, which resolves no extensionless import. + +import { ownWords } from "./snapWordBoundaries.ts"; +import type { SttVadSegment } from "./transcriptionContract"; + +/** What `/emissions` answers, decoded. */ +export interface CtcEmissions { + vocab: string[]; + blank: number; + /** Seconds between two frames. */ + strideSec: number; + /** Per requested region: its first frame's time and `frames * vocab` log-probs. */ + regions: Array<{ startSec: number; frames: number; logprobs: Float32Array }>; +} + +/** The fields of a helper word this pass reads and rewrites. */ +export interface AlignableWord { + word: string; + startSec: number; + endSec: number; + /** Always inside the word; decides which stretch of speech owns it. */ + anchorSec: number; +} + +// Calibration, fitted on tools/stt-eval/word-timing (TTS and LibriSpeech agree). +// The model is sure of a letter a little after the sound starts and before it +// ends, so CTC shrinks words: the edges move outward. Inside continuous speech +// the word boundary lies between the last letter of one word and the first of +// the next; where the two estimates cross, the boundary is their midpoint. +/** Added to the frame of a word's first letter. */ +export const START_OFFSET_SEC = -0.045; +/** Added to the end of the frame of a word's last letter. */ +export const END_OFFSET_SEC = 0.025; +/** A gap between two words' letters this long is a pause, not a boundary. */ +export const PAUSE_SEC = 0.1; +/** After a pause, a word's first letter trails its onset further (breath, closure, soft attack). */ +export const PAUSE_START_OFFSET_SEC = -0.06; +/** Before a pause, a word decays past its last letter; early there is audible, late is silence. */ +export const PAUSE_END_OFFSET_SEC = 0.15; + +/** Audio kept on each side of a stretch of speech, so its first and last letters have context. */ +export const REGION_MARGIN_SEC = 0.3; + +/** As in snapWordBoundaries.ts: the audio the helper keeps past each speech offset. */ +const TAIL_SEC = 0.1; + +const WILDCARD = -1; + +/** Seconds of the upload the helper should score: the speech plus a margin, merged. */ +export function emissionRegions( + speech: SttVadSegment[], + durationSec: number, +): Array<[number, number]> { + const out: Array<[number, number]> = []; + for (const s of speech) { + const a = Math.max(0, s.startSec - REGION_MARGIN_SEC); + const b = Math.min(durationSec, s.endSec + REGION_MARGIN_SEC); + const last = out[out.length - 1]; + if (last && a <= last[1]) last[1] = Math.max(last[1], b); + else if (b > a) out.push([a, b]); + } + return out; +} + +/** + * A word as the model's token ids: `[]` when there is nothing to say + * (punctuation), `null` when a letter is not in the vocabulary (digits, symbols, + * another script). Case follows the vocabulary, an accented letter the + * vocabulary lacks falls back to its base letter, and the apostrophe is + * normalised. + */ +export function wordTokens(word: string, index: Map): number[] | null { + const upper = index.has("A") && !index.has("a"); + let text = word.normalize("NFC").replace(/[’‘`´]/g, "'"); + text = upper ? text.toUpperCase() : text.toLowerCase(); + const out: number[] = []; + for (const ch of text) { + const id = index.get(ch); + if (id !== undefined) { + out.push(id); + continue; + } + if (/\p{S}/u.test(ch)) return null; // "%", "+", "€": said aloud, but not spelled so + if (/[\p{P}\s]/u.test(ch)) continue; // silent, apostrophes and hyphens included + const base = [...ch.normalize("NFD").replace(/\p{M}/gu, "")]; + const ids = base.map((c) => index.get(c)); + if (ids.length === 0 || ids.some((x) => x === undefined)) return null; + out.push(...(ids as number[])); + } + return out; +} + +/** + * CTC Viterbi forced alignment of `labels` over `T` frames of `V` log-probs. + * `WILDCARD` scores as the frame's best non-blank token. Returns, per label, the + * first and last frame the best path spends on it, or null when the labels + * cannot fit in the frames. + */ +export function ctcViterbi( + logprobs: Float32Array, + T: number, + V: number, + blank: number, + labels: number[], +): Array<[number, number]> | null { + const L = labels.length; + const S = 2 * L + 1; + if (L === 0 || T === 0) return null; + const wild = new Float32Array(T); + if (labels.includes(WILDCARD)) { + for (let t = 0; t < T; t++) { + let best = Number.NEGATIVE_INFINITY; + for (let v = 0; v < V; v++) if (v !== blank) best = Math.max(best, logprobs[t * V + v]); + wild[t] = best; + } + } + const lab = (s: number) => (s % 2 === 0 ? blank : labels[(s - 1) >> 1]); + const em = (t: number, s: number) => { + const l = lab(s); + return l === WILDCARD ? wild[t] : logprobs[t * V + l]; + }; + const NEG = Number.NEGATIVE_INFINITY; + let prev = new Float64Array(S).fill(NEG); + let cur = new Float64Array(S); + const back = new Uint8Array(T * S); // how far back the best predecessor is: 0, 1 or 2 + prev[0] = em(0, 0); + prev[1] = em(0, 1); + for (let t = 1; t < T; t++) { + for (let s = 0; s < S; s++) { + let best = prev[s]; + let from = 0; + if (s >= 1 && prev[s - 1] > best) { + best = prev[s - 1]; + from = 1; + } + // A label may follow the one before the blank directly, unless it repeats it. + if (s >= 2 && s % 2 === 1 && lab(s) !== lab(s - 2) && prev[s - 2] > best) { + best = prev[s - 2]; + from = 2; + } + cur[s] = best === NEG ? NEG : best + em(t, s); + back[t * S + s] = from; + } + [prev, cur] = [cur, prev]; + } + let s = prev[S - 2] > prev[S - 1] ? S - 2 : S - 1; + if (prev[s] === NEG) return null; + const spans: Array<[number, number]> = labels.map(() => [-1, -1]); + for (let t = T - 1; t >= 0; t--) { + if (s % 2 === 1) { + const k = (s - 1) >> 1; + spans[k][0] = t; + if (spans[k][1] < 0) spans[k][1] = t; + } + s -= back[t * S + s]; + } + return spans; +} + +/** + * Re-time `words` on `emissions`. Each speech stretch owns the words + * `ownWords` (snapWordBoundaries.ts) gives it; they are aligned on the frames + * between the neighbouring stretches, with the vocabulary's word delimiter + * between words when it has one. Words outside every stretch, stretches the + * model cannot fit, and a vocabulary with no word in it keep the helper's + * times. Punctuation collapses onto the end of the word before it. + */ +export function alignWordsOnEmissions( + words: W[], + speech: SttVadSegment[], + emissions: CtcEmissions, +): W[] { + const out = words.map((w) => ({ ...w })); + const index = new Map(emissions.vocab.map((t, i) => [t, i])); + const delimiter = index.get("|"); + const V = emissions.vocab.length; + const { strideSec } = emissions; + const ranges = ownWords(words, speech); + for (let i = 0; i < speech.length; i++) { + const { startSec: onset, endSec: offset } = speech[i]; + const [from, to] = ranges[i]; + const owned = Array.from({ length: to - from }, (_, n) => from + n); + // A region's frames stop up to one receptive field (25 ms) short of the audio + // it was cut from, so a stretch that runs to the end of the upload ends + // past them; two frames of slack keep it. `hi` below stays on the frames. + const region = emissions.regions.find( + (r) => r.startSec <= onset && r.startSec + (r.frames + 2) * strideSec >= offset, + ); + if (!region || owned.length === 0) continue; + // The frames of this stretch: its margin, stopping at the neighbours' speech. + const lo = Math.max( + region.startSec, + (speech[i - 1]?.endSec ?? Number.NEGATIVE_INFINITY) + TAIL_SEC, + onset - REGION_MARGIN_SEC, + ); + const hi = Math.min( + region.startSec + region.frames * strideSec, + speech[i + 1]?.startSec ?? Number.POSITIVE_INFINITY, + offset + REGION_MARGIN_SEC, + ); + const f0 = Math.max(0, Math.ceil((lo - region.startSec) / strideSec)); + const f1 = Math.min(region.frames, Math.floor((hi - region.startSec) / strideSec)); + if (f1 <= f0) continue; + + const labels: number[] = []; + const timed: Array<{ j: number; first: number; last: number }> = []; + for (const j of owned) { + const toks = wordTokens(out[j].word, index); + if (toks?.length === 0) continue; + if (labels.length > 0 && delimiter !== undefined) labels.push(delimiter); + timed.push({ j, first: labels.length, last: labels.length + (toks?.length ?? 1) - 1 }); + labels.push(...(toks ?? [WILDCARD])); + } + const spans = ctcViterbi( + region.logprobs.subarray(f0 * V, f1 * V), + f1 - f0, + V, + emissions.blank, + labels, + ); + if (!spans) continue; + const at = (f: number) => region.startSec + (f0 + f) * strideSec; + const s = timed.map((w) => at(spans[w.first][0])); + const e = timed.map((w) => at(spans[w.last][1] + 1)); + timed.forEach(({ j }, q) => { + const prevEnd = e[q - 1] ?? Number.NEGATIVE_INFINITY; + const nextStart = s[q + 1] ?? Number.POSITIVE_INFINITY; + out[j].startSec = + s[q] - prevEnd >= PAUSE_SEC + ? Math.max(s[q] + PAUSE_START_OFFSET_SEC, (prevEnd + s[q]) / 2) + : s[q] + START_OFFSET_SEC; + out[j].endSec = + nextStart - e[q] >= PAUSE_SEC + ? Math.min(e[q] + PAUSE_END_OFFSET_SEC, (e[q] + nextStart) / 2) + : e[q] + END_OFFSET_SEC; + const before = q > 0 ? out[timed[q - 1].j] : null; + if (before && before.endSec > out[j].startSec) { + const mid = (before.endSec + out[j].startSec) / 2; + before.endSec = mid; + out[j].startSec = mid; + } + }); + for (let n = 1; n < owned.length; n++) { + const j = owned[n]; + if (timed.some((w) => w.j === j)) continue; + out[j].startSec = out[owned[n - 1]].endSec; + out[j].endSec = out[j].startSec; + } + } + return out; +} + +/** The helper's `/emissions` JSON. */ +export interface EmissionsJson { + vocab: string[]; + blank: number; + stride_s: number; + regions: Array<{ start: number; frames: number; logprobs: string }>; + /** Seconds the helper spent, model load included. */ + elapsed_s?: number; +} + +/** Decodes the helper's `/emissions` JSON; null when it is not one. */ +export function parseEmissions(json: EmissionsJson): CtcEmissions | null { + if (!Array.isArray(json?.vocab) || !Array.isArray(json.regions) || !(json.stride_s > 0)) + return null; + const V = json.vocab.length; + const regions: CtcEmissions["regions"] = []; + for (const r of json.regions) { + // Copied: a pooled Buffer can start on an offset Float32Array refuses. + const bytes = Uint8Array.from(Buffer.from(r.logprobs, "base64")); + if (bytes.byteLength !== r.frames * V * 4) return null; + regions.push({ startSec: r.start, frames: r.frames, logprobs: new Float32Array(bytes.buffer) }); + } + return { vocab: json.vocab, blank: json.blank, strideSec: json.stride_s, regions }; +} diff --git a/electron/stt/extractAudio.test.ts b/electron/stt/extractAudio.test.ts index 24e9694c86..c437b1ae9e 100644 --- a/electron/stt/extractAudio.test.ts +++ b/electron/stt/extractAudio.test.ts @@ -8,10 +8,11 @@ const resolveFfmpegMock = vi.fn<() => string | null>(); vi.mock("node:child_process", () => ({ spawn: (...args: unknown[]) => spawnMock(...args) })); vi.mock("../media/audioPeaks", () => ({ resolveFfmpeg: () => resolveFfmpegMock() })); -const { extractMono16kPcm, FfmpegUnavailableError, NoAudioTrackError } = await import( - "./extractAudio" +const { extractMono16kPcm, FfmpegUnavailableError, MediaUnreadableError, NoAudioTrackError } = + await import("./extractAudio"); +const { STT_MEDIA_UNREADABLE, STT_NATIVE_EXTRACTION_UNAVAILABLE } = await import( + "./transcriptionContract" ); -const { STT_NATIVE_EXTRACTION_UNAVAILABLE } = await import("./transcriptionContract"); /** A stand-in for the ffmpeg child: two pipes and a close event, nothing more. */ function fakeChild() { @@ -95,6 +96,60 @@ describe("extractMono16kPcm", () => { await expect(promise).rejects.toBeInstanceOf(NoAudioTrackError); }); + // Issue #968: the renderer explains these as "check folder access" and retries + // them, so both must carry the marker, which is all that survives the IPC. + it("reports a read the OS refused as unreadable media, not as no audio", async () => { + const child = fakeChild(); + spawnMock.mockReturnValue(child); + const promise = extractMono16kPcm("/Users/me/Downloads/a.mp4"); + child.stderr.end("/Users/me/Downloads/a.mp4: Operation not permitted"); + child.stdout.end(); + child.emit("close", 1); + + await expect(promise).rejects.toBeInstanceOf(MediaUnreadableError); + await expect(promise).rejects.toThrow(STT_MEDIA_UNREADABLE); + }); + + it("reports an ffmpeg that hangs opening the file as unreadable media", async () => { + vi.useFakeTimers(); + try { + const child = fakeChild(); + spawnMock.mockReturnValue(child); + const promise = extractMono16kPcm("/Users/me/Downloads/a.mp4"); + const settled = expect(promise).rejects.toThrow(STT_MEDIA_UNREADABLE); + await vi.advanceTimersByTimeAsync(60_000); + await settled; + expect(child.kill).toHaveBeenCalled(); + } finally { + vi.useRealTimers(); + } + }); + + it("reports a timeout AFTER audio came out as a plain stall, not unreadable media", async () => { + // The file was readable; pointing the user at folder access would be wrong. + vi.useFakeTimers(); + try { + const child = fakeChild(); + spawnMock.mockReturnValue(child); + const promise = extractMono16kPcm("/Users/me/Downloads/a.mp4"); + const settled = promise.then( + () => { + throw new Error("expected a rejection"); + }, + (error: Error) => error, + ); + child.stdout.write(f32le([0.5])); + await vi.advanceTimersByTimeAsync(60_000); + const error = await settled; + expect(error).not.toBeInstanceOf(MediaUnreadableError); + expect(error.message).not.toContain(STT_MEDIA_UNREADABLE); + expect(error.message).toBe("ffmpeg timed out after 60000ms on /Users/me/Downloads/a.mp4"); + expect(child.kill).toHaveBeenCalledWith("SIGKILL"); + } finally { + vi.useRealTimers(); + } + }); + it("keeps the samples when ffmpeg exits non-zero AFTER writing audio", async () => { // A truncated file still yields usable audio; throwing it away would lose a // transcript over a trailing byte. diff --git a/electron/stt/extractAudio.ts b/electron/stt/extractAudio.ts index 08168d21e4..28dd9f177d 100644 --- a/electron/stt/extractAudio.ts +++ b/electron/stt/extractAudio.ts @@ -21,7 +21,7 @@ import { spawn } from "node:child_process"; import { resolveFfmpeg } from "../media/audioPeaks"; -import { STT_NATIVE_EXTRACTION_UNAVAILABLE } from "./transcriptionContract"; +import { STT_MEDIA_UNREADABLE, STT_NATIVE_EXTRACTION_UNAVAILABLE } from "./transcriptionContract"; /** What whisper.cpp wants, and what `decodePeaks` already asks ffmpeg for. */ const SAMPLE_RATE = 16_000; @@ -50,6 +50,22 @@ export class NoAudioTrackError extends Error { } } +/** + * The media file could not be read at all: the OS refused it, or ffmpeg hung + * opening it. Unlike `NoAudioTrackError` this is NOT permanent: it clears once + * the user grants access, so it carries `STT_MEDIA_UNREADABLE` for the renderer + * to explain rather than remember. + */ +export class MediaUnreadableError extends Error { + constructor(filePath: string, detail: string) { + super(`${STT_MEDIA_UNREADABLE}: cannot read ${filePath}: ${detail}`); + this.name = "MediaUnreadableError"; + } +} + +/** What ffmpeg's stderr says when the OS refused to open the input. */ +const ACCESS_DENIED = /operation not permitted|permission denied/i; + /** * Decode `filePath` to mono 16 kHz float samples. * @@ -105,8 +121,16 @@ export async function extractMono16kPcm( const timer = setTimeout(() => { child.kill("SIGKILL"); + // Only a hang before the first byte says the file could not be opened (a + // pending macOS folder-access prompt produces exactly that, issue #968). Once + // audio has come out the media was readable, and the stall is ffmpeg's. + const nothingRead = total === 0 && carry === null; finish(() => - reject(new Error(`ffmpeg timed out after ${EXTRACT_TIMEOUT_MS}ms on ${filePath}`)), + reject( + nothingRead + ? new MediaUnreadableError(filePath, `ffmpeg timed out after ${EXTRACT_TIMEOUT_MS}ms`) + : new Error(`ffmpeg timed out after ${EXTRACT_TIMEOUT_MS}ms on ${filePath}`), + ), ); }, EXTRACT_TIMEOUT_MS); @@ -137,7 +161,13 @@ export async function extractMono16kPcm( // caller treats both the same way — this asset will not transcribe — so they // share an error type; `stderr` carries which it was. if (code !== 0 && total === 0) { - finish(() => reject(new NoAudioTrackError(filePath, stderr.trim()))); + // Except a read the OS refused: that also exits non-zero with nothing + // decoded, but remembering it as "no audio" would never retry the file + // once access is granted. + const error = ACCESS_DENIED.test(stderr) + ? new MediaUnreadableError(filePath, stderr.trim()) + : new NoAudioTrackError(filePath, stderr.trim()); + finish(() => reject(error)); return; } const out = new Float32Array(total); diff --git a/electron/stt/index.test.ts b/electron/stt/index.test.ts index 69ef1723b4..c5af87fe9f 100644 --- a/electron/stt/index.test.ts +++ b/electron/stt/index.test.ts @@ -34,6 +34,9 @@ vi.mock("./whisperServer", () => { vi.mock("./modelManager", () => ({ ensureModels: vi.fn(async () => undefined), + ensureAligner: vi.fn(async ({ language }: { language: string }) => `/fake/${language}.gguf`), + cachedAligner: vi.fn(async () => null), + alignerPath: vi.fn((base: string, language: string): string | null => `${base}/${language}.gguf`), modelPaths: (base: string) => ({ whisper: `${base}/whisper-ggml/ggml-small-q8_0.bin`, }), @@ -470,6 +473,110 @@ describe("SttManager", () => { expect(fakeWhisperServer.start).toHaveBeenCalledOnce(); }); + describe("word aligner", () => { + /** Chunks report what `alignerFor("fr")` gave them, as the helper would use it. */ + function recordAligner() { + const asked: Array = []; + fakeWhisperServer.transcribe.mockImplementation( + async (opts: { alignerFor?: (language: string) => Promise }) => { + asked.push((await opts.alignerFor?.("fr")) ?? null); + return { + segments: [], + wordSegments: [], + detectedLanguage: "fr", + backend: "whispercpp-cpu" as const, + }; + }, + ); + return asked; + } + const long = new Float32Array(16_000 * 400); + + async function mocks() { + const mm = await import("./modelManager"); + const ensure = vi.mocked(mm.ensureAligner); + const cached = vi.mocked(mm.cachedAligner); + ensure.mockReset(); + cached.mockReset(); + cached.mockResolvedValue(null); + return { ensure, cached }; + } + + it("never holds a chunk on the download, and uses the aligner once it lands", async () => { + const { ensure } = await mocks(); + let land: (file: string) => void = () => undefined; + ensure.mockImplementation(() => new Promise((resolve) => (land = resolve))); + const asked = recordAligner(); + const mgr = new SttManager(); + await mgr.init({ modelsBaseDir: "/tmp/fake-stt-models" }); + + // The download never finishes during this run: every chunk goes on without it. + await mgr.transcribe({ samples: long, language: "fr" }); + expect(asked.length).toBeGreaterThan(1); + expect(asked.every((x) => x === null)).toBe(true); + expect(ensure).toHaveBeenCalledOnce(); + + land("/fake/fr.gguf"); + await new Promise((resolve) => setTimeout(resolve, 0)); + asked.length = 0; + await mgr.transcribe({ samples: new Float32Array(16_000), language: "fr" }); + expect(asked).toEqual(["/fake/fr.gguf"]); + expect(ensure).toHaveBeenCalledOnce(); + }); + + it("uses a verified copy already on disk without downloading", async () => { + const { ensure, cached } = await mocks(); + cached.mockResolvedValue("/fake/fr.gguf"); + const asked = recordAligner(); + const mgr = new SttManager(); + await mgr.init({ modelsBaseDir: "/tmp/fake-stt-models" }); + await mgr.transcribe({ samples: long, language: "fr" }); + expect(asked.every((x) => x === "/fake/fr.gguf")).toBe(true); + expect(cached).toHaveBeenCalledOnce(); + expect(ensure).not.toHaveBeenCalled(); + }); + + it("aborts the download on cancel and on shutdown, then retries on a later run", async () => { + const { ensure } = await mocks(); + const signals: AbortSignal[] = []; + ensure.mockImplementation( + ({ signal }: { signal?: AbortSignal }) => + new Promise((_, reject) => { + if (signal) signals.push(signal); + signal?.addEventListener("abort", () => reject(signal.reason)); + }), + ); + recordAligner(); + const mgr = new SttManager(); + await mgr.init({ modelsBaseDir: "/tmp/fake-stt-models" }); + await mgr.transcribe({ samples: long, language: "fr" }); + expect(signals).toHaveLength(1); + + mgr.cancel(); + expect(signals[0].aborted).toBe(true); + await new Promise((resolve) => setTimeout(resolve, 0)); + + // Not cached as failed for good: the next transcription starts it again. + await mgr.transcribe({ samples: long, language: "fr" }); + expect(signals).toHaveLength(2); + expect(signals[1].aborted).toBe(false); + await mgr.shutdown(); + expect(signals[1].aborted).toBe(true); + }); + + it("gives a language without an aligner nothing, and downloads nothing", async () => { + const { ensure } = await mocks(); + const mm = await import("./modelManager"); + vi.mocked(mm.alignerPath).mockReturnValueOnce(null); + const asked = recordAligner(); + const mgr = new SttManager(); + await mgr.init({ modelsBaseDir: "/tmp/fake-stt-models" }); + await mgr.transcribe({ samples: new Float32Array(16_000), language: "fr" }); + expect(asked).toEqual([null]); + expect(ensure).not.toHaveBeenCalled(); + }); + }); + it("fans status out to every sink, and detaching one leaves the others", async () => { const mgr = new SttManager(); const a = vi.fn<(e: SttStatusEvent) => void>(); diff --git a/electron/stt/index.ts b/electron/stt/index.ts index 656d7bf6dc..11c760f0bf 100644 --- a/electron/stt/index.ts +++ b/electron/stt/index.ts @@ -2,17 +2,20 @@ import path from "node:path"; import { app, type IpcMain } from "electron"; import { planChunks } from "./chunking"; import { extractMono16kPcm } from "./extractAudio"; -import { ensureModels, modelPaths } from "./modelManager"; import { - STT_VAD_UNAVAILABLE, - type SttPhraseSegment, - type SttStatusEvent, - type SttTiming, - type SttTranscribeRequest, - type SttTranscribeResponse, - type SttVadResponse, - type SttVadSegment, - type SttWordSegment, + alignerPath, + cachedAligner, + ensureAligner, + ensureModels, + modelPaths, +} from "./modelManager"; +import type { + SttPhraseSegment, + SttStatusEvent, + SttTiming, + SttTranscribeRequest, + SttTranscribeResponse, + SttWordSegment, } from "./transcriptionContract"; import { WhisperServerManager } from "./whisperServer"; @@ -108,6 +111,60 @@ export class SttManager { */ private cancelEpoch = 0; + /** + * The word aligner per language (ctcAlign.ts). A chunk never waits for a + * download: it is an improvement, and a 350 MB fetch inside a chunk would hold + * the helper's queue for as long as the network takes, past any request + * ceiling and out of Cancel's reach. So: + * - a copy already on disk is verified in place the first time a language is + * asked for in a session (a local read, no network), then reused; + * - otherwise the download starts in the background and the chunks keep + * whisper's times until it lands; later chunks and transcriptions use it; + * - Cancel and quit abort it, and so does a stall (`ensureAligner`). A failed + * one is not retried for the rest of the run, only on the next transcription. + */ + private readonly alignersReady = new Map(); + private readonly alignersChecked = new Set(); + private readonly alignerDownloads = new Map(); + private readonly alignersFailed = new Set(); + + private readonly alignerFor = async (language: string): Promise => { + const ready = this.alignersReady.get(language); + if (ready) return ready; + if (!alignerPath(this.getModelsDir(), language)) return null; + if (!this.alignersChecked.has(language)) { + this.alignersChecked.add(language); + const cached = await cachedAligner(this.getModelsDir(), language); + if (cached) { + this.alignersReady.set(language, cached); + return cached; + } + } + if (!this.alignersFailed.has(language) && !this.alignerDownloads.has(language)) { + this.downloadAligner(language); + } + return null; + }; + + private downloadAligner(language: string): void { + const controller = new AbortController(); + this.alignerDownloads.set(language, controller); + console.info(`[stt] downloading the "${language}" word aligner in the background`); + ensureAligner({ baseDir: this.getModelsDir(), language, signal: controller.signal }) + .then((file) => { + if (file) this.alignersReady.set(language, file); + console.info(`[stt] "${language}" word aligner ready`); + }) + .catch((error: unknown) => { + this.alignersFailed.add(language); + console.warn( + `[stt] no word aligner for "${language}", keeping whisper's word times: ` + + `${error instanceof Error ? error.message : String(error)}`, + ); + }) + .finally(() => this.alignerDownloads.delete(language)); + } + /** * The extraction in flight, if any. `cancelEpoch` alone stops the CHUNK loop, which is * checked between chunks — so a cancel during the decode left ffmpeg running to @@ -147,6 +204,7 @@ export class SttManager { cancel(): void { this.cancelEpoch++; this.extraction?.abort(); + for (const download of this.alignerDownloads.values()) download.abort(cancelledError()); } /** @@ -238,7 +296,7 @@ export class SttManager { for (let attempt = 1; attempt <= CHUNK_ATTEMPTS; attempt++) { if (this.shuttingDown) throw cancelledError(); try { - return await this.server.transcribe({ samples, language }); + return await this.server.transcribe({ samples, language, alignerFor: this.alignerFor }); } catch (error) { lastError = error; if (this.shuttingDown) throw cancelledError(); @@ -276,6 +334,7 @@ export class SttManager { await this.init(); const epoch = this.cancelEpoch; + this.alignersFailed.clear(); // Extraction is part of the run, and on a long file it is the part the user used // to watch the editor freeze through. Doing it here means the renderer hands over // a path and gets segments back, holding none of the audio. No new status phase: @@ -440,26 +499,9 @@ export class SttManager { async shutdown(): Promise { if (this.shuttingDown) return; this.shuttingDown = true; - this.cancelEpoch++; + this.cancel(); await this.server.shutdown(); } - - /** True when the helper reported Silero VAD loaded and ready. */ - isVadAvailable(): boolean { - return this.server.status.vadAvailable; - } - - /** - * Run Voice Activity Detection (Silero VAD) to detect speech segments in samples. - */ - async detectSpeech(samples: Float32Array): Promise { - if (this.shuttingDown) throw cancelledError(); - await this.init(); - if (!this.server.status.vadAvailable) { - throw new Error(STT_VAD_UNAVAILABLE); - } - return this.server.detectVadSegments({ samples }); - } } let singleton: SttManager | null = null; @@ -518,13 +560,6 @@ export function registerSttIpc(ipcMain: IpcMain): void { } }, ); - ipcMain.handle( - "stt:vad", - async (_event, req: { samples: Float32Array }): Promise => { - const segments = await manager.detectSpeech(req.samples); - return { segments }; - }, - ); ipcMain.handle("stt:cancel", () => { manager.cancel(); }); diff --git a/electron/stt/modelManager.test.ts b/electron/stt/modelManager.test.ts index 4419062347..dcee6a24f4 100644 --- a/electron/stt/modelManager.test.ts +++ b/electron/stt/modelManager.test.ts @@ -4,7 +4,16 @@ import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises" import { tmpdir } from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import { areModelsPresent, ensureModels, modelPaths, STT_MODELS } from "./modelManager"; +import { + alignerPath, + areModelsPresent, + CTC_ALIGNERS, + cachedAligner, + ensureAligner, + ensureModels, + modelPaths, + STT_MODELS, +} from "./modelManager"; describe("modelManager", () => { let dir: string; @@ -254,4 +263,116 @@ describe("modelManager", () => { STT_MODELS.whisper.files[0].expectedSha256 = originalSha; } }); + + describe("CTC aligners", () => { + it("pins every aligner to a digest and names its upstream revision", () => { + for (const file of Object.values(CTC_ALIGNERS)) { + expect(file.expectedSha256).toMatch(/^[0-9a-f]{64}$/); + expect(file.source).toMatch(/@[0-9a-f]{40}$/); + // Hosted under a `v0.0.0-*` release tag, never a moving one. + expect(file.url).toMatch(/\/releases\/download\/v0\.0\.0-[^/]+\//); + } + }); + + it("has none for a language it does not cover, prototype keys included", async () => { + expect(alignerPath(dir, "de")).toBeNull(); + expect(alignerPath(dir, "constructor")).toBeNull(); + expect(await ensureAligner({ baseDir: dir, language: "de" })).toBeNull(); + }); + + it("downloads, verifies and returns the aligner of a covered language", async () => { + const bytes = Buffer.from("gguf weights"); + const original = CTC_ALIGNERS.fr.expectedSha256; + CTC_ALIGNERS.fr.expectedSha256 = createHash("sha256").update(bytes).digest("hex"); + try { + const file = await ensureAligner({ + baseDir: dir, + language: "fr", + fetcher: async () => new Response(bytes, { status: 200 }), + }); + expect(file).toBe(alignerPath(dir, "fr")); + expect(await readFile(file as string)).toEqual(bytes); + } finally { + CTC_ALIGNERS.fr.expectedSha256 = original; + } + }); + + it("refuses a download whose digest does not match", async () => { + await expect( + ensureAligner({ + baseDir: dir, + language: "en", + fetcher: async () => new Response("tampered", { status: 200 }), + }), + ).rejects.toThrow(/SHA-256 mismatch/); + expect(existsSync(alignerPath(dir, "en") as string)).toBe(false); + }); + + /** A body that sends a few bytes, then nothing, ever. */ + const stalledBody = () => + new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(8)); + }, + }), + { status: 200 }, + ); + + it("aborts a download that stops sending, and leaves no partial file", async () => { + const file = alignerPath(dir, "fr") as string; + await expect( + ensureAligner({ + baseDir: dir, + language: "fr", + stallMs: 50, + fetcher: async () => stalledBody(), + }), + ).rejects.toThrow(/stalled/); + expect(existsSync(file)).toBe(false); + expect(existsSync(`${file}.partial`)).toBe(false); + }); + + it("stops when its signal aborts, mid-body or between attempts", async () => { + const controller = new AbortController(); + const download = ensureAligner({ + baseDir: dir, + language: "fr", + signal: controller.signal, + fetcher: async () => stalledBody(), + }); + setTimeout(() => controller.abort(new Error("cancelled")), 20); + await expect(download).rejects.toThrow("cancelled"); + + // A 503 puts it in a backoff of seconds: the abort must not wait for it. + const again = new AbortController(); + const started = Date.now(); + const retrying = ensureAligner({ + baseDir: dir, + language: "fr", + signal: again.signal, + fetcher: async () => new Response("busy", { status: 503 }), + }); + setTimeout(() => again.abort(new Error("quit")), 20); + await expect(retrying).rejects.toThrow("quit"); + expect(Date.now() - started).toBeLessThan(1000); + }); + + it("verifies a cached copy without the network", async () => { + expect(await cachedAligner(dir, "fr")).toBeNull(); + const file = alignerPath(dir, "fr") as string; + const bytes = Buffer.from("gguf weights"); + await mkdir(path.dirname(file), { recursive: true }); + await writeFile(file, bytes); + // Present but not the pinned bytes: not usable. + expect(await cachedAligner(dir, "fr")).toBeNull(); + const original = CTC_ALIGNERS.fr.expectedSha256; + CTC_ALIGNERS.fr.expectedSha256 = createHash("sha256").update(bytes).digest("hex"); + try { + expect(await cachedAligner(dir, "fr")).toBe(file); + } finally { + CTC_ALIGNERS.fr.expectedSha256 = original; + } + }); + }); }); diff --git a/electron/stt/modelManager.ts b/electron/stt/modelManager.ts index 959551237f..d9d4aef32a 100644 --- a/electron/stt/modelManager.ts +++ b/electron/stt/modelManager.ts @@ -20,8 +20,11 @@ import { pipeline } from "node:stream/promises"; * The file is verified by SHA-256 and written atomically (via .partial rename) * to prevent partial downloads from being treated as complete. * - * Word timestamps come from whisper.cpp's native DTW token timestamps, so no - * separate VAD model is required. See `technical-documentation/architecture/transcription-and-captions.md`. + * Word timestamps come from whisper.cpp's native DTW token timestamps. The + * Silero VAD model only decides which audio whisper decodes, and gives the + * speech edges phrases are anchored on. The CTC aligners (`CTC_ALIGNERS`) re-time + * the words for the languages that have one, and are fetched only when a + * transcription detects such a language. See `technical-documentation/architecture/transcription-and-captions.md`. */ export type SttModelId = "whisper" | "silero-vad"; @@ -93,6 +96,89 @@ export const STT_MODELS: Record = { }, }; +/** + * The CTC aligners that re-time whisper's words (electron/stt/ctcAlign.ts), per + * language whisper reports. A language missing here keeps whisper's DTW times. + * + * Each file is a wav2vec2 CTC model converted to GGUF by + * `scripts/convert-wav2vec2-gguf.mjs` (deterministic: re-running it on the source + * named in `source` reproduces the digest below), then published under a `v0.0.0-*` + * release tag like the other binaries that need a permanent URL but are not a + * product version (see scripts/fetch-onnxruntime.mjs). Apache-2.0, both of them. + */ +const ALIGNER_RELEASE = + "https://github.com/getopenscreen/openscreen/releases/download/v0.0.0-ctc-aligners-1"; + +export interface CtcAlignerFile extends SttModelFile { + /** HuggingFace repo and revision the file was converted from. */ + source: string; +} + +export const CTC_ALIGNERS: Record = { + en: { + name: "w2v-en-base-q8_0.gguf", + url: `${ALIGNER_RELEASE}/w2v-en-base-q8_0.gguf`, + expectedSha256: "b7f21a97208f368d3505bd9a7bc9ff3795b1169d8e3b036028082eb25eb464ae", + approximateBytes: 109_040_064, + source: "facebook/wav2vec2-base-960h@22aad52d435eb6dbaf354bdad9b0da84ce7d6156", + }, + fr: { + name: "w2v-fr-large-q8_0.gguf", + url: `${ALIGNER_RELEASE}/w2v-fr-large-q8_0.gguf`, + expectedSha256: "e3c284da3e27564db07ac226bf6402a4d7856b806455b3e0f5283f29f9495f48", + approximateBytes: 348_037_120, + // The repo's safetensors conversion PR: main only has pytorch_model.bin. + source: "jonatasgrosman/wav2vec2-large-xlsr-53-french@70db24a266633ffcc8edce4e72f3a5cb69d602d6", + }, +}; + +const ALIGNER_DIR = "ctc-aligner"; + +/** Where the aligner for `language` lives, or null when there is none for it. */ +export function alignerPath(baseDir: string, language: string): string | null { + const file = Object.keys(CTC_ALIGNERS).includes(language) ? CTC_ALIGNERS[language] : null; + return file ? path.join(baseDir, ALIGNER_DIR, file.name) : null; +} + +/** + * The aligner for `language` when it is already on disk and intact, without + * touching the network: null when it is missing, corrupt, or the language has + * none. A local read, so a transcription can afford to wait for it. + */ +export async function cachedAligner(baseDir: string, language: string): Promise { + const filePath = alignerPath(baseDir, language); + if (!filePath || !existsSync(filePath)) return null; + const actual = await sha256OfFile(filePath).catch(() => ""); + return actual === CTC_ALIGNERS[language].expectedSha256 ? filePath : null; +} + +/** Abort an aligner download that receives no bytes for this long. */ +export const ALIGNER_STALL_MS = 30_000; + +/** + * Make sure the aligner for `language` is on disk, downloading and verifying it + * like the whisper model. Null when the language has none; throws when the + * download fails, stalls for `stallMs` or is aborted through `signal`, which the + * caller turns into "keep whisper's times". + */ +export async function ensureAligner(opts: { + baseDir: string; + language: string; + signal?: AbortSignal; + stallMs?: number; + fetcher?: typeof fetch; +}): Promise { + const filePath = alignerPath(opts.baseDir, opts.language); + if (!filePath) return null; + const file = CTC_ALIGNERS[opts.language]; + await ensureFile(filePath, file.url, file.expectedSha256, { + fetcher: opts.fetcher, + signal: opts.signal, + stallMs: opts.stallMs ?? ALIGNER_STALL_MS, + }); + return filePath; +} + export function modelPaths(baseDir: string): Record { return { whisper: path.join(baseDir, STT_MODELS.whisper.cacheDir, MODEL_FILE), @@ -131,8 +217,19 @@ export async function sha256OfFile(filePath: string): Promise { const MAX_ATTEMPTS = 6; const RETRYABLE_STATUS = new Set([408, 425, 429, 500, 502, 503, 504]); -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); +function sleep(ms: number, signal?: AbortSignal): Promise { + return new Promise((resolve, reject) => { + if (signal?.aborted) return reject(signal.reason); + const timer = setTimeout(resolve, ms); + signal?.addEventListener( + "abort", + () => { + clearTimeout(timer); + reject(signal.reason); + }, + { once: true }, + ); + }); } function backoffMs(attempt: number, retryAfter: string | null): number { @@ -145,19 +242,34 @@ function backoffMs(attempt: number, retryAfter: string | null): number { return Math.min(60_000, 2_000 * 2 ** (attempt - 1)) + Math.floor(Math.random() * 1000); } -async function fetchWithRetry(url: string, fetcher: typeof fetch): Promise { +/** + * `signal` aborts the request and the waits between attempts; `waiting` is told + * when a backoff starts and ends, so a stall timer does not count it. + */ +async function fetchWithRetry( + url: string, + fetcher: typeof fetch, + signal?: AbortSignal, + waiting: (yes: boolean) => void = () => undefined, +): Promise { let lastErr: unknown; + const backoff = async (ms: number) => { + waiting(true); + await sleep(ms, signal); + waiting(false); + }; for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { try { const res = await fetcher(url, { headers: { "user-agent": "openscreen-stt" }, + signal, }); if (res.ok && res.body) return res; if (res.status >= 400 && res.status < 500 && !RETRYABLE_STATUS.has(res.status)) { throw new Error(`Failed to download ${url}: HTTP ${res.status} ${res.statusText}`); } if (RETRYABLE_STATUS.has(res.status) && attempt < MAX_ATTEMPTS) { - await sleep(backoffMs(attempt, res.headers.get("retry-after"))); + await backoff(backoffMs(attempt, res.headers.get("retry-after"))); continue; } throw new Error(`Failed to download ${url}: HTTP ${res.status} ${res.statusText}`); @@ -166,8 +278,8 @@ async function fetchWithRetry(url: string, fetcher: typeof fetch): Promise= MAX_ATTEMPTS) throw err; - await sleep(backoffMs(attempt, null)); + if (signal?.aborted || attempt >= MAX_ATTEMPTS) throw signal?.reason ?? err; + await backoff(backoffMs(attempt, null)); } } throw lastErr; @@ -178,6 +290,10 @@ export interface DownloadOptions { onProgress?: (bytes: number) => void; /** Override fetch (for tests); defaults to `globalThis.fetch`. */ fetcher?: typeof fetch; + /** Aborts the download (and the waits between attempts). */ + signal?: AbortSignal; + /** Abort when no byte arrives for this long, backoffs aside. Off when unset. */ + stallMs?: number; } /** @@ -211,17 +327,44 @@ async function ensureFile( await mkdir(path.dirname(filePath), { recursive: true }); const fetcher = options.fetcher ?? fetch; - const res = await fetchWithRetry(fileUrl, fetcher); const tmp = `${filePath}.partial`; - let downloaded = 0; - - const source = Readable.fromWeb(res.body as never); - source.on("data", (chunk: Buffer | Uint8Array) => { - downloaded += chunk.length; - options.onProgress?.(downloaded); - }); - const { createWriteStream } = await import("node:fs"); - await pipeline(source, createWriteStream(tmp)); + // One controller for the caller's abort and the stall timer: a connection that + // stops sending leaves `pipeline` pending forever, with nothing else to end it. + const controller = new AbortController(); + const onAbort = () => controller.abort(options.signal?.reason); + if (options.signal?.aborted) onAbort(); + options.signal?.addEventListener("abort", onAbort, { once: true }); + let stall: ReturnType | undefined; + const arm = (on = true) => { + clearTimeout(stall); + if (!on || !options.stallMs) return; + const ms = options.stallMs; + stall = setTimeout( + () => controller.abort(new Error(`download stalled: no data for ${ms / 1000}s`)), + ms, + ); + }; + try { + arm(); + const res = await fetchWithRetry(fileUrl, fetcher, controller.signal, (waiting) => + arm(!waiting), + ); + let downloaded = 0; + const source = Readable.fromWeb(res.body as never); + source.on("data", (chunk: Buffer | Uint8Array) => { + arm(); + downloaded += chunk.length; + options.onProgress?.(downloaded); + }); + const { createWriteStream } = await import("node:fs"); + await pipeline(source, createWriteStream(tmp), { signal: controller.signal }); + } catch (error) { + await rm(tmp, { force: true }).catch(() => undefined); + throw controller.signal.aborted ? (controller.signal.reason ?? error) : error; + } finally { + arm(false); + options.signal?.removeEventListener("abort", onAbort); + } if (expectedSha256) { const actual = await sha256OfFile(tmp); diff --git a/electron/stt/snapWordBoundaries.test.ts b/electron/stt/snapWordBoundaries.test.ts index 2668c6966f..f3272b38d9 100644 --- a/electron/stt/snapWordBoundaries.test.ts +++ b/electron/stt/snapWordBoundaries.test.ts @@ -1,76 +1,192 @@ import { describe, expect, it } from "vitest"; -import { snapWordBoundariesToAudio } from "./snapWordBoundaries"; +import { anchorWordsOnSpeech, type HelperWord, ownWords } from "./snapWordBoundaries"; import type { SttWordSegment } from "./transcriptionContract"; -const SAMPLE_RATE = 16_000; - -/** Mono 16 kHz buffer that is loud everywhere except the given silent spans. */ -function audioWithSilences(durationSec: number, silences: Array<[number, number]>): Float32Array { - const samples = new Float32Array(Math.round(durationSec * SAMPLE_RATE)); - for (let i = 0; i < samples.length; i++) { - const t = i / SAMPLE_RATE; - const silent = silences.some(([from, to]) => t >= from && t < to); - // Alternating ±0.5 gives a flat, non-zero RMS without needing a real tone. - samples[i] = silent ? 0 : i % 2 === 0 ? 0.5 : -0.5; - } - return samples; -} - -const word = (w: Partial = {}): SttWordSegment => ({ - word: "w", - startSec: 0, - endSec: 0.1, - ...w, +/** A helper word; its anchor defaults to its start, as for a request's first word. */ +const word = ( + text: string, + startSec: number, + endSec: number, + anchorSec = startSec, +): HelperWord => ({ + word: text, + startSec, + endSec, + anchorSec, }); +const ms = (sec: number) => Math.round(sec * 1000) / 1000; +const times = (words: SttWordSegment[]) => words.map((w) => [w.word, ms(w.startSec), ms(w.endSec)]); -describe("snapWordBoundariesToAudio", () => { - it("pulls a late boundary back into the silence that precedes it", () => { - // Speech stops at 1.0 and resumes at 1.2; whisper reports the next word - // starting at 1.3 — 100 ms after the audio actually resumed. - const samples = audioWithSilences(3, [[1.0, 1.2]]); - const [snapped] = snapWordBoundariesToAudio([word({ startSec: 1.3, endSec: 1.8 })], samples); - expect(snapped.startSec).toBeGreaterThanOrEqual(1.0); - expect(snapped.startSec).toBeLessThan(1.2); +describe("anchorWordsOnSpeech", () => { + it("returns the helper's times without speech intervals, minus the anchor", () => { + const out = anchorWordsOnSpeech([word("a", 1, 1.4, 1.2), word("b", 1.4, 1.4, 1.4)]); + expect(out).toEqual([ + { word: "a", startSec: 1, endSec: 1.4 }, + { word: "b", startSec: 1.4, endSec: 1.42 }, + ]); }); - it("leaves a boundary alone when nothing quieter precedes it", () => { - // A word ending a phrase: whisper is already right, the frames before the - // boundary are all speech, so the quietest frame in the window is the - // boundary itself and it must not drift. - const samples = audioWithSilences(3, [[1.5, 2.0]]); - const [snapped] = snapWordBoundariesToAudio([word({ startSec: 1.0, endSec: 1.5 })], samples); - expect(snapped.endSec).toBeCloseTo(1.5, 2); + it("puts a phrase's first word on its onset whether DTW put it late or early", () => { + // "Salut" opens the request, so it has no previous token and starts late, + // on its own first token. "Bah" starts where the previous token ended, at + // the end of the first phrase: in the pause, before its own speech. + const words = [ + word("Salut", 2.15, 2.5), + word("!", 2.5, 2.6, 2.6), + word("Bah", 2.6, 3.8, 3.61), + word("voilà", 3.8, 4.35, 4.01), + ]; + const speech = [ + { startSec: 1.57, endSec: 2.56 }, + { startSec: 3.36, endSec: 4.35 }, + ]; + expect(times(anchorWordsOnSpeech(words, speech))).toEqual([ + ["Salut", 1.57, 2.56], + ["!", 2.56, 2.56], + ["Bah", 3.36, 3.8], + ["voilà", 3.8, 4.35], + ]); }); - it("never moves a boundary more than the lookback window", () => { - const samples = audioWithSilences(3, [[0.0, 1.0]]); - const [snapped] = snapWordBoundariesToAudio([word({ startSec: 2.0, endSec: 2.5 })], samples); - expect(snapped.startSec).toBeGreaterThanOrEqual(2.0 - 0.15); + it("ends each phrase's last word where its speech stops, stretched or cut back", () => { + const words = [ + word("tic,", 1.95, 2.59), + word("tac", 2.59, 2.79, 2.7), + word("!", 2.79, 2.9, 2.9), + ]; + const speech = [ + { startSec: 1.95, endSec: 2.37 }, + { startSec: 2.59, endSec: 3.04 }, + ]; + expect(times(anchorWordsOnSpeech(words, speech))).toEqual([ + ["tic,", 1.95, 2.37], + ["tac", 2.59, 3.04], + ["!", 3.04, 3.04], + ]); }); - it("keeps a boundary shared by two words shared", () => { - const samples = audioWithSilences(3, [[1.0, 1.2]]); - const [first, second] = snapWordBoundariesToAudio( - [word({ startSec: 0.5, endSec: 1.3 }), word({ startSec: 1.3, endSec: 1.8 })], - samples, + it("leaves an edge alone when the word is too far past it to be that edge", () => { + // 1.2 s after the onset and 1.1 s before the offset: more likely a neighbour + // of words whisper dropped than the phrase's own edges. + const tooFar = [word("w", 2.2, 2.5)]; + expect(times(anchorWordsOnSpeech(tooFar, [{ startSec: 1, endSec: 3.6 }]))).toEqual( + times(tooFar), ); - expect(first.endSec).toBeCloseTo(second.startSec, 6); }); - it("leaves boundaries that fall outside the decoded audio alone", () => { - // Clamping these into range would collapse every boundary onto the end of - // the buffer instead of leaving the unmeasurable ones untouched. - const samples = audioWithSilences(0.1, []); - const words = [word({ startSec: 5.51, endSec: 6.85 })]; - expect(snapWordBoundariesToAudio(words, samples)).toEqual(words); + it("gives a word to the stretch its anchor falls in, the kept tail included", () => { + // "b" is anchored in the first stretch's 0.1 s tail: it closes that phrase. + // "c" starts in the first stretch (the previous token's end) but is + // anchored in the second, so it opens the second. + const words = [ + word("a", 1, 1.9, 1.5), + word("b", 1.9, 2.05, 2.05), + word("c", 2.05, 3.5, 3.2), + word("d", 3.5, 4, 3.8), + ]; + const speech = [ + { startSec: 1, endSec: 2 }, + { startSec: 3, endSec: 4 }, + ]; + expect(times(anchorWordsOnSpeech(words, speech))).toEqual([ + ["a", 1, 1.9], + ["b", 1.9, 2], + ["c", 3, 3.5], + ["d", 3.5, 4], + ]); + }); + + // Issue #948, measured on a Zira TTS clip: "…as if nothing happened. The + // transcript…" with speech stopping at 30.59 and resuming at 31.33. DTW + // anchored "happened." at 30.69, past the first stretch's tail. + const pause = [ + { startSec: 28.13, endSec: 30.59 }, + { startSec: 31.33, endSec: 35.23 }, + ]; + + it("keeps a sentence's last word in the stretch it closes when DTW anchors it in the pause", () => { + const words = [ + word("and", 27.97, 28.35, 28.35), + word("nothing", 29.65, 30.03, 30.03), + word("happened.", 30.03, 30.69, 30.69), + word("The", 30.69, 31.47, 31.47), + word("transcript", 31.47, 32.13, 32.13), + ]; + expect(times(anchorWordsOnSpeech(words, pause))).toEqual([ + ["and", 28.13, 28.35], + ["nothing", 29.65, 30.03], + ["happened.", 30.03, 30.59], + ["The", 31.33, 31.47], + ["transcript", 31.47, 32.13], + ]); + expect(ownWords(words, pause)).toEqual([ + [0, 3], + [3, 5], + ]); + }); + + it("keeps the punctuation whisper split off a sentence's last word with it", () => { + const words = [ + word("et", 28.2, 28.4, 28.4), + word("vraiment", 28.4, 30.69, 30.69), + word("?", 30.69, 30.7, 30.7), + word("Oui", 30.7, 31.6, 31.5), + ]; + expect(ownWords(words, pause)).toEqual([ + [0, 3], + [3, 4], + ]); + }); + + it("does not read an opening bracket after a sentence's last word as part of its end", () => { + // "b." is anchored in the pause, then whisper split "(" off the next word: + // "b.(" is no sentence end, so "b." would have opened the next stretch. + const words = [ + word("a", 28.2, 30.03, 30.03), + word("b.", 30.03, 30.69, 30.69), + word("(", 30.69, 31.4, 31.4), + word("c)", 31.4, 32, 32), + ]; + expect(ownWords(words, pause)).toEqual([ + [0, 2], + [2, 4], + ]); }); - it("keeps degenerate words non-empty and passes words through without audio", () => { - const samples = audioWithSilences(3, [[1.0, 1.2]]); - const [degenerate] = snapWordBoundariesToAudio([word({ startSec: 1.3, endSec: 1.3 })], samples); - expect(degenerate.endSec).toBeGreaterThan(degenerate.startSec); + it("lets a one-word sentence open a stretch when it is anchored in its speech", () => { + const words = [ + word("Agreed?", 28.13, 30.5, 30.4), + word("Yes.", 30.5, 31.6, 31.5), + word("Good", 31.6, 32, 32), + ]; + expect(ownWords(words, pause)).toEqual([ + [0, 1], + [1, 3], + ]); + }); + + it("gives a sentence's last word to the next stretch when the one before has no words", () => { + const words = [word("Done.", 31, 31.6, 31.2), word("Next", 31.6, 32, 32)]; + expect(ownWords(words, pause)).toEqual([ + [0, 0], + [0, 2], + ]); + }); - const untouched = [word({ startSec: 1.3, endSec: 1.8 })]; - expect(snapWordBoundariesToAudio(untouched, new Float32Array(0))).toEqual(untouched); + it("puts the words in order, none inverted, whatever stretch owns them", () => { + // The aligner-on shape of the report, as the aligner left it when the next + // stretch owned "happened": a 20 ms word after the pause and "The" earlier. + const aligned = [ + word("nothing", 29.68, 30.59, 30.03), + word("happened", 31.33, 31.35, 30.69), + word("The", 30.69, 31.48, 31.47), + word("transcript", 31.48, 32.12, 32.13), + ]; + const out = anchorWordsOnSpeech(aligned, pause); + for (const [j, w] of out.entries()) { + expect(w.endSec).toBeGreaterThanOrEqual(w.startSec); + if (j > 0) expect(w.startSec).toBeGreaterThanOrEqual(out[j - 1].startSec); + } + expect(out.map((w) => w.word)).toEqual(["nothing", "happened", "The", "transcript"]); }); }); diff --git a/electron/stt/snapWordBoundaries.ts b/electron/stt/snapWordBoundaries.ts index d5d0b6f10c..0cbcb6991f 100644 --- a/electron/stt/snapWordBoundaries.ts +++ b/electron/stt/snapWordBoundaries.ts @@ -1,100 +1,165 @@ -// Pulls whisper.cpp's DTW word boundaries back onto the audio they describe. +// Puts the edges of every phrase on the speech the helper's VAD found. // -// whisper.cpp reports one time per token and derives word spans from it, so a -// word's `start` is the point where the decoder *emitted* the token, not where -// the speaker started saying it. Measured on real recordings, that lands -// consistently 80–150 ms late, and because consecutive words share a boundary -// (`word[i].end === word[i+1].start`) the whole transcript is dragged right by -// roughly a syllable. +// The helper times a word from the end of the token before it to the end of its +// own last token (whisper.cpp's t_dtw marks where a token ENDS, see +// whisper-stt/src/main.cpp). Inside continuous speech that lands within ~30 ms +// of the audio (measured on a TTS corpus with exact word times, +// tools/stt-eval/word-timing), so inner boundaries are left alone: an RMS snap +// on top of them would drag correct boundaries early. // -// That is invisible in captions but not in the transcript editor: deleting -// words there turns the selection into a trim of exactly -// `[firstWord.startSec, lastWord.endSec]`, so a late boundary leaves the attack -// of the first removed word audible and bites into the following kept word. +// The edges of a phrase are the exception. The token before a phrase's first +// word is the previous phrase's last one, so that word "starts" at the end of +// the previous phrase, in the pause, or DTW puts it late when the helper +// starts a new decode window there. Its last word ends on whatever token DTW +// last aligned, short of the speech or past it. When the helper sent its speech +// intervals, both are put on the edges of the stretch of speech they belong to. // -// The correction is to look at the audio instead of guessing an offset: each -// boundary moves back to the quietest 10 ms frame within the preceding -// `LOOKBACK_SEC`. It is self-limiting — on a decaying tail (a word that ends a -// phrase, where whisper is already right) the quietest frame IS the reported -// one, so the boundary doesn't move at all. +// That matters because deleting words in the transcript editor trims exactly +// `[firstWord.startSec, lastWord.endSec]`: a late boundary leaves the attack of +// the first removed word audible, an early one bites into the kept word. -import type { SttWordSegment } from "./transcriptionContract"; +import type { SttVadSegment, SttWordSegment } from "./transcriptionContract"; -/** Matches `writeSamplesAsWav` — the samples handed to whisper are mono 16 kHz. */ -const SAMPLE_RATE = 16_000; +/** A word as the helper reports it: `anchorSec` is always inside the word. */ +export interface HelperWord extends SttWordSegment { + /** End of the word's first token. Only tells which stretch of speech owns the word. */ + anchorSec: number; +} -/** RMS envelope resolution; finer than any boundary error worth correcting. */ -const FRAME_SEC = 0.01; +/** Keep degenerate words non-empty. */ +const MIN_WORD_SEC = 0.02; /** - * How far back a boundary may travel — the calibration knob for this - * correction. Sized to the measured DTW lag (~80–150 ms). Widen it and - * boundaries inside continuous speech start snapping onto the *previous* - * syllable's trough; narrow it and the lag survives. + * How far a phrase's first word may start after its speech, or its last word + * end before it, and still be stretched onto that edge — the calibration knob of + * the anchoring step. Past it, the word is more likely a neighbour of one + * whisper dropped, and stretching it over that audio would be a guess. Pulling + * a word forward to the onset, or cutting it back to where the speech stops, + * needs no such bound: the VAD says nothing is said outside it. */ -const LOOKBACK_SEC = 0.15; +const MAX_ANCHOR_SEC = 1; -/** Keep degenerate words (whisper sometimes reports end <= start) non-empty. */ -const MIN_WORD_SEC = 0.02; +/** + * Audio the helper keeps past each speech offset, as whisper.cpp does, so a soft + * ending survives. A word whose anchor falls in there belongs to the stretch it + * trails. + */ +const TAIL_SEC = 0.1; -/** Per-frame RMS of the mono signal — the cheapest usable "is this speech" proxy. */ -function rmsEnvelope(samples: Float32Array): Float32Array { - const frameLength = Math.round(SAMPLE_RATE * FRAME_SEC); - const frameCount = Math.floor(samples.length / frameLength); - const envelope = new Float32Array(frameCount); - for (let f = 0; f < frameCount; f++) { - const start = f * frameLength; - let sum = 0; - for (let i = start; i < start + frameLength; i++) { - const v = samples[i] ?? 0; - sum += v * v; - } - envelope[f] = Math.sqrt(sum / frameLength); +/** French puts a space before `!`, `?`, `:` and `;`, so whisper emits them as words. */ +const isPunctuation = (word: string) => /^[\p{P}\p{S}]+$/u.test(word); + +/** An opening bracket or quote starts what follows, so it never closes a sentence. */ +const opens = (word: string) => /^[\p{Ps}\p{Pi}]+$/u.test(word); + +/** `.`, `!`, `?`, `…` or a full-width form, maybe inside a closing quote or bracket. */ +const endsSentence = (text: string) => /[.!?…。!?.。][\p{Pe}\p{Pf}"']*$/u.test(text); + +/** End of the audio the helper kept for stretch `i`: its speech, plus a tail short of the next. */ +const keptUntil = (speech: SttVadSegment[], i: number) => + Math.min(speech[i].endSec + TAIL_SEC, speech[i + 1]?.startSec ?? Number.POSITIVE_INFINITY); + +/** + * The words each speech stretch owns, as `[from, to)` ranges of `words`: those + * anchored before the end of the audio the helper kept for it. Both the aligner + * (ctcAlign.ts) and the edge anchoring below read this, so a word is aligned in + * the stretch whose edges it is put on. + * + * One exception. DTW can anchor a sentence's last word in the pause after it, + * past the kept tail: the next stretch then put it on its onset, after the + * words that follow it, and the aligner looked for it in the wrong audio (issue + * #948). So a word that ends a sentence and is anchored in the pause, before the + * next stretch's speech, closes the stretch before it, with its punctuation, + * when that stretch has words. One anchored inside the next speech still opens + * it: on LibriSpeech, nine one-word sentences ("Heaven!", "What?") open a + * stretch that way, and giving them to the stretch before cost phrase deletes. + */ +export function ownWords( + words: ReadonlyArray<{ word: string; anchorSec: number }>, + speech: SttVadSegment[], +): Array<[number, number]> { + const out: Array<[number, number]> = []; + let k = 0; + for (let i = 0; i < speech.length; i++) { + const from = k; + const until = keptUntil(speech, i); + while (k < words.length && words[k].anchorSec < until) k++; + // The word that would open the next stretch, and the punctuation after it. + let first = k; + while (first < words.length && isPunctuation(words[first].word)) first++; + let end = first + 1; + while (end < words.length && isPunctuation(words[end].word) && !opens(words[end].word)) end++; + if ( + i + 1 < speech.length && + first < words.length && + words[first].anchorSec < speech[i + 1].startSec && + words.slice(from, k).some((w) => !isPunctuation(w.word)) && + endsSentence( + words + .slice(first, end) + .map((w) => w.word) + .join(""), + ) + ) + k = end; + out.push([from, k]); } - return envelope; + return out; } /** - * Move every word boundary back to the quietest frame in the `LOOKBACK_SEC` - * preceding it. Boundaries shared by two words snap identically (same input - * time), so the transcript stays gap-free where whisper made it gap-free. - * Returns the words unchanged when there is no audio to measure against. + * Put the first and last word of every speech stretch on the stretch's edges. + * The onset already carries the VAD's 30 ms pad, so a cut there lands just + * before the attack. Words own the speech and no pause: the last word ends on + * the offset, and the punctuation closing the phrase collapses to a point + * there, wherever DTW dropped it. Without `speech` (the helper ran without its + * VAD model), the words come back as the helper timed them. */ -export function snapWordBoundariesToAudio( - words: SttWordSegment[], - samples: Float32Array, +export function anchorWordsOnSpeech( + words: HelperWord[], + speech?: SttVadSegment[], ): SttWordSegment[] { - const envelope = rmsEnvelope(samples); - if (envelope.length === 0) return words; + const out: SttWordSegment[] = words.map(({ anchorSec: _, ...w }) => ({ + ...w, + endSec: Math.max(w.endSec, w.startSec + MIN_WORD_SEC), + })); + if (speech) anchorEdges(out, words, speech); + // Whatever the stretches did, words come out in order and never inverted. A + // start past the next word's comes back to it rather than pushing the next + // one: the onset clamp is what overshoots (a word whisper invented, put on + // the onset after the real one the aligner found earlier). + for (let j = out.length - 1; j >= 0; j--) { + if (j + 1 < out.length) out[j].startSec = Math.min(out[j].startSec, out[j + 1].startSec); + out[j].endSec = Math.max(out[j].endSec, out[j].startSec); + } + return out; +} - const lookbackFrames = Math.round(LOOKBACK_SEC / FRAME_SEC); - const snap = (timeSec: number): number => { - const hi = Math.round(timeSec / FRAME_SEC); - // A boundary outside the audio we can measure (whisper occasionally reports - // times past the end of the samples) has nothing to snap to — clamping it - // into range would drag it to the end of the buffer instead. - if (hi <= 0 || hi >= envelope.length) return timeSec; - const lo = Math.max(0, hi - lookbackFrames); - let bestFrame = hi; - let bestRms = envelope[hi]; - // Strict `<` while walking backwards keeps the frame CLOSEST to the - // reported boundary when a whole stretch is equally quiet, so digital - // silence can't drag a boundary the full window. - for (let f = hi - 1; f >= lo; f--) { - if (envelope[f] < bestRms) { - bestRms = envelope[f]; - bestFrame = f; +function anchorEdges(out: SttWordSegment[], words: HelperWord[], speech: SttVadSegment[]) { + const owned = ownWords(words, speech); + for (let i = 0; i < speech.length; i++) { + const { startSec: onset, endSec: offset } = speech[i]; + let first = -1; + let last = -1; + for (let k = owned[i][0]; k < owned[i][1]; k++) { + if (isPunctuation(out[k].word)) continue; + if (first < 0) first = k; + last = k; + } + if (first < 0) continue; + if (out[first].startSec - onset <= MAX_ANCHOR_SEC) { + out[first].startSec = onset; + out[first].endSec = Math.max(out[first].endSec, onset + MIN_WORD_SEC); + for (let j = first - 1; j >= 0 && out[j].endSec > onset; j--) { + out[j].endSec = onset; + out[j].startSec = Math.min(out[j].startSec, onset); } } - return bestFrame * FRAME_SEC; - }; - - return words.map((w) => { - const startSec = snap(w.startSec); - return { - ...w, - startSec, - endSec: Math.max(snap(w.endSec), startSec + MIN_WORD_SEC), - }; - }); + if (offset - out[last].endSec > MAX_ANCHOR_SEC) continue; + out[last].endSec = Math.max(offset, out[last].startSec + MIN_WORD_SEC); + for (let j = last + 1; j < out.length && isPunctuation(out[j].word); j++) { + out[j].startSec = out[last].endSec; + out[j].endSec = out[last].endSec; + } + } } diff --git a/electron/stt/transcriptionContract.ts b/electron/stt/transcriptionContract.ts index c9b7d48509..4a2563a782 100644 --- a/electron/stt/transcriptionContract.ts +++ b/electron/stt/transcriptionContract.ts @@ -141,7 +141,17 @@ export interface SttTranscribeRequest { * a literal typed twice is a fallback that silently stops working. */ export const STT_NATIVE_EXTRACTION_UNAVAILABLE = "stt:native-extraction-unavailable"; -export const STT_VAD_UNAVAILABLE = "stt:vad-unavailable"; + +/** + * Marker carried in the error message when audio extraction could not read the + * `sourcePath` itself: the OS refused the read, or ffmpeg hung opening the file + * (macOS holds the read behind a pending folder-access prompt, issue #968). + * + * Only `extractAudio.ts` sets it, so the renderer can tell "this file's folder is + * not readable" from an engine failure that happens to mention a permission (model + * loading, the whisper helper). Same IPC reason as above for being a string. + */ +export const STT_MEDIA_UNREADABLE = "stt:media-unreadable"; /** IPC response: main → renderer. */ export interface SttTranscribeResponse { @@ -165,8 +175,3 @@ export interface SttVadSegment { startSec: number; endSec: number; } - -/** Response from the VAD speech detection pass. */ -export interface SttVadResponse { - segments: SttVadSegment[]; -} diff --git a/electron/stt/whisperServer.test.ts b/electron/stt/whisperServer.test.ts index 314926e587..773235577f 100644 --- a/electron/stt/whisperServer.test.ts +++ b/electron/stt/whisperServer.test.ts @@ -3,9 +3,8 @@ import { mkdtemp, rm, stat, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { STT_VAD_UNAVAILABLE } from "./transcriptionContract"; import { writeSamplesAsWav } from "./wav"; -import { MAX_VAD_CHUNK_SAMPLES, mergeVadIntervals, WhisperServerManager } from "./whisperServer"; +import { WhisperServerManager } from "./whisperServer"; vi.mock("node:child_process", async (importOriginal) => { // `node:child_process` is CJS, so the ESM namespace also carries a `default` @@ -350,6 +349,148 @@ describe("WhisperServerManager", () => { } }); + it("anchors a phrase's first word on the onset of the speech its anchor falls in", async () => { + // "Salut" starts where the previous token ended, in the first stretch's + // tail, but its anchor puts it in the second stretch. + const fakeJson = { + segments: [ + { + text: " Salut", + start: 1.57, + end: 2.56, + words: [{ word: " Salut", start: 1.05, end: 2.56, anchor: 1.8 }], + }, + ], + speech: [ + { start: 0.2, end: 1 }, + { start: 1.57, end: 2.56 }, + ], + backend: "whispercpp-cpu", + }; + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response(JSON.stringify(fakeJson), { status: 200 })), + ); + try { + const mgr = new WhisperServerManager(); + (mgr as unknown as { process: unknown; port: number }).process = {}; + (mgr as unknown as { process: unknown; port: number }).port = 9999; + const result = await mgr.transcribe({ samples: new Float32Array(16_000 * 3) }); + expect(result.wordSegments).toEqual([{ word: "Salut", startSec: 1.57, endSec: 2.56 }]); + } finally { + vi.unstubAllGlobals(); + } + }); + + describe("CTC word aligner", () => { + const inference = { + segments: [ + { + text: " ab ba", + start: 0.2, + end: 1, + words: [ + { word: " ab", start: 0.3, end: 0.5, anchor: 0.4 }, + { word: " ba", start: 0.5, end: 0.8, anchor: 0.6 }, + ], + }, + ], + speech: [{ start: 0.2, end: 1 }], + detected_language: "fr", + backend: "whispercpp-vulkan", + timing: { elapsed_s: 0.5, audio_s: 2, rtf: 0.25 }, + }; + // "ab" spoken on frames 20-21, the word delimiter on 22, "ba" on 24-25. + const vocab = ["", "|", "a", "b"]; + const frames = Array(60).fill(0); + frames[20] = 2; + frames[21] = 3; + frames[22] = 1; + frames[24] = 3; + frames[25] = 2; + const logprobs = new Float32Array(60 * vocab.length).fill(-12); + frames.forEach((tok, f) => { + logprobs[f * vocab.length + tok] = -0.01; + }); + const emissions = { + vocab, + blank: 0, + stride_s: 0.02, + elapsed_s: 0.1, + regions: [ + { start: 0, frames: 60, logprobs: Buffer.from(logprobs.buffer).toString("base64") }, + ], + }; + + async function run(emissionsReply: Response, reply: object = inference) { + const fetchMock = vi.fn(async (url: string, _init?: RequestInit) => + url.endsWith("/emissions") + ? emissionsReply + : new Response(JSON.stringify(reply), { status: 200 }), + ); + vi.stubGlobal("fetch", fetchMock); + try { + const mgr = new WhisperServerManager(); + (mgr as unknown as { process: unknown; port: number }).process = {}; + (mgr as unknown as { process: unknown; port: number }).port = 9999; + const alignerFor = vi.fn(async (language: string) => `/models/${language}.gguf`); + const result = await mgr.transcribe({ + samples: new Float32Array(16_000 * 2), + alignerFor, + }); + return { result, alignerFor, fetchMock }; + } finally { + vi.unstubAllGlobals(); + } + } + + it("re-times the words on the aligner of the detected language, and counts its cost", async () => { + const { result, alignerFor, fetchMock } = await run( + new Response(JSON.stringify(emissions), { status: 200 }), + ); + expect(alignerFor).toHaveBeenCalledWith("fr"); + const body = fetchMock.mock.calls[1][1]?.body as FormData; + expect(body.get("model")).toBe("/models/fr.gguf"); + expect(JSON.parse(String(body.get("regions")))).toEqual([[0, 1.3]]); + // The phrase edges stay on the speech; the inner boundary moves to the letters. + const ms = (x: number) => Math.round(x * 1000); + expect(result.wordSegments.map((w) => [w.word, ms(w.startSec), ms(w.endSec)])).toEqual([ + ["ab", 200, 450], + ["ba", 450, 1000], + ]); + expect(result.timing?.elapsedSec).toBeCloseTo(0.6); + expect(result.timing?.rtf).toBeCloseTo(0.3); + }); + + it("leaves the aligner out on the CPU, without even asking for it", async () => { + const { result, alignerFor, fetchMock } = await run( + new Response(JSON.stringify(emissions), { status: 200 }), + { ...inference, backend: "whispercpp-cpu" }, + ); + expect(alignerFor).not.toHaveBeenCalled(); + expect(fetchMock).toHaveBeenCalledOnce(); + expect(result.wordSegments.map((w) => [w.word, w.startSec, w.endSec])).toEqual([ + ["ab", 0.2, 0.5], + ["ba", 0.5, 1], + ]); + }); + + it("keeps whisper's times when the aligner fails", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + try { + const { result } = await run(new Response("not found", { status: 404 })); + expect(result.wordSegments.map((w) => [w.word, w.startSec, w.endSec])).toEqual([ + ["ab", 0.2, 0.5], + ["ba", 0.5, 1], + ]); + expect(result.timing?.elapsedSec).toBe(0.5); + expect(warn).toHaveBeenCalledWith(expect.stringMatching(/word aligner failed/)); + } finally { + warn.mockRestore(); + } + }); + }); + it("spawns whisper-stt-server with --model", async () => { const fs = await import("node:fs/promises"); const { spawn } = await import("node:child_process"); @@ -729,175 +870,4 @@ describe("WhisperServerManager", () => { await rm(dir, { recursive: true, force: true }); } }); - - it("detectVadSegments extracts segments from /vad response", async () => { - const fakeJson = { - segments: [ - { start: 0.25, end: 1.5 }, - { start: 2.1, end: 4.8 }, - ], - }; - vi.stubGlobal( - "fetch", - vi.fn(async () => new Response(JSON.stringify(fakeJson), { status: 200 })), - ); - try { - const mgr = new WhisperServerManager(); - (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).process = {}; - (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).port = 9999; - (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).vadAvailable = - true; - - const segments = await mgr.detectVadSegments({ samples: new Float32Array(1600) }); - expect(segments).toEqual([ - { startSec: 0.25, endSec: 1.5 }, - { startSec: 2.1, endSec: 4.8 }, - ]); - } finally { - vi.unstubAllGlobals(); - } - }); - - it("detectVadSegments surfaces server error from /vad endpoint", async () => { - vi.stubGlobal( - "fetch", - vi.fn(async () => new Response("VAD model was not loaded", { status: 400 })), - ); - try { - const mgr = new WhisperServerManager(); - (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).process = {}; - (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).port = 9999; - (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).vadAvailable = - true; - - await expect(mgr.detectVadSegments({ samples: new Float32Array(1600) })).rejects.toThrow( - /whisper-stt-server \/vad HTTP 400/, - ); - } finally { - vi.unstubAllGlobals(); - } - }); - - it("filters out /vad segments missing end that map to zero-length intervals", async () => { - const fakeJson = { - segments: [ - { start: 1.5 }, // missing end -> maps to startSec: 1.5, endSec: 1.5 - ], - }; - vi.stubGlobal( - "fetch", - vi.fn(async () => new Response(JSON.stringify(fakeJson), { status: 200 })), - ); - try { - const mgr = new WhisperServerManager(); - (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).process = {}; - (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).port = 9999; - (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).vadAvailable = - true; - - const segments = await mgr.detectVadSegments({ samples: new Float32Array(1600) }); - expect(segments).toEqual([]); - } finally { - vi.unstubAllGlobals(); - } - }); - - it("handles malformed WAV /vad HTTP 400 without crashing helper availability", async () => { - vi.stubGlobal( - "fetch", - vi.fn( - async () => new Response(JSON.stringify({ error: "failed to parse WAV" }), { status: 400 }), - ), - ); - try { - const mgr = new WhisperServerManager(); - (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).process = { - pid: 999, - }; - (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).port = 9999; - (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).vadAvailable = - true; - - await expect(mgr.detectVadSegments({ samples: new Float32Array(100) })).rejects.toThrow( - /whisper-stt-server \/vad HTTP 400/, - ); - expect(mgr.status.running).toBe(true); - } finally { - vi.unstubAllGlobals(); - } - }); - - it("rejects detectVadSegments when VAD is unavailable", async () => { - const mgr = new WhisperServerManager(); - (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).process = {}; - (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).port = 9999; - (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).vadAvailable = - false; - - await expect(mgr.detectVadSegments({ samples: new Float32Array(1600) })).rejects.toThrow( - STT_VAD_UNAVAILABLE, - ); - }); - - describe("mergeVadIntervals", () => { - it("returns empty array for empty input", () => { - expect(mergeVadIntervals([])).toEqual([]); - }); - - it("returns single interval as-is", () => { - expect(mergeVadIntervals([{ startSec: 1, endSec: 2 }])).toEqual([{ startSec: 1, endSec: 2 }]); - }); - - it("merges overlapping and adjacent intervals", () => { - const input = [ - { startSec: 1, endSec: 3 }, - { startSec: 2.5, endSec: 5 }, - { startSec: 5, endSec: 6.2 }, - { startSec: 8, endSec: 10 }, - ]; - expect(mergeVadIntervals(input)).toEqual([ - { startSec: 1, endSec: 6.2 }, - { startSec: 8, endSec: 10 }, - ]); - }); - - it("drops inverted intervals where endSec <= startSec", () => { - const input = [ - { startSec: 2, endSec: 1 }, - { startSec: 3, endSec: 4 }, - ]; - expect(mergeVadIntervals(input)).toEqual([{ startSec: 3, endSec: 4 }]); - }); - - it("drops singleton intervals where endSec <= startSec", () => { - expect(mergeVadIntervals([{ startSec: 2, endSec: 1 }])).toEqual([]); - expect(mergeVadIntervals([{ startSec: 2, endSec: 2 }])).toEqual([]); - }); - }); - - it("chunks long audio across MAX_VAD_CHUNK_SAMPLES and offsets timestamps", async () => { - const fetchMock = vi.fn(async () => { - const callIndex = fetchMock.mock.calls.length - 1; - const segments = callIndex === 0 ? [{ start: 10, end: 180 }] : [{ start: 0, end: 20 }]; - return new Response(JSON.stringify({ segments }), { status: 200 }); - }); - vi.stubGlobal("fetch", fetchMock); - try { - const mgr = new WhisperServerManager(); - (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).process = {}; - (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).port = 9999; - (mgr as unknown as { process: unknown; port: number; vadAvailable: boolean }).vadAvailable = - true; - - // Total duration = 200 seconds @ 16 kHz = 3,200,000 samples (> MAX_VAD_CHUNK_SAMPLES = 2,880,000) - const samples = new Float32Array(MAX_VAD_CHUNK_SAMPLES + 16_000 * 20); - const segments = await mgr.detectVadSegments({ samples }); - - expect(fetchMock).toHaveBeenCalledTimes(2); - // 180s in chunk 0 + 20s in chunk 1 with offset 180s -> merged [10, 200] - expect(segments).toEqual([{ startSec: 10, endSec: 200 }]); - } finally { - vi.unstubAllGlobals(); - } - }); }); diff --git a/electron/stt/whisperServer.ts b/electron/stt/whisperServer.ts index a9698674f8..2543a79bb1 100644 --- a/electron/stt/whisperServer.ts +++ b/electron/stt/whisperServer.ts @@ -6,15 +6,19 @@ import os from "node:os"; import path from "node:path"; import type { Readable } from "node:stream"; -import { resolveBinaryPath } from "./gpuDetector"; -import { snapWordBoundariesToAudio } from "./snapWordBoundaries"; import { - STT_VAD_UNAVAILABLE, - type SttBackend, - type SttPhraseSegment, - type SttTiming, - type SttVadSegment, - type SttWordSegment, + alignWordsOnEmissions, + type EmissionsJson, + emissionRegions, + parseEmissions, +} from "./ctcAlign"; +import { resolveBinaryPath } from "./gpuDetector"; +import { anchorWordsOnSpeech } from "./snapWordBoundaries"; +import type { + SttBackend, + SttPhraseSegment, + SttTiming, + SttWordSegment, } from "./transcriptionContract"; import { cleanupWav, writeSamplesAsWav } from "./wav"; @@ -76,7 +80,13 @@ export interface WhisperServerStatus { backend: SttBackend | null; startedAtMs: number | null; lastError: string | null; - vadAvailable: boolean; +} + +export interface TranscribeOptions { + samples: Float32Array; + language?: string; + /** The aligner model for a language, or null for none (see `transcribe`). */ + alignerFor?: (language: string) => Promise; } /** Per-word entry inside a whisper-stt-server `/inference` JSON segment. */ @@ -84,6 +94,8 @@ interface WhisperJsonWord { word?: string; start?: number; end?: number; + /** End of the word's first token: always inside the word (see snapWordBoundaries.ts). */ + anchor?: number; probability?: number; } @@ -115,6 +127,8 @@ interface WhisperJsonResponse { detected_language?: string; backend?: string; timing?: WhisperJsonTiming; + /** Silero VAD's speech intervals; absent when the helper runs without the VAD model. */ + speech?: Array<{ start?: number | string; end?: number | string }>; } export class WhisperServerManager { @@ -125,11 +139,7 @@ export class WhisperServerManager { private lastError: string | null = null; private startedAtMs: number | null = null; private inFlight: Promise = Promise.resolve(); - private starting: Promise<{ - port: number; - backend: SttBackend; - vadAvailable: boolean; - }> | null = null; + private starting: Promise<{ port: number; backend: SttBackend }> | null = null; /** Buffered stderr from the helper; surfaced on shutdown + poll failures. */ private stderrTail = ""; @@ -154,12 +164,12 @@ export class WhisperServerManager { }); } - /** Check the server's HTTP root for a 200; resolves once responsive. Returns VAD availability. */ + /** Check the server's HTTP root for a 200; resolves once responsive. */ private static async pollUntilReady( baseUrl: string, timeoutMs = 30_000, shouldContinue: () => boolean = () => true, - ): Promise<{ vad: boolean }> { + ): Promise { const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { if (!shouldContinue()) throw new Error("whisper-stt-server exited before readiness"); @@ -170,10 +180,7 @@ export class WhisperServerManager { ); try { const res = await fetch(baseUrl, { method: "GET", signal: controller.signal }); - if (res.ok) { - const body = typeof res.json === "function" ? await res.json().catch(() => null) : null; - return { vad: Boolean((body as { vad?: boolean } | null)?.vad) }; - } + if (res.ok) return; } catch { // not up yet } finally { @@ -208,8 +215,6 @@ export class WhisperServerManager { if (log) console.error(`[stt] ${message}`); } - private vadAvailable = false; - /** True when a process is alive and a model is loaded. */ get status(): WhisperServerStatus { return { @@ -219,22 +224,15 @@ export class WhisperServerManager { backend: this.backend, startedAtMs: this.startedAtMs, lastError: this.lastError, - vadAvailable: this.vadAvailable, }; } - get isVadAvailable(): boolean { - return this.vadAvailable; - } - /** * Spawn the helper if not running and return once `/` returns 200. Idempotent — * if a server is already up we just return its port so the caller never pays * the cold-start cost twice. */ - async start( - options: WhisperServerStartOptions, - ): Promise<{ port: number; backend: SttBackend; vadAvailable: boolean }> { + async start(options: WhisperServerStartOptions): Promise<{ port: number; backend: SttBackend }> { if (this.starting) return this.starting; this.starting = this.startImpl(options).finally(() => { this.starting = null; @@ -244,7 +242,7 @@ export class WhisperServerManager { private async startImpl( options: WhisperServerStartOptions, - ): Promise<{ port: number; backend: SttBackend; vadAvailable: boolean }> { + ): Promise<{ port: number; backend: SttBackend }> { if (this.shuttingDown) { throw new Error("whisper-stt-server manager is shutting down"); } @@ -252,7 +250,6 @@ export class WhisperServerManager { return { port: this.port, backend: this.backend ?? options.backend ?? "whispercpp-cpu", - vadAvailable: this.vadAvailable, }; } @@ -281,9 +278,7 @@ export class WhisperServerManager { throw new Error(`Whisper GGML model not found at ${options.modelPath}`); } - const launch = async ( - forceCpu: boolean, - ): Promise<{ port: number; backend: SttBackend; vadAvailable: boolean }> => { + const launch = async (forceCpu: boolean): Promise<{ port: number; backend: SttBackend }> => { const port = await WhisperServerManager.pickFreePort(); if (this.shuttingDown) { throw new Error("whisper-stt-server manager is shutting down"); @@ -331,7 +326,6 @@ export class WhisperServerManager { this.process = null; this.port = null; this.startedAtMs = null; - this.vadAvailable = false; } }); child.once("error", (err) => { @@ -347,7 +341,6 @@ export class WhisperServerManager { this.process = null; this.port = null; this.startedAtMs = null; - this.vadAvailable = false; } }); @@ -362,9 +355,8 @@ export class WhisperServerManager { }); child.once("error", reject); }); - let readyInfo: { vad: boolean }; try { - readyInfo = await Promise.race([ + await Promise.race([ WhisperServerManager.pollUntilReady( `http://127.0.0.1:${port}`, 30_000, @@ -384,8 +376,7 @@ export class WhisperServerManager { this.recordError(message, { log: !alreadyLogged }); throw new Error(message); } - this.vadAvailable = Boolean(readyInfo?.vad); - return { port, backend: activeBackend, vadAvailable: this.vadAvailable }; + return { port, backend: activeBackend }; }; try { @@ -411,7 +402,6 @@ export class WhisperServerManager { /** Send SIGTERM and wait for the helper to exit. Resolves even if it was already down. */ async stop(): Promise { - this.vadAvailable = false; if (!this.process) { this.port = null; this.startedAtMs = null; @@ -464,6 +454,33 @@ export class WhisperServerManager { } } + /** + * The CTC aligner's letter scores for `regions` of the same upload + * (`/emissions`, see ctcAlign.ts). Throws on any failure, including a helper + * built before the endpoint existed (404): the caller keeps whisper's times. + */ + private async runEmissions(opts: { + wavPath: string; + modelPath: string; + regions: Array<[number, number]>; + }): Promise { + const form = new FormData(); + const fileBuffer = await readFile(opts.wavPath); + form.set("file", new Blob([fileBuffer], { type: "audio/wav" }), path.basename(opts.wavPath)); + form.set("model", opts.modelPath); + form.set("regions", JSON.stringify(opts.regions)); + const res = await fetch(`${this.baseUrl()}/emissions`, { + method: "POST", + body: form, + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }); + if (!res.ok) { + const text = await res.text().catch(() => ""); + throw new Error(`whisper-stt-server /emissions HTTP ${res.status}: ${text.slice(0, 512)}`); + } + return (await res.json()) as EmissionsJson; + } + private async runMultipartInfer(opts: { wavPath: string; language?: string; @@ -564,8 +581,16 @@ export class WhisperServerManager { return { elapsedSec, audioSec, rtf }; } - /** Run one transcription; serializes concurrent callers. */ - async transcribe(opts: { samples: Float32Array; language?: string }): Promise<{ + /** + * Run one transcription; serializes concurrent callers. + * + * `alignerFor` resolves the CTC aligner model for the language whisper + * detected, or null when there is none or it is not ready yet (it must not + * wait on the network: no download runs inside a chunk, see SttManager). With one, + * the words are re-timed on it before their phrase edges go on the speech + * (ctcAlign.ts); any failure there keeps whisper's own times. + */ + async transcribe(opts: TranscribeOptions): Promise<{ segments: SttPhraseSegment[]; wordSegments: SttWordSegment[]; detectedLanguage: string; @@ -578,7 +603,7 @@ export class WhisperServerManager { return task; } - private async transcribeImpl(opts: { samples: Float32Array; language?: string }): Promise<{ + private async transcribeImpl(opts: TranscribeOptions): Promise<{ segments: SttPhraseSegment[]; wordSegments: SttWordSegment[]; detectedLanguage: string; @@ -597,142 +622,69 @@ export class WhisperServerManager { return { text, startSec, endSec: Math.max(endSec, startSec + 0.05) }; }) .filter((s) => s.text.length > 0); - // whisper.cpp's DTW boundaries run ~80–150 ms behind the audio, which the - // transcript editor turns into imprecise trims (see snapWordBoundaries.ts). - // Re-anchor them on the same samples whisper was given. - const wordSegments: SttWordSegment[] = snapWordBoundariesToAudio( - raw - .flatMap((seg) => - (seg.words ?? []).map((w) => { - const word = (w.word ?? "").trim(); - const startSec = this.toSec(w.start, 0); - const endSec = this.toSec(w.end, startSec + 0.05); - const confidence = typeof w.probability === "number" ? w.probability : undefined; - return { word, startSec, endSec: Math.max(startSec + 0.02, endSec), confidence }; - }), - ) - .filter((w) => w.word.length > 0), - opts.samples, - ); + const speech = json.speech?.map((s) => ({ + startSec: this.toSec(s.start, 0), + endSec: this.toSec(s.end, 0), + })); const detectedLanguage = json.detected_language ?? json.language ?? "auto"; + let words = raw + .flatMap((seg) => + (seg.words ?? []).map((w) => { + const word = (w.word ?? "").trim(); + const startSec = this.toSec(w.start, 0); + const endSec = this.toSec(w.end, startSec + 0.05); + const anchorSec = this.toSec(w.anchor, startSec); + const confidence = typeof w.probability === "number" ? w.probability : undefined; + return { + word, + startSec, + endSec: Math.max(startSec + 0.02, endSec), + anchorSec, + confidence, + }; + }), + ) + .filter((w) => w.word.length > 0); + let alignSec = 0; const backend = this.toBackend(json.backend); - const timing = this.toTiming(json.timing); - return { segments, wordSegments, detectedLanguage, backend, timing }; - } finally { - await cleanupWav(wavPath); - } - } - - /** Run Voice Activity Detection (Silero VAD) to detect speech intervals. */ - async detectVadSegments(opts: { samples: Float32Array }): Promise { - const task = this.inFlight.then(() => this.detectVadSegmentsImpl(opts)); - this.inFlight = task.catch(() => undefined); - return task; - } - - private async detectVadChunk(samples: Float32Array): Promise { - const wavPath = await writeSamplesAsWav(samples); - try { - const url = `${this.baseUrl()}/vad`; - const form = new FormData(); - const fileBuffer = await readFile(wavPath); - const blob = new Blob([fileBuffer], { type: "audio/wav" }); - form.set("file", blob, path.basename(wavPath)); - - let res: Response; - try { - res = await fetch(url, { - method: "POST", - body: form, - signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), - }); - } catch (error) { - throw Object.assign( - new Error( - `whisper-stt-server /vad failed: ` + - `${error instanceof Error ? error.message : String(error)}; ` + - `stderr=${this.stderrTail.slice(-256)}`, - ), - { cause: error }, - ); + // GPU only. On the CPU the aligner's forward pass costs +29% (English) to + // +58% (French) of a transcription, against +9% to +14% on Vulkan, and + // character-level DTW alone is already within 16 ms median there + // (tools/stt-eval/word-timing, issue #948). Not asking also means a + // CPU-only machine never downloads the model. + if (speech?.length && words.length && opts.alignerFor && backend !== "whispercpp-cpu") { + try { + const modelPath = await opts.alignerFor(detectedLanguage); + if (modelPath) { + const reply = await this.runEmissions({ + wavPath, + modelPath, + regions: emissionRegions(speech, opts.samples.length / 16_000), + }); + const emissions = parseEmissions(reply); + if (!emissions) throw new Error("unreadable /emissions reply"); + words = alignWordsOnEmissions(words, speech, emissions); + alignSec = this.toSec(reply.elapsed_s, 0); + } + } catch (error) { + console.warn( + `[stt] word aligner failed, keeping whisper's word times: ${error instanceof Error ? error.message : String(error)}`, + ); + } } - - if (!res.ok) { - const text = await res.text().catch(() => ""); - throw new Error(`whisper-stt-server /vad HTTP ${res.status}: ${text.slice(0, 512)}`); + // Phrase edges go on the helper's speech intervals, aligner or not: the + // VAD is the better judge of where speech starts after a pause (see + // snapWordBoundaries.ts). + const wordSegments: SttWordSegment[] = anchorWordsOnSpeech(words, speech); + const timing = this.toTiming(json.timing); + // The aligner is part of what this chunk cost. + if (timing && alignSec > 0) { + timing.elapsedSec += alignSec; + timing.rtf = timing.elapsedSec / timing.audioSec; } - - const json = (await res.json().catch((error: unknown) => { - throw Object.assign( - new Error( - `whisper-stt-server /vad response was unreadable: ` + - `${error instanceof Error ? error.message : String(error)}`, - ), - { cause: error }, - ); - })) as { segments?: Array<{ start?: number | string; end?: number | string }> }; - - const raw = json.segments ?? []; - return raw.map((seg) => { - const startSec = this.toSec(seg.start, 0); - const endSec = this.toSec(seg.end, startSec); - return { startSec, endSec }; - }); + return { segments, wordSegments, detectedLanguage, backend, timing }; } finally { await cleanupWav(wavPath); } } - - private async detectVadSegmentsImpl(opts: { samples: Float32Array }): Promise { - await this.ensureReady(); - if (!this.vadAvailable) { - throw new Error(STT_VAD_UNAVAILABLE); - } - if (opts.samples.length === 0) return []; - - const allSegments: SttVadSegment[] = []; - const totalSamples = opts.samples.length; - for (let offset = 0; offset < totalSamples; offset += MAX_VAD_CHUNK_SAMPLES) { - const chunkSamples = opts.samples.subarray( - offset, - Math.min(totalSamples, offset + MAX_VAD_CHUNK_SAMPLES), - ); - const chunkSegments = await this.detectVadChunk(chunkSamples); - const offsetSec = offset / 16_000; - for (const seg of chunkSegments) { - allSegments.push({ - startSec: Math.round((seg.startSec + offsetSec) * 1000) / 1000, - endSec: Math.round((seg.endSec + offsetSec) * 1000) / 1000, - }); - } - } - - return mergeVadIntervals(allSegments); - } -} - -/** - * Silero VAD processes audio quickly, but buffering hours of 16 kHz audio as - * single WAV blobs can exhaust Node memory. We bound individual VAD passes to - * chunks of at most 3 minutes (180s = 2,880,000 samples @ 16 kHz), keeping - * memory under ~6 MB per WAV, and merge the resulting intervals. - */ -export const MAX_VAD_CHUNK_SAMPLES = 16_000 * 180; - -/** - * Merges contiguous or overlapping speech segments into non-overlapping intervals. - */ -export function mergeVadIntervals(segments: SttVadSegment[]): SttVadSegment[] { - if (segments.length <= 1) return segments.filter((seg) => seg.endSec > seg.startSec); - const merged: SttVadSegment[] = []; - for (const seg of segments) { - if (seg.endSec <= seg.startSec) continue; - const last = merged[merged.length - 1]; - if (last && seg.startSec <= last.endSec) { - last.endSec = Math.max(last.endSec, seg.endSec); - } else { - merged.push({ startSec: seg.startSec, endSec: seg.endSec }); - } - } - return merged; } diff --git a/electron/windows.ts b/electron/windows.ts index 0faaef02f0..72f2fe938a 100644 --- a/electron/windows.ts +++ b/electron/windows.ts @@ -505,6 +505,10 @@ export function createEditorWindow(query: Record = {}): BrowserW contextIsolation: true, webSecurity: false, backgroundThrottling: false, + // Media track lists: the preview's hidden
+ + ) : ( +
+
+
{t("exportDialog.frameRate")}
+
+ {GIF_FRAME_RATES.map((r) => ( + + ))} +
+
+
+
{t("exportDialog.size")}
+
+ {(Object.keys(GIF_SIZE_PRESETS) as GifSizePreset[]).map((s) => ( + + ))} +
+
+
+ {t("exportDialog.loopGif")} + +
+
+ {gifFrameRate} FPS · {gifSizeLabel} ·{" "} + {gifLoop ? t("exportDialog.loopOn") : t("exportDialog.loopOff")} +
+
+ )} - {t("exportDialog.pickFormatAndExport")} - - ); - } + // Nothing to say before an export: the format is the first control on screen and + // already picked. The old "Pick a format and press Export to start" plate was written + // for the UI that hid the format toggle under Advanced, and stayed up through the save + // picker where it was simply false. + if (phase === "idle" || phase === "configuring") return null; if (phase === "done") { return (
- - , + + + + + , ); // Flush the session load so the assistant bubble (and its copy button) is on screen. await waitFor(() => { - expect(view.getAllByTitle("chat.copyMessage").length).toBeGreaterThan(0); + expect(view.getAllByRole("button", { name: "chat.copyMessage" }).length).toBeGreaterThan(0); }); return view; } +const copyButtonsOf = (view: ReturnType) => + view.getAllByRole("button", { name: "chat.copyMessage" }); + describe("ChatStripPanel Copy message (issue #738)", () => { it("copies the full message through the preload bridge and reports success, never touching the denied navigator clipboard", async () => { stubClipboard(copyToClipboard); const view = await renderPanelWithAssistantMessage(); // Two copy buttons (user + assistant); the assistant message is the last one rendered. - const copyButtons = view.getAllByTitle("chat.copyMessage"); + const copyButtons = copyButtonsOf(view); // user + assistant + the empty trailing assistant turn. expect(copyButtons).toHaveLength(3); await act(async () => { @@ -193,7 +199,7 @@ describe("ChatStripPanel Copy message (issue #738)", () => { stubClipboard(undefined); const view = await renderPanelWithAssistantMessage(); - const copyButtons = view.getAllByTitle("chat.copyMessage"); + const copyButtons = copyButtonsOf(view); // The fallback inherits Electron's denial in a bare context: the click // lands on the navigator path and reports the failure toast. await act(async () => { @@ -229,7 +235,7 @@ describe("ChatStripPanel Copy message (issue #738)", () => { // copy button sits last. The handler must not branch on content — an // empty string crosses to the bridge exactly like any other content and // still reports success. - const copyButtons = view.getAllByTitle("chat.copyMessage"); + const copyButtons = copyButtonsOf(view); await act(async () => { fireEvent.click(copyButtons[copyButtons.length - 1]); await Promise.resolve(); diff --git a/src/components/ai-edition/LeftPanel.providerRefresh.test.tsx b/src/components/ai-edition/LeftPanel.providerRefresh.test.tsx index f14a8d73c2..0501e50932 100644 --- a/src/components/ai-edition/LeftPanel.providerRefresh.test.tsx +++ b/src/components/ai-edition/LeftPanel.providerRefresh.test.tsx @@ -45,6 +45,7 @@ vi.mock("@/contexts/I18nContext", () => ({ useScopedT: () => (key: string) => key, })); +import { TooltipProvider } from "@/components/ui/tooltip"; import { EditorDialogsProvider, useEditorDialogActions } from "@/contexts/EditorDialogsContext"; import { ChatStripPanel } from "./LeftPanel"; @@ -80,10 +81,12 @@ afterEach(() => { describe("ChatStripPanel, against the lifted provider dialog", () => { it("re-reads the LLM snapshot when the dialog closes, and not when it opens", async () => { render( - - - - , + + + + + + , ); // Mount: the dialog is closed, so the same effect that watches for a close seeds the // composer's view of the provider config. diff --git a/src/components/ai-edition/LeftPanel.tooltips.test.tsx b/src/components/ai-edition/LeftPanel.tooltips.test.tsx new file mode 100644 index 0000000000..9796e9b642 --- /dev/null +++ b/src/components/ai-edition/LeftPanel.tooltips.test.tsx @@ -0,0 +1,220 @@ +// @vitest-environment jsdom +// The chat panel's icon-only controls name themselves in the shared tooltip, not in a native +// `title`. Rewind and Compact say what they do, Send shows Enter as a chip, and a button that is +// greyed for now (Send with no provider, Compact with no conversation) still opens its tooltip: +// that is where "Set up a provider" is said, and a natively disabled button takes no pointer +// events, so it uses `aria-disabled`. + +import "@testing-library/jest-dom"; +import { act, cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const snapshot = vi.hoisted(() => ({ + value: { + config: null as { provider: string; model: string } | null, + connectedProviders: [] as string[], + availableProviders: [] as unknown[], + credentialSummary: [] as unknown[], + }, +})); +const sessions = vi.hoisted(() => ({ list: [] as unknown[] })); +const chatCompact = vi.hoisted(() => vi.fn()); + +vi.mock("@/native/client", () => ({ + nativeBridgeClient: { + aiEdition: { + llmGetSnapshot: () => Promise.resolve(snapshot.value), + chatListSessions: () => Promise.resolve(sessions.list), + chatSelectSession: () => + Promise.resolve({ + id: "session-1", + projectId: "project-1", + title: "Conversation 1", + createdAt: "2026-09-23T00:00:00Z", + updatedAt: "2026-09-23T00:00:00Z", + messages: [ + { + id: "m1", + role: "user", + content: "Cut the silences", + checkpointId: "cp1", + createdAt: "2026-09-23T00:00:01Z", + }, + { + id: "m2", + role: "assistant", + content: "Done", + createdAt: "2026-09-23T00:00:02Z", + }, + ], + }), + chatBudget: () => Promise.resolve(null), + chatCompact, + llmListProviderModels: () => Promise.resolve({ models: [] }), + }, + }, +})); + +vi.mock("sonner", () => ({ toast: { success: vi.fn(), error: vi.fn(), info: vi.fn() } })); + +// An echoing translator: the assertions read against keys, not prose that drifts with copy edits. +vi.mock("@/contexts/I18nContext", () => ({ + useI18n: () => ({ locale: "en", setLocale: vi.fn() }), + useScopedT: () => (key: string) => key, +})); + +import { TooltipProvider } from "@/components/ui/tooltip"; +import { EditorDialogsProvider } from "@/contexts/EditorDialogsContext"; +import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; +import { ChatStripPanel } from "./LeftPanel"; + +class StubResizeObserver { + observe = vi.fn(); + unobserve = vi.fn(); + disconnect = vi.fn(); +} + +const oneSession = [ + { id: "session-1", title: "Conversation 1", messageCount: 2, createdAt: "2026-09-23T00:00:00Z" }, +]; + +beforeEach(() => { + vi.stubGlobal("ResizeObserver", StubResizeObserver); + // jsdom implements no scrolling, and the transcript pins itself to the bottom on every render. + Element.prototype.scrollTo = vi.fn(); + (window as unknown as { electronAPI?: unknown }).electronAPI = { + onAiEditionChatEvent: () => vi.fn(), + }; + useProjectStore.setState({ projectId: "project-1" }); + snapshot.value = { + config: null, + connectedProviders: [], + availableProviders: [], + credentialSummary: [], + }; + sessions.list = []; + chatCompact.mockClear(); +}); + +afterEach(() => { + cleanup(); + vi.unstubAllGlobals(); + (window as unknown as { electronAPI?: unknown }).electronAPI = undefined; +}); + +function renderPanel() { + return render( + + + + + , + ); +} + +/** Opens a control's tooltip the way the keyboard does (focus opens it at once) and returns what + * it says and whether it shows a shortcut chip. */ +async function tooltipOn(control: HTMLElement) { + act(() => control.focus()); + const text = (await screen.findByRole("tooltip")).textContent; + const chip = document.querySelector('[data-slot="tooltip-content"] kbd')?.textContent ?? null; + act(() => control.blur()); + await waitFor(() => expect(screen.queryByRole("tooltip")).toBeNull()); + return { text, chip }; +} + +describe("ChatStripPanel tooltips", () => { + it("names the header buttons in the shared tooltip, with no native title", async () => { + sessions.list = oneSession; + renderPanel(); + await screen.findByRole("button", { name: "chat.deleteConversation" }); + + for (const name of [ + "chat.compactContext", + "chat.aiSettings", + "chat.history", + "chat.newConversation", + "chat.renameConversation", + "chat.deleteConversation", + ]) { + const button = screen.getByRole("button", { name }); + expect(button).not.toHaveAttribute("title"); + expect((await tooltipOn(button)).text).toBe(name); + } + }); + + // Plain text has no interactive element to hold a tooltip, so its hint stays native. + it("keeps the native title only on plain text", async () => { + sessions.list = oneSession; + renderPanel(); + expect(await screen.findByTitle("chat.clickToRename")).toBeInTheDocument(); + }); + + it("says what Rewind does, not only what the button is called", async () => { + sessions.list = oneSession; + renderPanel(); + const rewind = await screen.findByRole("button", { name: "chat.rewindToMessage" }); + expect(rewind).not.toHaveAttribute("title"); + expect((await tooltipOn(rewind)).text).toBe("chat.rewindTip"); + }); + + it("names Copy in the shared tooltip too", async () => { + sessions.list = oneSession; + renderPanel(); + const [copy] = await screen.findAllByRole("button", { name: "chat.copyMessage" }); + expect(copy).not.toHaveAttribute("title"); + expect((await tooltipOn(copy)).text).toBe("chat.copyMessage"); + }); + + // Nothing to talk to yet: Send is greyed, but it is where the reason is said. + it("says why Send is off when no provider is set up, and shows no chip then", async () => { + renderPanel(); + const send = await screen.findByRole("button", { name: "chat.send" }); + await waitFor(() => expect(send).toHaveAttribute("aria-disabled", "true")); + expect(send).not.toBeDisabled(); + expect(send).not.toHaveAttribute("title"); + const tip = await tooltipOn(send); + expect(tip.text).toBe("chat.composerDisabledNoProvider"); + expect(tip.chip).toBeNull(); + }); + + it("shows Enter as a chip on Send once it can send, never inside the string", async () => { + snapshot.value = { + config: { provider: "openai", model: "gpt-x" }, + connectedProviders: ["openai"], + availableProviders: [], + credentialSummary: [], + }; + renderPanel(); + fireEvent.change(await screen.findByPlaceholderText("chat.composerPlaceholder"), { + target: { value: "Cut the silences" }, + }); + const send = screen.getByRole("button", { name: "chat.send" }); + await waitFor(() => expect(send).not.toHaveAttribute("aria-disabled")); + const tip = await tooltipOn(send); + expect(tip.text).toBe("chat.sendEnter"); + expect(tip.chip).toBe("Enter"); + }); + + // While the title is being edited the button is greyed, but it stays hoverable and does nothing. + it("keeps Rename tooltip-able while the title is being edited", async () => { + sessions.list = oneSession; + renderPanel(); + const rename = await screen.findByRole("button", { name: "chat.renameConversation" }); + fireEvent.click(rename); + await screen.findByDisplayValue("Conversation 1"); + expect(rename).toHaveAttribute("aria-disabled", "true"); + expect(rename).not.toBeDisabled(); + expect((await tooltipOn(rename)).text).toBe("chat.renameConversation"); + }); + + it("keeps Compact tooltip-able while there is no conversation, and does nothing", async () => { + renderPanel(); + const compact = await screen.findByRole("button", { name: "chat.compactContext" }); + expect(compact).toHaveAttribute("aria-disabled", "true"); + expect(compact).not.toBeDisabled(); + expect((await tooltipOn(compact)).text).toBe("chat.compactContext"); + fireEvent.click(compact); + expect(chatCompact).not.toHaveBeenCalled(); + }); +}); diff --git a/src/components/ai-edition/LeftPanel.tsx b/src/components/ai-edition/LeftPanel.tsx index 81c62c3069..82e8092683 100644 --- a/src/components/ai-edition/LeftPanel.tsx +++ b/src/components/ai-edition/LeftPanel.tsx @@ -2,6 +2,7 @@ import { ArrowLeft, Check, Loader2, X } from "lucide-react"; import { useCallback, useEffect, useRef, useState } from "react"; import { createPortal } from "react-dom"; import { toast } from "sonner"; +import { Tooltip } from "@/components/ui/tooltip"; import { useEditorDialogActions, useEditorDialogSection } from "@/contexts/EditorDialogsContext"; import { useScopedT } from "@/contexts/I18nContext"; import { @@ -471,6 +472,7 @@ export function ChatStripPanel() { const [connectedProviders, setConnectedProviders] = useState(null); // unknown ≠ none; see chatAvailability.ts. const canChat = canSendChat(llmConfig, connectedProviders); + const sendDisabled = busy || !input.trim() || !canChat; const [modelPopoverOpen, setModelPopoverOpen] = useState(false); const modelButtonRef = useRef(null); const [modelPopoverRect, setModelPopoverRect] = useState<{ @@ -1006,92 +1008,95 @@ export function ChatStripPanel() { {t("chat.contextPercent", { percent: Math.min(100, Math.round(budget.ratio * 100)) })} - - - + + + - + + + + + + + + + + + + +
@@ -1160,61 +1165,66 @@ export function ChatStripPanel() { t("chat.untitledConversation")} )} - - + + + + + + + + + + + + ) : null} @@ -1293,21 +1303,59 @@ export function ChatStripPanel() { }} > {m.role === "user" && m.checkpointId ? ( + // The name stays "Rewind to this message"; the tip adds what it does, which + // only the confirmation dialog said before. + + + + ) : null} + - ) : null} - + {m.toolCalls?.length ? (
@@ -1546,28 +1560,34 @@ export function ChatStripPanel() { onOpenFullSettings={openProviderSettings} /> ) : null} - + + + + + +
{ + vi.spyOn(SVGElement.prototype, "getBoundingClientRect").mockReturnValue( + DOMRect.fromRect({ x: 0, y: 0, width: 1000, height: 500 }), + ); +}); +afterAll(() => vi.restoreAllMocks()); + +const BLUR = { + id: "b", + startMs: 0, + endMs: 1000, + type: "blur", + content: "", + position: { x: 10, y: 20 }, + size: { width: 30, height: 40 }, + style: {}, + zIndex: 1, + blurData: { type: "mosaic", shape: "rectangle", color: "white", intensity: 12, blockSize: 12 }, +} as unknown as AxcutAnnotationRegion; + +// The footage zoomed ×2 around its middle: one unit of footage is 1.6 units of frame. +const ZOOMED = rectQuad(-0.3, -0.3, 1.6, 1.6); + +function gimbal() { + const onChange = vi.fn(); + const onCommit = vi.fn(); + const onClick = vi.fn(); + const { container } = render( + , + ); + return { container, onChange, onCommit, onClick }; +} + +const at = (x: number, y: number) => ({ clientX: x, clientY: y, pointerId: 1 }); + +describe("MaskGimbal", () => { + it("moves the mask by what the pointer covers of the footage, zoomed", () => { + const { container, onChange, onCommit } = gimbal(); + const body = container.querySelector("polygon") as SVGPolygonElement; + fireEvent.pointerDown(body, at(300, 200)); + // 160 px of a 1000 px frame, zoomed ×2 over 1.6: a tenth of the footage. + fireEvent.pointerMove(body, at(460, 200)); + fireEvent.pointerUp(body, at(460, 200)); + const last = onChange.mock.lastCall?.[1]; + expect(last.position.x).toBeCloseTo(20, 9); + expect(last.position.y).toBeCloseTo(20, 9); + expect(onCommit).toHaveBeenCalledTimes(1); + }); + + it("resizes from a corner while the opposite one stays", () => { + const { container, onChange } = gimbal(); + const bottomRight = container.querySelectorAll("circle")[2]; + fireEvent.pointerDown(bottomRight, at(0, 0)); + // Frame point (0.5, 0.5) is the middle of the footage. + fireEvent.pointerMove(bottomRight, at(500, 250)); + const last = onChange.mock.lastCall?.[1]; + expect(last.position).toEqual({ x: 10, y: 20 }); + expect(last.size.width).toBeCloseTo(40, 9); + expect(last.size.height).toBeCloseTo(30, 9); + }); + + it("selects through on a click that moves nothing", () => { + const { container, onChange, onCommit, onClick } = gimbal(); + const body = container.querySelector("polygon") as SVGPolygonElement; + fireEvent.pointerDown(body, at(300, 200)); + fireEvent.pointerUp(body, at(300, 200)); + expect(onChange).not.toHaveBeenCalled(); + expect(onCommit).not.toHaveBeenCalled(); + expect(onClick).toHaveBeenCalledWith("b"); + }); +}); diff --git a/src/components/ai-edition/MaskGimbal.tsx b/src/components/ai-edition/MaskGimbal.tsx new file mode 100644 index 0000000000..35645f154d --- /dev/null +++ b/src/components/ai-edition/MaskGimbal.tsx @@ -0,0 +1,189 @@ +// Le gimbal d'un flou de confidentialité. Le masque suit le contenu : le compositeur le pose sur +// le métrage zoomé, et incliné sous la 3D (`privacy_mask`). Le gimbal passe donc par le métrage +// de l'image (`FootageQuad`) : contour, poignées et gestes vivent dans le repère du métrage, et +// chaque point est projeté comme le compositeur le peint. Un `Rnd` droit, posé sur le métrage au +// repos, s'écartait du masque dès le premier zoom. + +import type { PointerEvent as ReactPointerEvent } from "react"; +import { useRef } from "react"; +import { + type FootageQuad, + footageAt, + footagePoint, +} from "@/lib/ai-edition/annotations/footageQuad"; +import type { AxcutAnnotationRegion } from "@/lib/ai-edition/schema"; + +type Region = AxcutAnnotationRegion; + +interface MaskGimbalProps { + annotation: Region; + /** Le métrage dans l'image, en fractions de l'image. */ + quad: FootageQuad; + /** L'image de l'aperçu, px. */ + frameWidth: number; + frameHeight: number; + onChange: (id: string, patch: Partial) => void; + /** Écriture disque, une fois en fin de geste. */ + onCommit?: () => void; + onClick: (id: string) => void; + zIndex: number; +} + +const GREEN = "#34B27B"; +/** Le plus petit masque que laissent les poignées, en % du métrage. */ +const MIN_PCT = 1; +/** Les poignées dans l'ordre des coins : TL, TR, BR, BL. */ +const CORNER_CURSORS = ["nwse-resize", "nesw-resize", "nwse-resize", "nesw-resize"] as const; + +type Gesture = + | { + kind: "move"; + start: readonly [number, number]; + origin: { x: number; y: number }; + moved: boolean; + } + | { kind: "corner"; fixed: readonly [number, number]; moved: boolean }; + +const clamp = (value: number, lo: number, hi: number) => Math.min(hi, Math.max(lo, value)); + +export function MaskGimbal({ + annotation, + quad, + frameWidth, + frameHeight, + onChange, + onCommit, + onClick, + zIndex, +}: MaskGimbalProps) { + const svgRef = useRef(null); + const gesture = useRef(null); + const { position, size } = annotation; + // Le masque en fractions du métrage. + const [u0, v0] = [position.x / 100, position.y / 100]; + const [u1, v1] = [u0 + size.width / 100, v0 + size.height / 100]; + const onFrame = (u: number, v: number) => { + const [x, y] = footagePoint(quad, u, v); + return [x * frameWidth, y * frameHeight] as const; + }; + const corners = [ + [u0, v0], + [u1, v0], + [u1, v1], + [u0, v1], + ] as const; + // Un ovale se dessine en ovale, projeté ; le tracé libre, comme le masque, par sa boîte. + const outline = + annotation.blurData?.shape === "oval" + ? Array.from({ length: 48 }, (_, k) => { + const t = (k / 48) * 2 * Math.PI; + return onFrame( + u0 + (u1 - u0) * (0.5 + 0.5 * Math.cos(t)), + v0 + (v1 - v0) * (0.5 + 0.5 * Math.sin(t)), + ); + }) + : corners.map(([u, v]) => onFrame(u, v)); + + /** Le point du métrage sous le pointeur, en fractions du métrage. */ + const under = (event: ReactPointerEvent): readonly [number, number] => { + const r = svgRef.current?.getBoundingClientRect(); + if (!r || r.width <= 0 || r.height <= 0) return [0, 0]; + return footageAt(quad, (event.clientX - r.left) / r.width, (event.clientY - r.top) / r.height); + }; + + const begin = (event: ReactPointerEvent, next: Gesture) => { + event.stopPropagation(); + event.currentTarget.setPointerCapture?.(event.pointerId); + gesture.current = next; + }; + + const onPointerMove = (event: ReactPointerEvent) => { + const g = gesture.current; + if (!g) return; + const [u, v] = under(event); + if (g.kind === "move") { + const x = clamp(g.origin.x + (u - g.start[0]) * 100, 0, 100 - size.width); + const y = clamp(g.origin.y + (v - g.start[1]) * 100, 0, 100 - size.height); + g.moved ||= x !== g.origin.x || y !== g.origin.y; + // En direct : c'est le natif qui peint, il suit le geste. L'écriture disque attend + // le relâchement (`onCommit`). + onChange(annotation.id, { position: { x, y } }); + return; + } + // Le coin opposé reste en place ; la poignée peut passer de l'autre côté, sans que le + // masque descende sous le minimum ni sorte du métrage. + const span = (c: number, fixed: number): [number, number] => { + const [lo, hi] = + c < fixed ? [Math.min(c, fixed - MIN_PCT), fixed] : [fixed, Math.max(c, fixed + MIN_PCT)]; + const top = Math.min(hi, 100); + return [Math.max(0, Math.min(lo, top - MIN_PCT)), top]; + }; + const [x0, x1] = span(clamp(u * 100, 0, 100), g.fixed[0]); + const [y0, y1] = span(clamp(v * 100, 0, 100), g.fixed[1]); + g.moved = true; + onChange(annotation.id, { + position: { x: x0, y: y0 }, + size: { width: x1 - x0, height: y1 - y0 }, + }); + }; + + const onPointerUp = (event: ReactPointerEvent) => { + const g = gesture.current; + gesture.current = null; + if (!g) return; + (event.target as Element).releasePointerCapture?.(event.pointerId); + if (g.moved) onCommit?.(); + else if (g.kind === "move") onClick(annotation.id); + }; + + const points = (list: readonly (readonly [number, number])[]) => + list.map(([x, y]) => `${x},${y}`).join(" "); + + return ( + + + begin(event, { kind: "move", start: under(event), origin: { ...position }, moved: false }) + } + /> + {corners.map(([u, v], k) => { + const [cx, cy] = onFrame(u, v); + const [ou, ov] = corners[(k + 2) % 4]; + return ( + + begin(event, { kind: "corner", fixed: [ou * 100, ov * 100], moved: false }) + } + /> + ); + })} + + ); +} diff --git a/src/components/ai-edition/McpServerSettings.test.tsx b/src/components/ai-edition/McpServerSettings.test.tsx new file mode 100644 index 0000000000..453a4961d2 --- /dev/null +++ b/src/components/ai-edition/McpServerSettings.test.tsx @@ -0,0 +1,102 @@ +// @vitest-environment jsdom +import "@testing-library/jest-dom"; +import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { I18nProvider } from "@/contexts/I18nContext"; +import type { AiEditionMcpStatus } from "@/native/contracts"; + +const TOKEN = "secret-token"; +const URL = "http://127.0.0.1:47821/mcp"; + +function status(enabled: boolean, allowEdits = false): AiEditionMcpStatus { + return { + enabled, + port: 47821, + allowEdits, + running: enabled, + url: URL, + token: enabled ? TOKEN : null, + error: null, + }; +} + +const bridge = vi.hoisted(() => ({ + mcpGetStatus: vi.fn(), + mcpSetEnabled: vi.fn(), + mcpSetPort: vi.fn(), + mcpSetAllowEdits: vi.fn(), + mcpRegenerateToken: vi.fn(), +})); + +vi.mock("@/native/client", () => ({ nativeBridgeClient: { aiEdition: bridge } })); + +import { claudeCodeCommand, codexCommand, McpServerSettings } from "./McpServerSettings"; + +const copyToClipboard = vi.fn(() => Promise.resolve()); + +function renderSection() { + return render( + + + , + ); +} + +describe("McpServerSettings", () => { + beforeEach(() => { + for (const fn of Object.values(bridge)) fn.mockReset(); + copyToClipboard.mockClear(); + (window as unknown as { electronAPI: unknown }).electronAPI = { copyToClipboard }; + }); + afterEach(cleanup); + + it("shows the server as off, with no token or commands", async () => { + bridge.mcpGetStatus.mockResolvedValue(status(false)); + renderSection(); + expect(await screen.findByTestId("mcp-server-settings")).toBeInTheDocument(); + expect(screen.getByTestId("mcp-server-toggle")).toHaveAttribute("aria-pressed", "false"); + expect(screen.queryByText(/claude mcp add/)).not.toBeInTheDocument(); + }); + + it("turns the server on and shows how to connect, token masked", async () => { + bridge.mcpGetStatus.mockResolvedValue(status(false)); + bridge.mcpSetEnabled.mockResolvedValue(status(true)); + renderSection(); + fireEvent.click(await screen.findByTestId("mcp-server-toggle")); + await waitFor(() => expect(bridge.mcpSetEnabled).toHaveBeenCalledWith(true)); + expect(await screen.findByText(codexCommand(URL))).toBeInTheDocument(); + expect(screen.queryByText(new RegExp(TOKEN))).not.toBeInTheDocument(); + }); + + it("keeps MCP edits off until they are turned on", async () => { + bridge.mcpGetStatus.mockResolvedValue(status(true)); + bridge.mcpSetAllowEdits.mockResolvedValue(status(true, true)); + renderSection(); + const toggle = await screen.findByTestId("mcp-edits-toggle"); + expect(toggle).toHaveAttribute("aria-pressed", "false"); + fireEvent.click(toggle); + await waitFor(() => expect(bridge.mcpSetAllowEdits).toHaveBeenCalledWith(true)); + await waitFor(() => + expect(screen.getByTestId("mcp-edits-toggle")).toHaveAttribute("aria-pressed", "true"), + ); + }); + + it("copies the real token into the Claude Code command", async () => { + bridge.mcpGetStatus.mockResolvedValue(status(true)); + renderSection(); + await screen.findByText(codexCommand(URL)); + const copyButtons = screen.getAllByRole("button", { name: "Copy" }); + // Token, Claude Code, Codex — in that order. + fireEvent.click(copyButtons[1]); + expect(copyToClipboard).toHaveBeenCalledWith(claudeCodeCommand(URL, TOKEN)); + }); + + it("stays out of the way when the status cannot be read", async () => { + bridge.mcpGetStatus.mockImplementation(() => { + throw new Error("no bridge"); + }); + renderSection(); + await waitFor(() => expect(bridge.mcpGetStatus).toHaveBeenCalled()); + expect(screen.queryByTestId("mcp-server-settings")).not.toBeInTheDocument(); + }); +}); diff --git a/src/components/ai-edition/McpServerSettings.tsx b/src/components/ai-edition/McpServerSettings.tsx new file mode 100644 index 0000000000..785911bea2 --- /dev/null +++ b/src/components/ai-edition/McpServerSettings.tsx @@ -0,0 +1,311 @@ +// The MCP server section of AI settings: turn the local server on, pick its +// port, and copy what an MCP client needs to connect. The server itself lives +// in electron/mcp/; this only drives it through the native bridge. + +import { AlertCircle, Check, Copy, Eye, EyeOff, Loader2, RefreshCw } from "lucide-react"; +import { useCallback, useEffect, useState } from "react"; +import { toast } from "sonner"; +import { useScopedT } from "@/contexts/I18nContext"; +import { nativeBridgeClient } from "@/native/client"; +import type { AiEditionMcpStatus } from "@/native/contracts"; +import styles from "./NewEditorShell.module.css"; + +const CODEX_TOKEN_ENV = "OPENSCREEN_MCP_TOKEN"; +const MASKED_TOKEN = "••••••••"; + +export function claudeCodeCommand(url: string, token: string): string { + return `claude mcp add --transport http openscreen ${url} --header "Authorization: Bearer ${token}"`; +} + +export function codexCommand(url: string): string { + return `codex mcp add openscreen --url ${url} --bearer-token-env-var ${CODEX_TOKEN_ENV}`; +} + +// Commands and the token wrap onto as many lines as they need. `nowrap` here made +// the `1fr` grid column as wide as the longest command, which pushed the copy +// buttons out of the dialog and gave the whole modal a horizontal scrollbar. +const codeStyle: React.CSSProperties = { + flex: 1, + minWidth: 0, + whiteSpace: "pre-wrap", + overflowWrap: "anywhere", + padding: "6px 8px", + borderRadius: 6, + background: "var(--bg-2, rgba(127,127,127,0.12))", + font: "12px/1.5 var(--font-mono)", + color: "var(--fg-2)", + // A command is copied and typed as-is, so it has to render as-is: Geist Mono's + // contextual alternates drew " --header" as "--header", hiding the space. + fontVariantLigatures: "none", + fontFeatureSettings: '"liga" 0, "calt" 0', + userSelect: "text", +}; + +const rowStyle: React.CSSProperties = { + display: "flex", + alignItems: "flex-start", + gap: 8, + minWidth: 0, +}; + +export function McpServerSettings({ open }: { open: boolean }) { + const te = useScopedT("editor"); + const [status, setStatus] = useState(null); + const [portDraft, setPortDraft] = useState(""); + const [busy, setBusy] = useState(false); + const [showToken, setShowToken] = useState(false); + + const adopt = useCallback((next: AiEditionMcpStatus) => { + setStatus(next); + setPortDraft(String(next.port)); + }, []); + + useEffect(() => { + if (!open) { + setShowToken(false); + return; + } + // Any failure, even a synchronous one, just hides the section: it must never + // take the provider settings around it down with it. + void Promise.resolve() + .then(() => nativeBridgeClient.aiEdition.mcpGetStatus()) + .then(adopt, () => setStatus(null)); + }, [open, adopt]); + + const run = async (task: () => Promise) => { + setBusy(true); + try { + adopt(await task()); + } catch (err) { + toast.error(te("mcpServer.updateFailed"), { + description: err instanceof Error ? err.message : String(err), + }); + } finally { + setBusy(false); + } + }; + + const copy = (text: string) => { + const bridge = window.electronAPI?.copyToClipboard; + const write = bridge ? bridge(text) : navigator.clipboard.writeText(text); + void write.then( + () => toast.success(te("mcpServer.copied")), + () => toast.error(te("mcpServer.copyFailed")), + ); + }; + + if (!status) return null; + + const port = Number(portDraft); + const portChanged = portDraft !== String(status.port); + const token = status.token; + const shownToken = token && showToken ? token : MASKED_TOKEN; + + return ( +
+
+
+

{te("mcpServer.title")}

+ {status.running ? ( + + + {te("mcpServer.statusRunning")} + + ) : ( + + {status.enabled ? te("mcpServer.statusError") : te("mcpServer.statusOff")} + + )} +
+

{te("mcpServer.description")}

+ + run(() => nativeBridgeClient.aiEdition.mcpSetEnabled(!status.enabled))} + /> + + + run(() => nativeBridgeClient.aiEdition.mcpSetAllowEdits(!status.allowEdits)) + } + /> + +
+ +
+ setPortDraft(e.target.value)} + disabled={busy} + style={{ width: 120 }} + /> + {portChanged ? ( + + ) : null} +
+
+ + {status.error ? ( +

+ + {status.error} +

+ ) : null} + + {status.enabled && token ? ( + <> +
+ +
+ {shownToken} + setShowToken((v) => !v)} + > + {showToken ? : } + + copy(token)}> + + + run(() => nativeBridgeClient.aiEdition.mcpRegenerateToken())} + > + {busy ? : } + +
+

{te("mcpServer.tokenHint")}

+
+ +
+ +
+ {claudeCodeCommand(status.url, shownToken)} + copy(claudeCodeCommand(status.url, token))} + > + + +
+
+ +
+ +
+ {codexCommand(status.url)} + copy(codexCommand(status.url))} + > + + +
+

{te("mcpServer.codexHint", { env: CODEX_TOKEN_ENV })}

+
+ + ) : null} +
+
+ ); +} + +/** The panes' switch, not a system checkbox, inside its label so the text toggles it too. */ +function ToggleField({ + label, + text, + hint, + testId, + on, + disabled, + onToggle, +}: { + label: string; + text: string; + hint?: string; + testId: string; + on: boolean; + disabled: boolean; + onToggle: () => void; +}) { + return ( +
+ +
+ ); +} + +function IconButton({ + label, + onClick, + disabled, + children, +}: { + label: string; + onClick: () => void; + disabled?: boolean; + children: React.ReactNode; +}) { + return ( + + ); +} diff --git a/src/components/ai-edition/NativeCompositorOverlay.test.tsx b/src/components/ai-edition/NativeCompositorOverlay.test.tsx new file mode 100644 index 0000000000..6ee14ff156 --- /dev/null +++ b/src/components/ai-edition/NativeCompositorOverlay.test.tsx @@ -0,0 +1,390 @@ +// @vitest-environment jsdom +/** + * How the overlay steers the native view while the transport plays. + * + * The view runs its own clock and crosses clip boundaries by itself. The overlay used to + * re-send the clip at every cut anyway, which made the view seek back to a place it had just + * left, or drop the next clip it had preloaded: a hitch at every cut of an edited take. Now it + * reads where the view is from the position each frame reports, and only steps in when the + * view is somewhere else. + */ +import { act, cleanup, render } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { AxcutClip, AxcutDocument } from "@/lib/ai-edition/schema"; +import { axcutSchemaVersion } from "@/lib/ai-edition/schema"; +import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; +import { publishNativePosition } from "@/native/nativeSync"; + +const native = vi.hoisted(() => ({ + setActiveClip: vi.fn(async () => ({ ok: true })), + setNativePlaying: vi.fn(), + setNativeScene: vi.fn(), +})); +const i18n = vi.hoisted(() => ({ locale: "en" })); + +vi.mock("@/native", () => ({ + pushAllNativeParams: vi.fn(), + setActiveClip: native.setActiveClip, + setCurrentNativeViewId: vi.fn(), + setNativePlaying: native.setNativePlaying, + setNativeScene: native.setNativeScene, + subscribeNativeCompositor: () => () => undefined, + useIsCpuCompositor: () => false, + useNativeCompositorView: () => ({ viewId: 7, error: null }), +})); + +vi.mock("@/contexts/I18nContext", () => ({ + useI18n: () => ({ locale: i18n.locale }), + useScopedT: () => (key: string) => key, +})); + +const probe = vi.hoisted(() => ({ get: vi.fn((_path: string) => null) })); +vi.mock("@/native/webcamSizeCache", async (importOriginal) => ({ + ...(await importOriginal()), + getWebcamNativeSize: probe.get, +})); + +import { NativeCompositorOverlay } from "./NativeCompositorOverlay"; + +/** One take cut in two: source 0-5 s, then 7-12 s after a 2 s cut. */ +const FIRST: AxcutClip = { + id: "clip_a", + assetId: "asset_take", + sourceStartSec: 0, + sourceEndSec: 5, + timelineStartSec: 0, + timelineEndSec: 5, + wordRefs: [], + origin: "system", + reason: "", +}; +const SECOND: AxcutClip = { + ...FIRST, + id: "clip_b", + sourceStartSec: 7, + sourceEndSec: 12, + timelineStartSec: 5, + timelineEndSec: 10, +}; + +function makeDocument(): AxcutDocument { + return { + schemaVersion: axcutSchemaVersion, + project: { + id: "proj_sync", + title: "Sync", + createdAt: "2026-10-02T08:00:00.000Z", + updatedAt: "2026-10-02T08:00:00.000Z", + primaryAssetId: "asset_take", + }, + assets: [ + { + id: "asset_take", + kind: "video", + label: "take", + originalPath: "/take.mp4", + cameraTrack: null, + }, + ], + transcript: null, + transcripts: [], + timeline: { + clips: [FIRST, SECOND], + gaps: [], + trimRanges: [], + muteRanges: [], + speedRanges: [], + captionRanges: [], + }, + annotations: [], + zoomRanges: [], + audioTracks: [], + legacyEditor: null, + }; +} + +function setPlayhead(currentTimeSec: number, playing: boolean) { + act(() => { + useProjectStore.setState({ currentTimeSec, playing }); + }); +} + +describe("NativeCompositorOverlay while playing", () => { + let now = 1000; + + beforeEach(() => { + vi.clearAllMocks(); + vi.spyOn(performance, "now").mockImplementation(() => now); + useProjectStore.setState({ + projectId: "proj_sync", + document: makeDocument(), + revision: 1, + status: "ready", + error: null, + sourceDurationSec: 12, + currentTimeSec: 4.9, + playing: true, + dirty: false, + lastSavedAt: new Date(), + }); + }); + + afterEach(() => { + cleanup(); + vi.restoreAllMocks(); + publishNativePosition(null); + useProjectStore.getState().clear(); + }); + + async function mountAtFirstClip() { + render(); + // The first mount sends the starting clip, and its reply resumes the view: let both + // settle, so only what follows is under test. + await act(async () => { + await Promise.resolve(); + }); + native.setActiveClip.mockClear(); + native.setNativePlaying.mockClear(); + } + + it("lets the view cross a cut by itself", async () => { + await mountAtFirstClip(); + publishNativePosition({ clipIndex: 0, sourceTimeSec: 4.99 }, now); + + setPlayhead(5.02, true); + + expect(native.setActiveClip).not.toHaveBeenCalled(); + expect(native.setNativePlaying).not.toHaveBeenCalled(); + }); + + it("follows a jump to another clip at once, without pausing the view", async () => { + await mountAtFirstClip(); + publishNativePosition({ clipIndex: 0, sourceTimeSec: 4.9 }, now); + + setPlayhead(8, true); + + expect(native.setActiveClip).toHaveBeenCalledWith(7, "/take.mp4", "", 0, 1, 10, []); + expect(native.setNativePlaying).not.toHaveBeenCalledWith(false); + }); + + it("sends the asset's extra cameras with the clip", async () => { + const document = makeDocument(); + document.assets[0] = { + ...document.assets[0], + additionalCameraTracks: [ + { sourcePath: "/cam-2.mp4", startMs: 500, offsetMs: 0, visible: true, label: "" }, + { sourcePath: "/cam-3.mp4", startMs: 0, offsetMs: 0, visible: false, label: "" }, + ], + }; + useProjectStore.setState({ document }); + await mountAtFirstClip(); + publishNativePosition({ clipIndex: 0, sourceTimeSec: 4.9 }, now); + + setPlayhead(8, true); + + expect(native.setActiveClip).toHaveBeenCalledWith(7, "/take.mp4", "", 0, 1, 10, [ + { path: "/cam-2.mp4", offsetSec: 0.5 }, + { path: "", offsetSec: 0 }, + ]); + }); + + it("re-anchors a view that stays behind, once the gap holds", async () => { + await mountAtFirstClip(); + // The view stalled 0.4 s behind the playhead, inside the first clip. + publishNativePosition({ clipIndex: 0, sourceTimeSec: 2 }, now); + setPlayhead(2.4, true); + expect(native.setActiveClip).not.toHaveBeenCalled(); + + now += 120; + publishNativePosition({ clipIndex: 0, sourceTimeSec: 2.12 }, now); + setPlayhead(2.52, true); + + expect(native.setActiveClip).toHaveBeenCalledTimes(1); + expect(native.setActiveClip).toHaveBeenCalledWith(7, "/take.mp4", "", 0, 0, 2.52, []); + }); + + // At 16× a frame that takes 30 ms to arrive shows the playhead 0.48 s of programme ago. + it("leaves alone a view keeping pace inside a speed region", async () => { + useProjectStore.setState({ + document: { + ...makeDocument(), + legacyEditor: { speedRegions: [{ id: "speed_a", startMs: 0, endMs: 12000, speed: 16 }] }, + }, + currentTimeSec: 0.2, + }); + await mountAtFirstClip(); + + for (let frame = 0; frame < 8; frame++) { + now += 30; + publishNativePosition({ clipIndex: 0, sourceTimeSec: 0.2 + frame * 0.48 }, now); + setPlayhead(0.2 + (frame + 1) * 0.48, true); + } + + expect(native.setActiveClip).not.toHaveBeenCalled(); + }); + + // Two copies of one take: only the clip index tells them apart, and the scrub enters the left + // copy by its end. The view must be sent there, or it keeps the right copy's crop and zoom. + it("sends the left copy back when scrubbing into it while paused", async () => { + const left: AxcutClip = { + ...FIRST, + id: "clip_left", + sourceStartSec: 0, + sourceEndSec: 14.5, + timelineStartSec: 0, + timelineEndSec: 14.5, + }; + const right: AxcutClip = { + ...left, + id: "clip_right", + timelineStartSec: 14.5, + timelineEndSec: 29, + }; + const document = makeDocument(); + useProjectStore.setState({ + document: { ...document, timeline: { ...document.timeline, clips: [left, right] } }, + sourceDurationSec: 14.5, + currentTimeSec: 20, + playing: false, + }); + await mountAtFirstClip(); + publishNativePosition({ clipIndex: 1, sourceTimeSec: 5.5 }, now); + + for (const time of [16, 15, 14.6, 14.4, 12, 9]) { + now += 16; + setPlayhead(time, false); + } + + expect(native.setActiveClip).toHaveBeenCalledWith(7, "/take.mp4", "", 0, 0, 14.4, []); + expect(native.setActiveClip).toHaveBeenLastCalledWith(7, "/take.mp4", "", 0, 0, 14.4, []); + }); + + it("still sends the clip on a change while paused", async () => { + useProjectStore.setState({ playing: false }); + await mountAtFirstClip(); + publishNativePosition({ clipIndex: 0, sourceTimeSec: 4.9 }, now); + + setPlayhead(6, false); + + expect(native.setActiveClip).toHaveBeenCalledWith(7, "/take.mp4", "", 0, 1, 8, []); + }); + + // An addon that reports no position cannot be read, so it is driven as before. + it("pauses the view across the swap when it reports no position", async () => { + await mountAtFirstClip(); + + setPlayhead(5.02, true); + + expect(native.setNativePlaying).toHaveBeenCalledWith(false); + expect(native.setActiveClip).toHaveBeenCalledTimes(1); + }); +}); + +// The desk-view label is generated text inside the scene, so the scene is rebuilt when the +// user switches language — otherwise the preview keeps the old label until the next edit. +describe("NativeCompositorOverlay on a language change", () => { + beforeEach(() => { + vi.clearAllMocks(); + i18n.locale = "en"; + useProjectStore.setState({ + projectId: "proj_sync", + document: makeDocument(), + revision: 1, + status: "ready", + error: null, + sourceDurationSec: 12, + currentTimeSec: 1, + playing: false, + dirty: false, + lastSavedAt: new Date(), + }); + }); + + afterEach(() => { + cleanup(); + i18n.locale = "en"; + useProjectStore.getState().clear(); + }); + + it("pushes the scene again when the locale changes", async () => { + const { rerender } = render(); + await act(async () => { + await Promise.resolve(); + }); + native.setNativeScene.mockClear(); + + i18n.locale = "de"; + rerender(); + + expect(native.setNativeScene).toHaveBeenCalledTimes(1); + }); +}); + +// With another camera in camera 1's role, the view decodes that camera as its webcam and the +// scene sizes the PiP from that camera's probed size, exactly as the scene draws it. +describe("NativeCompositorOverlay with a main camera", () => { + beforeEach(() => { + vi.clearAllMocks(); + const document = makeDocument(); + document.assets[0] = { + ...document.assets[0], + cameraTrack: { sourcePath: "/cam-1.mp4", startMs: 0, offsetMs: 0, visible: true }, + additionalCameraTracks: [ + { sourcePath: "/cam-2.mp4", startMs: 500, offsetMs: 0, visible: true, label: "" }, + ], + }; + useProjectStore.setState({ + projectId: "proj_sync", + document: { ...document, legacyEditor: { mainCamera: 1 } }, + revision: 1, + status: "ready", + error: null, + sourceDurationSec: 12, + currentTimeSec: 1, + playing: false, + dirty: false, + lastSavedAt: new Date(), + }); + }); + + afterEach(() => { + cleanup(); + useProjectStore.getState().clear(); + }); + + it("sends the main camera as the clip's webcam and probes its size", async () => { + render(); + await act(async () => { + await Promise.resolve(); + }); + expect(native.setActiveClip).toHaveBeenCalledWith(7, "/take.mp4", "/cam-2.mp4", 0.5, 0, 1, [ + { path: "/cam-1.mp4", offsetSec: 0 }, + ]); + expect(probe.get).toHaveBeenCalledWith("/cam-2.mp4"); + expect(probe.get).not.toHaveBeenCalledWith("/cam-1.mp4"); + }); + + // Choosing the main camera on the clip the view already shows must reopen its decoders: the + // scene alone only says what to draw, the view keeps decoding the files it was given. + it("sends the clip again when the main camera changes under the playhead", async () => { + const withMain = useProjectStore.getState().document as AxcutDocument; + useProjectStore.setState({ document: { ...withMain, legacyEditor: null } }); + render(); + await act(async () => { + await Promise.resolve(); + }); + expect(native.setActiveClip).toHaveBeenLastCalledWith(7, "/take.mp4", "/cam-1.mp4", 0, 0, 1, [ + { path: "/cam-2.mp4", offsetSec: 0.5 }, + ]); + native.setActiveClip.mockClear(); + + act(() => { + useProjectStore.setState({ document: withMain }); + }); + + expect(native.setActiveClip).toHaveBeenCalledTimes(1); + expect(native.setActiveClip).toHaveBeenCalledWith(7, "/take.mp4", "/cam-2.mp4", 0.5, 0, 1, [ + { path: "/cam-1.mp4", offsetSec: 0 }, + ]); + }); +}); diff --git a/src/components/ai-edition/NativeCompositorOverlay.tsx b/src/components/ai-edition/NativeCompositorOverlay.tsx index e4af641027..9a26cde637 100644 --- a/src/components/ai-edition/NativeCompositorOverlay.tsx +++ b/src/components/ai-edition/NativeCompositorOverlay.tsx @@ -1,10 +1,12 @@ import { useEffect, useMemo, useRef, useSyncExternalStore } from "react"; -import { useScopedT } from "@/contexts/I18nContext"; +import { useI18n, useScopedT } from "@/contexts/I18nContext"; +import { readSpeedRegions } from "@/lib/ai-edition/document/timeline"; import { noteUiProbeClipSwitch } from "@/lib/ai-edition/perf/uiFrameProbe"; import { getEditorSettings } from "@/lib/ai-edition/store/editorSettings"; import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; -import { assetCameraSource } from "@/lib/ai-edition/timeline/camera"; +import { findActiveSpeedRegion, type SpeedRegion } from "@/lib/ai-edition/timeline/speed"; import { resolveNativePosition } from "@/lib/ai-edition/timeline/timelineMap"; +import { withMainCamera } from "@/lib/mainCamera"; import { pushAllNativeParams, setActiveClip, @@ -15,7 +17,19 @@ import { useIsCpuCompositor, useNativeCompositorView, } from "@/native"; -import { buildSceneDescription, resolveVisibleClips } from "@/native/sceneDescription"; +import { + type DriftWatch, + getNativePosition, + IDLE_DRIFT_WATCH, + NATIVE_DRIFT_TOLERANCE_SEC, + nativeLeadSec, + watchDrift, +} from "@/native/nativeSync"; +import { + buildSceneDescription, + clipCameraInputs, + resolveVisibleClips, +} from "@/native/sceneDescription"; import { getWebcamNativeSize, getWebcamNativeSizeRevision, @@ -48,6 +62,9 @@ import { export function NativeCompositorOverlay() { const canvasRef = useRef(null); const previousActiveClipIdRef = useRef(null); + // The camera files last sent with the active clip: a change on the same clip (a main camera + // chosen) must reopen the view's decoders, which the scene alone does not do. + const previousCamerasRef = useRef(null); const document = useProjectStore((s) => s.document); const currentTimeSec = useProjectStore((s) => s.currentTimeSec); // ponytail: re-render whenever the webcam dim cache changes (the WebcamOverlay @@ -83,21 +100,25 @@ export function NativeCompositorOverlay() { // `{screenPath,…}` = vraies sources de l'asset primaire. const settings = useMemo(() => getEditorSettings(document), [document]); + // The assets as the scene sees them: with a main camera chosen, it sits in camera 1's place + // (`withMainCamera`), so every camera the view decodes or probes matches the scene's. + const sceneAssets = useMemo(() => (document ? withMainCamera(document).assets : []), [document]); + // The real camera path, independent of whether NATIVE is the one drawing it: the scene // still needs it to look up the probed webcam size, which shapes the PiP box. const cameraPath = useMemo(() => { if (!document) return undefined; const primary = - document.assets.find((a) => a.id === document.project.primaryAssetId) ?? document.assets[0]; - return primary ? assetCameraSource(primary).path || undefined : undefined; - }, [document]); + sceneAssets.find((a) => a.id === document.project.primaryAssetId) ?? sceneAssets[0]; + return primary ? clipCameraInputs(primary).webcamPath || undefined : undefined; + }, [document, sceneAssets]); const sources = useMemo(() => { if (!document) { return null; } const primary = - document.assets.find((a) => a.id === document.project.primaryAssetId) ?? document.assets[0]; + sceneAssets.find((a) => a.id === document.project.primaryAssetId) ?? sceneAssets[0]; if (!primary?.originalPath) { return {}; } @@ -106,19 +127,21 @@ export function NativeCompositorOverlay() { // the one accessor, so it can never disagree with the scene or the export. return { screenPath: primary.originalPath, - webcamPath: assetCameraSource(primary).path || undefined, + webcamPath: clipCameraInputs(primary).webcamPath || undefined, // sidecar convention (electron/ipc/handlers.ts readCursorRecordingFile) : la // télémétrie curseur vit à côté de la vidéo tant qu'elle n'a pas bougé. Absente → // le natif ignore juste le curseur (CursorTrack::load échoue silencieusement). cursorPath: `${primary.originalPath}.cursor.json`, }; - }, [document]); + }, [document, sceneAssets]); const ready = sources !== null; const { viewId, error } = useNativeCompositorView(canvasRef, { sources: sources ?? undefined, }); const t = useScopedT("editor"); + // The scene carries translated text (the desk-view label): a language switch rebuilds it. + const { locale } = useI18n(); // No usable GPU: the preview still renders every effect, just slowly (~8 fps with // everything on). Nothing is disabled — the output stays identical to the GPU path — // so this is a notice, not a degradation warning. @@ -137,7 +160,7 @@ export function NativeCompositorOverlay() { // `_webcamSizeRevision` ci-dessus) : le layout preset et cie pilotent le rendu (remplace le // layout fixture). Effet APRÈS celui du viewId ci-dessus → currentViewId est déjà publié // quand on pousse. - // biome-ignore lint/correctness/useExhaustiveDependencies: size revision + // biome-ignore lint/correctness/useExhaustiveDependencies: size revision, locale useEffect(() => { if (viewId === null || !document) { return; @@ -153,8 +176,9 @@ export function NativeCompositorOverlay() { // re-trigger this effect when the probed-size cache mutates; the actual value is // re-read fresh via getWebcamNativeSize() above on every run (biome flags this as // an "unnecessary" dependency, but removing it would mean a probed webcam size - // arriving after mount never gets pushed to native). - }, [viewId, document, sources, _webcamSizeRevision]); + // arriving after mount never gets pushed to native). `locale` is the same kind of + // trigger: the label text is read inside `buildSceneDescription`. + }, [viewId, document, sources, _webcamSizeRevision, locale]); // SYNCHRO COMPLETE DES PARAMS, en un seul endroit. // @@ -183,12 +207,27 @@ export function NativeCompositorOverlay() { const activeClipId = activeClip?.id ?? null; const activeClipIndex = activePosition?.clipIndex ?? null; const activeSourceTimeSec = activePosition?.sourceTimeSec ?? null; + // Matched on the raw ruler, like `currentTimeSec` and the regions' pills. + const speedRegions = useMemo( + () => (document ? readSpeedRegions(document) : []), + [document], + ); + const playbackSpeed = findActiveSpeedRegion(speedRegions, currentTimeSec * 1000)?.speed ?? 1; const pendingTargetClipIdRef = useRef(null); const playing = useProjectStore((s) => s.playing); + const wasPlayingRef = useRef(playing); - // Change les décodeurs screen/webcam uniquement quand le playhead entre dans un autre clip. + // Change les décodeurs screen/webcam quand le playhead entre dans un autre clip, à l'arrêt. + // + // En lecture, la vue enchaîne seule les clips, préchargement compris. Lui renvoyer le clip à + // chaque coupe la faisait rechercher une position qu'elle avait déjà, ou jeter le + // préchargement qu'elle s'apprêtait à utiliser : un recul de quelques images et un arrêt à + // chaque coupe. Elle n'est recalée que si elle est réellement ailleurs (guet de dérive + // ci-dessous), d'après la position que porte chaque frame. useEffect(() => { + const justPaused = wasPlayingRef.current && !playing; + wasPlayingRef.current = playing; if ( viewId === null || !document || @@ -199,34 +238,63 @@ export function NativeCompositorOverlay() { ) { return; } - if (previousActiveClipIdRef.current === activeClipId) { + const native = getNativePosition(); + const clipChanged = previousActiveClipIdRef.current !== activeClipId; + // Mise en pause pendant que la vue était sur un autre clip que la tête de lecture : elle + // y retourne, sans quoi le recalage en temps (`setNativeTime`) chercherait dans le + // mauvais fichier. + const nativeElsewhere = justPaused && native !== null && native.clipIndex !== activeClipIndex; + const asset = sceneAssets.find((candidate) => candidate.id === activeClip.assetId); + if (!asset?.originalPath) { return; } - const asset = document.assets.find((candidate) => candidate.id === activeClip.assetId); - if (!asset?.originalPath) { + const cameras = clipCameraInputs(asset); + const camerasKey = JSON.stringify(cameras); + const camerasChanged = !clipChanged && previousCamerasRef.current !== camerasKey; + if (!clipChanged && !nativeElsewhere && !camerasChanged) { return; } - const camera = assetCameraSource(asset); + previousCamerasRef.current = camerasKey; const targetClipId = activeClipId; // Sonde de fluidité (diagnostic) : sépare les mesures d'avant et d'après un // franchissement de clip, qui se sont déjà révélées non comparables. - noteUiProbeClipSwitch(previousActiveClipIdRef.current, activeClipId); + if (clipChanged) { + noteUiProbeClipSwitch(previousActiveClipIdRef.current, activeClipId); + } pendingTargetClipIdRef.current = targetClipId; previousActiveClipIdRef.current = targetClipId; + // Other files on the same clip: the view cannot pick them up by itself, whatever its lead. + if (playing && native !== null && !camerasChanged) { + const lead = nativeLeadSec( + native, + { clipIndex: activeClipIndex, sourceTimeSec: activeSourceTimeSec }, + nativeClips, + playbackSpeed, + performance.now(), + ); + // Une coupe franchie en lecture : la vue la franchit d'elle-même, au même endroit du + // programme. Un saut (clic ailleurs sur la timeline) la met loin : suivi tout de suite, + // sans attendre le délai du guet de dérive. + if (lead !== null && Math.abs(lead) <= NATIVE_DRIFT_TOLERANCE_SEC) { + return; + } + } - // Pause native across the decoder swap. Deliberately NOT kept in a variable to - // resume from later — see the `.then` below, which re-reads the live transport. - if (playing) { + // Sans position rapportée (addon antérieur), la vue est encore pilotée comme avant : en + // pause pendant le changement de décodeurs. Deliberately NOT kept in a variable to resume + // from later — see the `.then` below, which re-reads the live transport. + if (playing && native === null) { setNativePlaying(false); } setActiveClip( viewId, asset.originalPath, - camera.path, - camera.offsetSec, + cameras.webcamPath, + cameras.webcamOffsetSec, activeClipIndex, activeSourceTimeSec, + cameras.additionalCameras ?? [], ) .then(() => { if (pendingTargetClipIdRef.current !== targetClipId) { @@ -248,7 +316,78 @@ export function NativeCompositorOverlay() { previousActiveClipIdRef.current = null; } }); - }, [viewId, document, activeClipId, activeClip, activeClipIndex, activeSourceTimeSec, playing]); + }, [ + viewId, + document, + sceneAssets, + activeClipId, + activeClip, + activeClipIndex, + activeSourceTimeSec, + playing, + nativeClips, + playbackSpeed, + ]); + + // Guet de dérive, en lecture : la vue tourne sur sa propre horloge, et n'est recalée que si + // la position que porte sa dernière frame s'écarte de la tête de lecture et le reste. Couvre + // un saut de l'utilisateur pendant la lecture comme un blocage du thread de rendu. L'ancien + // recalage devinait la dérive à l'horloge murale, à vitesse 1 : dans une région à 2× il + // relançait une recherche dix fois par seconde. + const driftRef = useRef(IDLE_DRIFT_WATCH); + useEffect(() => { + if ( + viewId === null || + !document || + !playing || + !activeClip || + activeClipIndex === null || + activeSourceTimeSec === null + ) { + driftRef.current = { ...driftRef.current, outSinceMs: null }; + return; + } + const now = performance.now(); + const lead = nativeLeadSec( + getNativePosition(), + { clipIndex: activeClipIndex, sourceTimeSec: activeSourceTimeSec }, + nativeClips, + playbackSpeed, + now, + ); + const { watch, resync } = watchDrift(driftRef.current, lead, now); + driftRef.current = watch; + if (!resync) { + return; + } + const asset = sceneAssets.find((candidate) => candidate.id === activeClip.assetId); + if (!asset?.originalPath) { + return; + } + const cameras = clipCameraInputs(asset); + previousCamerasRef.current = JSON.stringify(cameras); + setActiveClip( + viewId, + asset.originalPath, + cameras.webcamPath, + cameras.webcamOffsetSec, + activeClipIndex, + activeSourceTimeSec, + cameras.additionalCameras ?? [], + ).catch((error: unknown) => { + console.warn("[compositor-view] re-anchoring the preview failed:", error); + }); + }, [ + viewId, + document, + sceneAssets, + playing, + activeClip, + activeClipIndex, + activeSourceTimeSec, + nativeClips, + playbackSpeed, + ]); if (!ready) { return null; diff --git a/src/components/ai-edition/NewEditorShell.chatOpen.test.tsx b/src/components/ai-edition/NewEditorShell.chatOpen.test.tsx index 27fb776f3d..ead8fed699 100644 --- a/src/components/ai-edition/NewEditorShell.chatOpen.test.tsx +++ b/src/components/ai-edition/NewEditorShell.chatOpen.test.tsx @@ -2,6 +2,7 @@ import "@testing-library/jest-dom"; import { act, cleanup, fireEvent, render, screen } from "@testing-library/react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { TooltipProvider } from "@/components/ui/tooltip"; import { EditorDialogsProvider } from "@/contexts/EditorDialogsContext"; import { useChatPromptBus } from "@/lib/ai-edition/store/useChatPromptBus"; @@ -48,9 +49,11 @@ import { NewEditorShell } from "./NewEditorShell"; function renderShell() { return render( - - - , + + + + + , ); } diff --git a/src/components/ai-edition/NewEditorShell.dialogShortcuts.test.tsx b/src/components/ai-edition/NewEditorShell.dialogShortcuts.test.tsx index 7c4201ead7..10834f9690 100644 --- a/src/components/ai-edition/NewEditorShell.dialogShortcuts.test.tsx +++ b/src/components/ai-edition/NewEditorShell.dialogShortcuts.test.tsx @@ -49,14 +49,17 @@ vi.mock("@/contexts/I18nContext", () => ({ useScopedT: () => (key: string) => key, })); +import { TooltipProvider } from "@/components/ui/tooltip"; import { EditorDialogsProvider } from "@/contexts/EditorDialogsContext"; import { NewEditorShell } from "./NewEditorShell"; function renderShell() { return render( - - - , + + + + + , ); } diff --git a/src/components/ai-edition/NewEditorShell.module.css b/src/components/ai-edition/NewEditorShell.module.css index 9b24797e6b..162591937f 100644 --- a/src/components/ai-edition/NewEditorShell.module.css +++ b/src/components/ai-edition/NewEditorShell.module.css @@ -361,6 +361,10 @@ overflow: hidden; display: grid; place-items: center; + /* Its own stacking context: the layers inside it (gimbals, zoom focus, webcam drag) order + themselves with z-indexes up to 1000 and more, which must not reach past the preview. A + selected annotation's gimbal painted over the export dialog (z-index 100). */ + isolation: isolate; } /* Once the native canvas has painted, it covers the whole card, and anything painted UNDER the same rounded clip leaks through its anti-aliased edge: the wallpaper placeholder and the inset @@ -722,7 +726,8 @@ position: relative; z-index: 1; /* Pixels come from the native canvas now; this