diff --git a/.github/workflows/announce-release.yml b/.github/workflows/announce-release.yml index 078195c882..01be8980ed 100644 --- a/.github/workflows/announce-release.yml +++ b/.github/workflows/announce-release.yml @@ -52,6 +52,7 @@ jobs: announce: name: Announce ${{ inputs.tag || github.event.release.tag_name }} runs-on: ubuntu-latest + environment: release steps: - name: Checkout uses: actions/checkout@v7 diff --git a/.github/workflows/aur-publish.yml b/.github/workflows/aur-publish.yml index b4eef00978..c2b70250bc 100644 --- a/.github/workflows/aur-publish.yml +++ b/.github/workflows/aur-publish.yml @@ -28,6 +28,7 @@ env: jobs: publish: runs-on: ubuntu-latest + environment: release if: (github.event_name == 'workflow_dispatch' || !github.event.release.prerelease) && vars.AUR_PACKAGE_NAME != '' steps: - name: Resolve and validate tag diff --git a/.github/workflows/auto-approve-linux.yml b/.github/workflows/auto-approve-linux.yml index 3cb51f98c9..f4b4d58f0a 100644 --- a/.github/workflows/auto-approve-linux.yml +++ b/.github/workflows/auto-approve-linux.yml @@ -15,6 +15,11 @@ jobs: approve: if: github.event.pull_request.user.login == 'Beetix' && !github.event.pull_request.draft runs-on: ubuntu-latest + # The app key lives in the `approver` environment, which only `main` may use. + # pull_request_target is evaluated against the default branch, so this job + # passes. A workflow pushed on any other branch cannot read the key, and so + # cannot approve a pull request in the app's name. + environment: approver permissions: pull-requests: read steps: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3c23c78a4c..57a5359766 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -300,6 +300,7 @@ jobs: # ffmpeg's configure, cargo and the output paths — four blind changes on a release # branch, none of them testable without a Mac. runs-on: ${{ matrix.arch == 'x64' && 'macos-15-intel' || 'macos-latest' }} + environment: release strategy: fail-fast: false matrix: @@ -430,6 +431,11 @@ jobs: restore-keys: | cargo-macos-${{ matrix.arch }}- + - name: Install Rust toolchain + run: | + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable --target ${{ matrix.arch == 'x64' && 'x86_64' || 'aarch64' }}-apple-darwin + echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + - name: Build Metal compositor addon run: npm run build:native:compositor:mac @@ -498,6 +504,14 @@ jobs: exit 1 fi + # Without it the installed app can never update itself, and nothing else says so: the + # probe fails a minute after launch and falls back to the release page (#1431). + - name: Verify .app carries app-update.yml + run: | + FILE="${{ steps.find_app.outputs.app_bundle }}/Contents/Resources/app-update.yml" + [[ -f "$FILE" ]] || { echo "::error::${FILE} is missing — scripts/after-pack.cjs did not run"; exit 1; } + cat "$FILE" + # electron-builder used to do this itself. Its macPackager carried a # `noIdentity && fallBackToAdhoc` branch that handed back `Identity("-")` # whenever no certificate was found — mandatory on arm64, where an unsigned @@ -667,13 +681,29 @@ jobs: - name: Staple notarization ticket if: steps.signing.outputs.enabled == 'true' - run: xcrun stapler staple "${{ steps.dmg.outputs.dmg_path }}" + # Apple's ticket lookup drops the connection now and then (error 68); retry it. + run: | + for attempt in 1 2 3 4 5; do + xcrun stapler staple "${{ steps.dmg.outputs.dmg_path }}" && exit 0 + [ "$attempt" -eq 5 ] && break + echo "Staple attempt $attempt failed, retrying in 30 s" + sleep 30 + done + exit 1 + timeout-minutes: 10 - name: Validate stapled DMG if: steps.signing.outputs.enabled == 'true' + # validate queries Apple too and fails the same way (error 68); retry it like staple. run: | - xcrun stapler validate "${{ steps.dmg.outputs.dmg_path }}" + for attempt in 1 2 3 4 5; do + xcrun stapler validate "${{ steps.dmg.outputs.dmg_path }}" && break + [ "$attempt" -eq 5 ] && exit 1 + echo "Validate attempt $attempt failed, retrying in 30 s" + sleep 30 + done spctl -a -vv -t install "${{ steps.dmg.outputs.dmg_path }}" + timeout-minutes: 10 - name: Upload macOS DMG uses: actions/upload-artifact@v7 @@ -854,6 +884,7 @@ jobs: publish-release: name: Publish GitHub release runs-on: ubuntu-latest + environment: release needs: - build-windows - build-macos @@ -1175,6 +1206,7 @@ jobs: publish-msstore: name: Publish to Microsoft Store runs-on: windows-latest + environment: release # The workflow-wide token is `contents: write` because publish-release needs # it. This job only reads: it checks the tree out so the CLI can identify the # project, downloads a same-run artifact (which uses the runtime token, not diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 3575c0249a..87e53a7a59 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -146,7 +146,9 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - sparse-checkout: website/wrangler.jsonc + sparse-checkout: | + website/wrangler.jsonc + website/scripts/indexnow.mjs sparse-checkout-cone-mode: false - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -157,3 +159,10 @@ jobs: env: CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} run: npx --yes wrangler@4.145.0 deploy + # Tell Bing and the other IndexNow engines which pages this deploy changed, so + # they recrawl now rather than whenever. Never fails the deploy: a missed ping + # only means the next regular crawl picks the change up. + - name: Ping IndexNow + working-directory: website + continue-on-error: true + run: node scripts/indexnow.mjs build diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index e43b6ff200..37669a7a11 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -32,11 +32,15 @@ jobs: prerelease: name: Cut pre-release runs-on: ubuntu-latest + environment: release steps: - name: Checkout uses: actions/checkout@v7 with: fetch-depth: 0 + # The persisted GITHUB_TOKEN header overrides the PAT in a push URL, so every + # push would run as github-actions[bot], which the release rulesets reject. + persist-credentials: false - name: Setup Node.js uses: ./.github/actions/setup @@ -122,7 +126,9 @@ jobs: # The lockfile's version is part of Nix's npm hash: refresh it in this same commit. bash "$RUNNER_TEMP/refresh-nix-npm-hash.sh" git add package.json package-lock.json nix/package.nix - git commit -m "chore(release): bump to ${PRERELEASE} [skip ci]" || echo "(version already at ${PRERELEASE})" + # --allow-empty: a re-cut must still leave a [skip ci] commit at the tag, or the PAT + # tag push fires build.yml on top of the explicit dispatch below. + git commit --allow-empty -m "chore(release): bump to ${PRERELEASE} [skip ci]" git push "$REMOTE" "$BRANCH" - name: Push RC tag on the release branch diff --git a/.github/workflows/promote.yml b/.github/workflows/promote.yml index 97f04bb7cd..43b4b19b1c 100644 --- a/.github/workflows/promote.yml +++ b/.github/workflows/promote.yml @@ -26,11 +26,15 @@ jobs: promote: name: Promote RC to stable runs-on: ubuntu-latest + environment: release steps: - name: Checkout uses: actions/checkout@v7 with: fetch-depth: 0 + # The persisted GITHUB_TOKEN header overrides the PAT in a push URL, so every + # push would run as github-actions[bot], which the release rulesets reject. + persist-credentials: false - name: Setup Node.js uses: ./.github/actions/setup @@ -95,10 +99,10 @@ jobs: - name: Push stable tag on the release branch tip env: - # Use GITHUB_TOKEN for the tag push: a tag is a ref, not a file change, so - # the workflows:write permission isn't needed. - # Note: GITHUB_TOKEN tag pushes do NOT trigger build.yml in this org's setup, - # so we explicitly trigger it via gh workflow run right after. + # Push with the PAT: the checkout keeps no credentials, and the release-tags + # ruleset only lets its owner delete a tag. The tip commit says [skip ci], so + # the push does not fire build.yml; it is dispatched explicitly right after. + TOKEN: ${{ secrets.OPENSCREEN_RELEASE_TOKEN }} STABLE_TAG: ${{ steps.version.outputs.stable_tag }} STABLE_VERSION: ${{ steps.version.outputs.stable_version }} run: | @@ -108,10 +112,11 @@ jobs: git checkout "$BRANCH" git reset --hard "origin/${BRANCH}" # Delete remote tag first (idempotent on rerun) and any local tag. - git push origin ":${STABLE_TAG}" 2>/dev/null || true + REMOTE="https://x-access-token:${TOKEN}@github.com/${GITHUB_REPOSITORY}.git" + git push "$REMOTE" ":${STABLE_TAG}" 2>/dev/null || true git tag -d "$STABLE_TAG" 2>/dev/null || true git tag "$STABLE_TAG" - git push origin "$STABLE_TAG" + git push "$REMOTE" "$STABLE_TAG" # ORDER MATTERS. Everything that actually publishes the release runs before the # main-sync below. The stable tag is already pushed at this point, so the build diff --git a/.github/workflows/publish-msstore.yml b/.github/workflows/publish-msstore.yml index cda0c31e8d..e3ee8d6efa 100644 --- a/.github/workflows/publish-msstore.yml +++ b/.github/workflows/publish-msstore.yml @@ -45,6 +45,7 @@ jobs: submit: name: Submit ${{ inputs.release_tag }} to the Store runs-on: windows-latest + environment: release # No job-level `if` on MSSTORE_PRODUCT_ID, deliberately. build.yml can afford # to skip: it is one job among many in an automatic release. This one is # something a person asked for by hand, and a skipped job is green and diff --git a/.github/workflows/publish-winget.yml b/.github/workflows/publish-winget.yml index f72f0055ac..869c99a70a 100644 --- a/.github/workflows/publish-winget.yml +++ b/.github/workflows/publish-winget.yml @@ -19,6 +19,7 @@ permissions: jobs: publish: runs-on: windows-latest + environment: release # La condition de configuration a QUITTÉ ce `if`, et c'est tout l'objet du # changement. `vars.WINGET_IDENTIFIER != ''` ici rendait le job `skipped`, et un # job sauté est vert : les huit dernières releases, v1.8.0 comprise, ont toutes diff --git a/.github/workflows/update-homebrew-cask.yml b/.github/workflows/update-homebrew-cask.yml index 8b9ba3127f..d56ce76447 100644 --- a/.github/workflows/update-homebrew-cask.yml +++ b/.github/workflows/update-homebrew-cask.yml @@ -16,6 +16,7 @@ permissions: jobs: update-cask: runs-on: ubuntu-latest + environment: release # The tap configuration has LEFT this `if:`, and that is the point of the change. # `vars.HOMEBREW_TAP_OWNER != '' && vars.HOMEBREW_TAP_REPO != ''` here made the job # `skipped`, and a skipped job is green: every release since this workflow was diff --git a/.rusuban/locks.json b/.rusuban/locks.json new file mode 100644 index 0000000000..d351a260c9 --- /dev/null +++ b/.rusuban/locks.json @@ -0,0 +1,52 @@ +{ + "mergeMethod": "rebase", + "lockedPaths": [ + ".github/**", + "scripts/**", + "nix/**", + "flake.*", + "package.json", + "package-lock.json", + "Cargo.toml", + "Cargo.lock", + "electron-builder.*", + ".npmrc" + ], + "maxLines": 300, + "maxFiles": 15, + "sizeExempt": [ + "*.test.*", + "*.spec.*", + "**/tests/**", + "**/test/**", + "**/__tests__/**", + "**/fixtures/**", + "*.md", + "technical-documentation/**", + "docs/**" + ], + "requiredChecks": ["CodeRabbit"], + "ignoredChecks": [ + "nix build .#openscreen", + "Sync PR activity to Discord", + "Sync ROADMAP.md to Discord", + "Mark linked issues fixed in main" + ], + "evidence": [ + { + "paths": [ + "crates/compositor/**", + "electron/native/**", + "electron/native-bridge/**", + "electron/recording/**", + "src/native/**", + "src/lib/exporter/**", + "*Preview*.js", + "*Preview*.jsx", + "*Preview*.ts", + "*Preview*.tsx" + ], + "what": "a pixel-identical export of the reference project; export time and preview frame rate within ±5 % of the base; a preview and an export of the same instant that show the same effects, composition and timing at the preview's size" + } + ] +} diff --git a/.rusuban/rules/01-autonomy.md b/.rusuban/rules/01-autonomy.md new file mode 100644 index 0000000000..c62df3156d --- /dev/null +++ b/.rusuban/rules/01-autonomy.md @@ -0,0 +1,88 @@ +# What the agents do alone, and what comes to the owner + +The owner is @EtienneLescot. Every card goes to the owner. + +## Alone + +The agents act alone on everything that is not a product or architecture decision: + +- fix a bug, and merge a PR that is not a feature once its gates are green (see 02-merging); +- approve a fork's CI after reading the diff; +- ask for a rebase, a fix or missing information; +- close a duplicate, an item that was replaced, or one that came due. + +## To the owner, always with a card + +- A feature, issue or PR. +- A change of UI or UX: visible English text, style, layout, a default value, a setting. +- An architecture choice: a new module, a new dependency, a new IPC or API surface, a data format. +- An impact on pixels, rendering or performance that no proof has ruled out. +- A refusal: declining a PR, closing a feature request, or answering a product question. + +## Issues + +- **A bug that can be reproduced and is documented enough**: the agents own it, and a worker fixes it. +- **A bug without enough information**: it waits on its reporter (`owner: "third"`, `waits: "@login"`). + Post one factual question. +- **A product, UI or architecture request**: to the owner, with a card. +- **A duplicate**: set `links.duplicateOf`, post a factual comment and close it. +- **A request that an existing setting already covers**: the agents answer with the setting and how + to reach it, close the issue as completed and invite a reopen if it falls short. When the request + also asks for a new control or a shortcut to it, that part goes to the owner with a card. +- **A feature request the owner has already said is not near-term**, with nobody working on it: the + agents label it `help wanted`, say so once on the issue, and stop revisiting it. It comes back only + when someone comments, opens a PR for it, or the owner asks. + +## Waiting on a third party: two days at most + +An item that waits on someone (a reporter, an author, a contributor, a merge, a release) comes back +two days later (`revisitAt`). Read the thread first: if they answered in the last two days, take the +answer in and judge again. Otherwise, move on without them. + +- **A bug fix or a consolidation** (bug, maintenance, cleanup, robustness, translation): the agents + own it, and a worker moves it forward. + - An issue: fix it from what is known (the thread, logs, the code, a reproduction on a machine the + agents can reach) and open the PR. + - A contributor's PR: finish it in their place, on their fork's branch, without closing their PR + (rebase, fixes, review threads). They keep the credit. + - If their fork refuses maintainer edits, ask them to allow them. Without an answer in two days, + carry their commits to a new PR from a fork under your account whose Actions stay disabled, + never from a branch of this repo: a branch here runs its workflows in the repo's trusted + context, where secrets can reach them; a fork's PR never runs its code with secrets or a + write token. Their name stays on their commits: + - read every commit first. One that touches `.github/`, `scripts/` or the build goes to the + owner on a card; + - link the two PRs, and say so on theirs. When theirs would close an issue, the new PR's + description carries its closing reference; + - before the new PR merges, read their PR's head again and bring in what they pushed since. + Once it merged, compare their head with the last one you carried: anything newer goes into + a follow-up PR. If the issue the new PR closed is not complete without it, reopen it. The + follow-up's description carries its closing reference only when it completes the issue, and + `Refs #N` otherwise, `Refs owner/repo#N` for an issue in another repo. + - A wait on a merge or a release: check whether it happened, then close the item or push what + blocks it. + - Close only when the code gives no lead and no machine reproduces it, with a factual comment that + invites a reopen with the missing information. +- **A new feature**, issue or PR: to the owner, with a card that presents it as it is: what is done, + what is missing. Offer actions you will take: merge it as it is, finish it yourself, keep it open as + "help wanted", or close it. + +Never hand a two-day wait back to the third party without a recent answer from them: that would be a +silent wait. + +## Nothing stays with the agents without work + +An item the agents own comes back after one day at most (`revisitAt`). With no work in progress, +delegate it to a worker, hand it to a named third party, or bring it to the owner with a card. + +## Priority + +- **P0**: a crash, lost data, or a recording or an export that fails, in a released version. +- **P1**: a core flow degraded (capture, preview, editing, export), or a regression. +- **P2**: any other bug, and maintenance. +- **P3**: cosmetic, and nice to have. + +## Areas + +`capture`, `preview`, `export`, `editor`, `ai`, `mcp`, `i18n`, `packaging`, `tooling`, `storage`, +`webcam`, `audio`. diff --git a/.rusuban/rules/02-merging.md b/.rusuban/rules/02-merging.md new file mode 100644 index 0000000000..6d2d1a43db --- /dev/null +++ b/.rusuban/rules/02-merging.md @@ -0,0 +1,87 @@ +# What merges alone, and how this repo merges + +## A PR may merge alone (`pr.auto: true`) + +It applies to every author, contributors included, judged on the head you read. A PR merges alone +only when it is one of these: + +- a bug fix; +- a translation into a language other than English; +- documentation; +- tests; +- maintenance. + +It never merges alone when it is: + +- a feature; +- a change of UI or UX; +- an architecture choice; +- an impact on pixels, rendering or performance that a proof has not ruled out. + +**A bug** is behavior that contradicts what is expected or documented: a crash, a desync, a wrong +export. A new setting, an interface text, a default value or a new visible behavior is a feature. + +The locks file adds what the server checks alone: + +- the supply-chain paths (CI, scripts, dependencies, packaging) never merge alone; +- at most 300 changed lines and 15 files, tests and documentation excluded. + +## Rendering and performance need a proof + +Preview and export share one source of truth for rendering: what you see is what you export. + +A change to capture, preview, compositor, export or rendering needs a proof of no impact: + +- a pixel-identical export of the reference project; +- export time and preview frame rate within ±5 % of the base, measured in the real app, heavy case + included; +- a preview and an export of the same instant, compared at the preview's size: they show the same + effects, with the same composition and the same timing. They need not match pixel for pixel. + +A bug fix or maintenance change under an evidence path that cannot reach the picture or its timing +(stop, salvage, error handling, logging, warnings, process lifetime) needs no proof: the judge says why +in `pr.why`, and the PR merges alone once its other gates are green. A change to how frames are +captured, encoded, composited or exported still needs its proof. + +The locks file names the zones. Elsewhere, set `pr.evidence.needed: true` yourself when the change can +reach the picture or its timing. A worker other than the judge makes the proof and records it. + +Performance comes first. A correctness fix that costs latency is split in two: + +- the part that costs nothing goes on its own; +- the rest goes to the owner as an explicit trade-off, on a card. + +## Rebase only + +- **This repo merges by rebase only, with no merge commits and no squash.** +- **Conflicts are fixed by rebasing on the base, never by a merge commit.** With a merge commit, GitHub + still calls the PR mergeable, but it cannot rebase it. Check the REST field `rebaseable`, not + `mergeable`. +- **During a train of merges, `rebaseable` can be stale.** Rebase on main, push, and wait for CI. +- **Merge first the PRs that touch shared files**, such as the end-to-end checklist and the locales. + +## No closing keyword in a commit + +- **A rebase merge replays every commit.** A `Fixes #N` in a commit closes the issue, even when the PR + only says "Part of #N". +- **Closing keywords** (`Fixes`, `Closes`, `Resolves`) belong in the PR's description only, and only + when the PR completes the issue. Agents write `Refs #N`. +- **Before a partial fix merges**, read its commit messages. + +## Checks + +- **The Nix build is not a merge gate**: the locks file ignores it. +- **CodeRabbit is required.** When its review is cancelled or skipped (it reviews at most 8 PRs an + hour here), comment `@coderabbitai review` on the PR, then wait for its review. If that request is + skipped too, ask once more an hour later; if the second one is skipped as well, the PR goes to the + owner on a card. Never more than two requests per head. +- **Do not trust the CI rollup on a fork's PR.** Its workflows wait for an approval, while the Discord + sync and CodeRabbit pass without one, so the rollup can read green when no real job ran. Judge from + the CI workflow's own run on the head. + +## Features run on the three systems + +- **A feature must work on macOS, Windows and Linux.** A feature PR limited to one system is not merged. +- **Before you propose to merge a feature, check the three systems.** If one is missing, comment to + ask the contributor for the port and a short end-to-end video per system (a real capture and an + export). diff --git a/.rusuban/rules/03-review.md b/.rusuban/rules/03-review.md new file mode 100644 index 0000000000..3851adbd30 --- /dev/null +++ b/.rusuban/rules/03-review.md @@ -0,0 +1,56 @@ +# Reviewing + +These are what the owner has turned down after CI was green. Check them before you recommend a merge. + +## Interface + +- **Every control has a visible effect in every reasonable state, not only at the edges.** If what it + changes is invisible, show it on the canvas or drop the control. +- **One control, one axis.** Two controls never fight over the same visible result, and a control + never moves an axis it does not name. +- **No raw internal values shown to the user**, such as signed percentages for a position. +- **One style per row**: never text buttons and icon buttons mixed in the same panel. +- **No explanatory sentence under a setting.** The label says it. +- **An option that does nothing in the current context is hidden**, not greyed out with a reason. +- **The one exception is a warning that prevents a real surprise.** +- **An automatic mode is available only on homogeneous inputs.** On mixed inputs it is not offered. +- **Prefer fixing the model underneath** to adding presets or controls over a broken one. + +## Translations + +- **Every new text key exists in the locale files.** A key missing everywhere shows raw in the app. +- **The locale check is not enough.** It only compares the locales with one another, and the tests mock + the translation function. + +## Rendering + +- **Every effect goes through the one source of rendering that preview and export share**: what you + see is what you export. A PR that renders an effect in only one of the two paths, or differently + in each, is not merged. + +## Performance + +**A PR on preview, capture, compositor or export carries before-and-after numbers**, measured in the +real app, heavy case included: + +- frame rate; +- time per frame; +- seek latency; +- CPU and GPU; +- frames dropped. + +No numbers, no merge. + +## PRs that meet + +- **Two PRs green on their own can break main together.** When a PR changes what a shared function does, + for example it no longer throws, look at every caller, including those another open PR adds. +- **When two open PRs touch the same files**, set `links.before` so the order is explicit. + +## Fixes made by workers + +- **A fix of an issue is made in the worker's own worktree**, on a new branch, and its PR is opened + with `Refs #N`. +- **Finishing a contributor's PR happens on their branch**, never in a competing PR (see + 01-autonomy). +- **The repo's AGENTS.md applies before the PR**: targeted tests, both typechecks, and lint. diff --git a/.rusuban/rules/04-github.md b/.rusuban/rules/04-github.md new file mode 100644 index 0000000000..a8bb1ac0ea --- /dev/null +++ b/.rusuban/rules/04-github.md @@ -0,0 +1,16 @@ +# Writing on GitHub + +- **Every message on GitHub is in English**: short, warm and factual. +- **What agents may post on their own is factual.** + - Allowed: fixed in a version, a request for information, a duplicate, a reminder, a review of a + bug fix, a request to rebase. + - Never on your own, always a card first: a refusal, closing a feature, a product answer. +- **Thank contributors.** When you finish a contributor's PR, it stays theirs: + - push to their fork's branch, as the maintainer can; + - keep their PR open; + - rebase it, never merge main into it; + - their name stays on their commits. +- **Approve a fork's CI only after reading the head's diff.** + - The diff must hold no secret, no network access, no install hook, and no doubtful dependency. + - Read any change under `.github/` or `scripts/` line by line. If in doubt, write a card. + - Approving one contributor's runs also releases their other open PRs: read those diffs too. diff --git a/AGENTS.md b/AGENTS.md index ceea69bb61..72accd4a02 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,7 +12,7 @@ OpenScreen is a free, open-source screen recorder and video editor (Electron + R - Test (e2e): `npm run test:e2e` (Playwright) - Lint: `npm run lint` (Biome 2.4) - Format: `npm run format` (Biome, tabs, double quotes, 100-col) -- i18n check: `npm run i18n:check` (validates the 15 locale folders) +- i18n check: `npm run i18n:check` (validates all the locale folders) **Use npm, not bun/pnpm/yarn/Deno.** Not a style preference. Node native modules are rebuilt against Electron's ABI by electron-builder + `@electron/rebuild`, which resolve the tree through `package-lock.json`. Another package manager writes a different lockfile, so that rebuild breaks. `packageManager` + `engines` in `package.json` pin the versions; CI installs with `npm ci`. Note what this does *not* cover: the standalone Swift (macOS) and C++ (Windows) capture helpers are separate executables, built by `npm run build:native:` and only *copied* into the package as `extraResources` — `build:win` even passes `--config.npmRebuild=false`. Nothing in a normal build compiles them. @@ -126,13 +126,20 @@ This section is the *mechanics*. **What to actually run is [`technical-documenta - **The HUD and the editor never share the screen.** Opening the editor closes the HUD, and closing the editor, the last window, quits the app. The way back is the editor's Record mode (see the flow below). - Frameless, transparent, always-on-top, `skipTaskbar`, centered at the **bottom of the primary display** (`createHudOverlayWindow`, 820×560 at construction, then resized to fit its content — measured 904×698 with the bar at the bottom and mostly empty reserve above it). It is **click-through** (`setIgnoreMouseEvents(ignore)`): moving the real cursor over an interactive control makes that region clickable and shows its tooltip, so `mouse_move` → screenshot → `left_click` works; a blind click on empty HUD area passes through to the desktop. - **Only a real OS mouse move reaches the HUD — on macOS as much as on Windows.** While the window is input-transparent Chromium delivers it no pointer events at all, so the main process samples the OS cursor instead: the `hud-overlay-cursor` poll in `electron/windows.ts` reads `screen.getCursorScreenPoint()` while the HUD is click-through and pushes the window-relative point to the renderer, which hit-tests it with `elementFromPoint(…).closest("[data-hud-interactive='true']")`. One path, both platforms — there is no platform branch. **Linux is the exception** (`!enabled && !isLinuxHud` in `LaunchWindow.tsx`, where the call is a no-op), so it is the one platform where a blind click on the HUD simply lands. What the poll keys off is the OS cursor's position *relative to the window*, so a resize or re-anchor that slides the bar under a motionless pointer produces a fresh sample too. What it can never key off is synthesised input: Playwright's `.click()`, `javascript_tool`-dispatched pointer events and everything like them move no pointer at all, so they never put one on a control. They arrive *below* the OS hit-test, fire the DOM handler, and look like they worked — while the click-through path was never exercised at all. `tests/e2e/windows-native-checklist.spec.ts` does click HUD test IDs and stays green for exactly that reason; it proves renderer wiring, not reachability, and a macOS spec written the same way would prove no more. Use computer-use (`mouse_move` → `left_click`), and never conclude from a passing injected click that a user could have clicked it. (Until #385 the lift was Electron's `{ forward: true }` — a global `WH_MOUSE_LL` hook on Windows — which Windows can revoke without telling the app, leaving the HUD painted and permanently dead. The poll replaced it. The rule for you is unchanged, because both mechanisms key off the real cursor.) -- Control row (left→right): drag handle; horizontal/vertical bar toggle; **source** button (`Screen` until a source is picked, then its name); system audio; microphone; **camera**; a gear, *Device settings*, which picks the microphone and the camera and shows a level meter and a preview; cursor mode (editable or system cursor); **record**; *Open Studio* while idle, or pause, restart and cancel while recording; notes; language (the locale code, e.g. `EN`); *Hide recording bar*; *Quit OpenScreen*. Per platform: on macOS 15.2+ the source button opens Apple's system picker and the HUD hides until it closes; on macOS 13 and 14 there is no microphone; on Linux there is no source button (the portal asks when recording starts) and no notes button. +- Control row (left→right): drag handle; horizontal/vertical bar toggle; **source** button (`Screen` until a source is picked, then its name); system audio; microphone; **camera**; a gear, *Device settings*, which picks the microphone and the camera and shows a level meter and a preview; cursor mode (editable or system cursor); **record**; *Open Studio* while idle, or pause, restart and cancel while recording; notes; language (the locale code, e.g. `EN`); *Hide recording bar*; *Quit OpenScreen*. Per platform: on macOS 15.2+ the source button opens Apple's system picker and the HUD hides until it closes; on macOS 13 and 14 there is no microphone; on Linux there is no source button (the portal asks when recording starts). - Record is never disabled for want of a source. With none picked, its tooltip reads "Choose a screen or window to record", and a click opens the picker, then starts recording once a source is chosen. **The tray icon** (notification area on Windows, menu bar on macOS, the desktop's indicator area on Linux) - Because the HUD skips the taskbar and can be minimized/hidden, the **system-tray icon is the reliable way to refocus the app**: **a click or double-click brings back the app's window** (`showMainWindow`: the HUD, or the editor while it is open), and so does its menu's *Open*. Its icon swaps to a red dot while recording. -- **Right-click → context menu**: when idle, *Open*, *Check for Updates* (not on a Store, Flathub, Snap or Nix install), *Update Settings* (packaged builds that update themselves), *About OpenScreen*, *Save Diagnostics*, *Permissions…* (macOS), *Quit*; while recording, only ***Stop Recording*** (mirrors the HUD's stop). Tooltip shows `OpenScreen` or `Recording: `. Use this to stop a recording if the HUD isn't reachable. +- **Right-click → context menu**: when idle, *Open*, *Update to * (only while a background check has found one; the same click as the HUD's and the editor's blue update button), *Check for Updates* (not on a Store, Flathub, Snap or Nix install), *About OpenScreen*, *Save Diagnostics*, *Permissions…* (macOS), *Quit*; while recording, only ***Stop Recording*** (mirrors the HUD's stop). Tooltip shows `OpenScreen` or `Recording: `. Use this to stop a recording if the HUD isn't reachable. + +**Updates** + +- **Run the update pass on every release candidate and after any change to the update path.** A build with a broken updater can only be left through that updater, so a defect here strands every user who installs it. What to run, and how to get a newer version to update to: [Updating in place](technical-documentation/testing/manual-e2e-checklist.md#updating-in-place--post-212), in the checklist. +- **It needs an installed build.** The background check and the blue update button both require `app.isPackaged`: a dev build shows neither, and that absence proves nothing. The Store package does not apply either; the Store updates it. +- **On macOS, only the RC path works:** install `rc.N`, publish `rc.N+1`. Editing `app-update.yml` inside a signed `.app`, which is how the local feed works on Windows and Linux, breaks the bundle's seal, and Squirrel then refuses the update. +- **Only the click installs.** Quitting with the button showing must install nothing. If an update installs on its own, at quit or at any other moment, that is a defect, not a feature to note. **End-to-end flow (record → edit)** @@ -173,7 +180,7 @@ The one rule to know before you merge anything: **there is one release branch pe - **Native capture is platform-fragile**: macOS uses ScreenCaptureKit (Swift), Windows uses WGC (C++/Win32). CI unit-tests the Swift capture core, builds both macOS helpers and checks the compositor on macOS, Windows and Linux. These checks do not exercise real capture, permissions or devices; manual smoke tests on the affected platforms are still required for native changes. See [CI coverage](technical-documentation/engineering/ci-workflows.md#tier-1-application-and-documentation-ci). - **Rendering is native**: the Rust compositor in `crates/compositor/` renders the editor preview and every export, MP4 and GIF. It runs on Direct3D 11 on Windows, Metal on macOS and wgpu on Linux, with a CPU fallback on Windows and Linux. The renderer only paints the preview frames it pulls onto a ``. Details: `technical-documentation/README.md`, then `architecture/native-compositor.md`. -- **i18n**: 15 locales in `src/i18n/locales//` (e.g. `src/i18n/locales/en/settings.json`). The `i18n:check` script validates them — run it after touching translation files. +- **i18n**: 16 locales in `src/i18n/locales//` (e.g. `src/i18n/locales/en/settings.json`). The `i18n:check` script validates them — run it after touching translation files. - **Website i18n** is separate: 8 locales under `website/i18n/`. After changing an English doc, page or `` string in `website/`, follow `website/i18n/TRANSLATING.md` ("Keeping translations up to date"); `npm run i18n:check` in `website/` lists what is behind. - **Build pipeline**: `npm run build` is full electron-builder. For iterating on renderer only, use `npm run build-vite` (Vite + tsc, no packaging). - **Product constraints**: the project is free forever and explicitly "not production-grade". Don't add paywalls, premium tiers, or logic that gates a feature on who the user is, and don't add upsell language to the README or UI copy. This is a hard constraint, not a judgement call. (A flag that hides an unfinished capture backend is fine — it gates on readiness, not on the user.) diff --git a/README.md b/README.md index c5a0b2ffb1..e0eb475b0b 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ A raw capture goes in and comes out ready to post — on X, Reddit, YouTube, a d - Timeline snapping guides and an audio waveform to make trimming easier. - Customizable keyboard shortcuts. - Export to MP4 or GIF in multiple aspect ratios and resolutions, rendered and encoded on the GPU (Metal on macOS, D3D11 on Windows, Vulkan on Linux) with an automatic CPU fallback. -- Languages supported: Arabic, Czech, English, Spanish, French, German, Italian, Japanese, Korean, Portuguese (Brazil), Russian, Turkish, Vietnamese, Simplified Chinese, and Traditional Chinese. +- Languages supported: Arabic, Czech, English, Spanish, French, German, Italian, Japanese, Korean, Dutch, Polish, Portuguese (Brazil), Russian, Turkish, Vietnamese, Simplified Chinese, and Traditional Chinese. ## Command-line interface (headless) @@ -103,14 +103,20 @@ Every platform has a recommended route below. On Windows that is the Microsoft S - **Windows**: version 1903+ (build 18362) with Intel 8th Gen / AMD Ryzen 2000 series or newer minimum; Windows 11 with Intel 12th Gen / Ryzen 4000 series or newer recommended - **macOS**: 13 (Ventura) or later — required by ScreenCaptureKit for capture -- **Linux**: `xdg-desktop-portal` and PipeWire for native capture and system audio; recording still works without them through the browser-capture fallback, with fewer capabilities (see [Platform differences](#platform-differences)) +- **Linux**: native X11 capture needs no ScreenCast portal; Wayland capture needs `xdg-desktop-portal` and PipeWire, and system audio needs PipeWire; recording still works without them through the browser-capture fallback, with fewer capabilities (see [Platform differences](#platform-differences)) - **RAM**: 8 GB minimum, 16 GB recommended Full table and notes on older integrated graphics: [system requirements](https://getopenscreen.com/docs/installation#system-requirements). ### macOS -Download the `.dmg` installer directly from the [Releases page](https://github.com/getopenscreen/openscreen/releases) and drag OpenScreen into your Applications folder. Builds from 1.9.0 onward are signed with a Developer ID certificate and notarized by Apple, so Gatekeeper does not block them and no terminal step is needed. +Install with [Homebrew](https://formulae.brew.sh/cask/openscreen): + +```bash +brew install --cask openscreen +``` + +Or download the `.dmg` installer directly from the [Releases page](https://github.com/getopenscreen/openscreen/releases) and drag OpenScreen into your Applications folder. Builds from 1.9.0 onward are signed with a Developer ID certificate and notarized by Apple, so Gatekeeper does not block them and no terminal step is needed. On first launch, open **System Settings > Privacy & Security** and grant the two permissions OpenScreen needs: **Screen Recording** and **Accessibility**. Recording cannot start until both are granted. @@ -213,8 +219,8 @@ You may need to grant screen recording permissions depending on your desktop env Everything in the editor and export is the same on macOS, Windows, and Linux: zooms, backgrounds, motion blur, crop/trim/speed, blur regions, annotations, auto-captions, AI editing, projects, export, and all languages. All three now record through a native capture pipeline; the remaining differences are narrower than they used to be: -- **Native recording**: macOS (ScreenCaptureKit), Windows (Windows Graphics Capture), and Linux (PipeWire via the ScreenCast portal) all record through a native pipeline for higher quality and clean window-level capture. On Linux the browser pipeline stays as an automatic fallback if the helper isn't available. -- **Custom cursors**: on macOS and Windows the real cursor is captured with shape, type, and clicks. Linux captures position and cursor shape through the portal, so the editable cursor overlay works there too. Click effects work on Linux as well, but not through the portal — Wayland exposes no portal for mouse buttons, so the capture helper reads the left button from evdev, which needs read access to your mouse: a udev rule for pointer devices (recommended) or the `input` group, set up as [Mouse clicks on Wayland](https://getopenscreen.com/docs/installation#mouse-clicks-on-wayland) shows. Without that, recording is unaffected and every cursor sample is simply a move. +- **Native recording**: macOS (ScreenCaptureKit), Windows (Windows Graphics Capture), and Linux (PipeWire via the ScreenCast portal, or direct X11 capture on an X11 desktop without one) all record through a native pipeline for higher quality and clean window-level capture. On Linux the browser pipeline stays as an automatic fallback if the helper isn't available. +- **Custom cursors**: on macOS and Windows the real cursor is captured with shape, type, and clicks. On X11, the helper captures cursor shape through XFixes and mouse buttons through XInput2, without access to `/dev/input`. On Wayland, Linux captures position and cursor shape through the portal, so the editable cursor overlay works there too. Click effects work on Wayland as well, but not through the portal — Wayland exposes no portal for mouse buttons, so the capture helper reads the left button from evdev, which needs read access to your mouse: a udev rule for pointer devices (recommended) or the `input` group, set up as [Mouse clicks on Wayland](https://getopenscreen.com/docs/installation#mouse-clicks-on-wayland) shows. Without that, recording is unaffected and every cursor sample is simply a move. - **Webcam**: every platform saves the webcam as a separate file next to the screen recording. Windows captures it natively in the recording helper; macOS and Linux record it in the app. It works as a picture-in-picture overlay on all three. - **System audio** support varies by OS: - **macOS**: works on every supported version. On macOS 14.2+ you'll be prompted to grant audio capture permission. @@ -227,11 +233,13 @@ This repository is the community-maintained continuation of OpenScreen. Official / trusted links: -* Original archived repository: https://github.com/siddharthvaddem/openscreen * Community continuation: https://github.com/getopenscreen/openscreen * Official website: https://getopenscreen.com +* Microsoft Store: https://apps.microsoft.com/detail/9MXQ1HQJL5G5 +* Homebrew: `brew install --cask openscreen` +* Original archived repository: https://github.com/siddharthvaddem/openscreen (stops at v1.5.0, like its `siddharthvaddem/openscreen` Homebrew tap) -For safety, download OpenScreen only from the official GitHub Releases linked from this repository. Third-party websites using the OpenScreen name are not affiliated with this continuation unless explicitly listed here. +For safety, install OpenScreen only from the official GitHub Releases linked from this repository, the Microsoft Store listing above, or the official Homebrew cask. Forks on GitHub and third-party websites using the OpenScreen name are not affiliated with this continuation unless explicitly listed here. There is no `getopenscreen/tap` Homebrew tap. ## Community diff --git a/build/trayTemplate.svg b/build/trayTemplate.svg new file mode 100644 index 0000000000..2a4eaf88a1 --- /dev/null +++ b/build/trayTemplate.svg @@ -0,0 +1,10 @@ + + + + + + + + + diff --git a/crates/compositor/shim.c b/crates/compositor/shim.c index 5f0741ed32..be7ebfeca3 100644 --- a/crates/compositor/shim.c +++ b/crates/compositor/shim.c @@ -8,6 +8,8 @@ AVStream* sn_fmt_stream(AVFormatContext* s, int i) { return s->streams[i]; } unsigned sn_fmt_nb_streams(AVFormatContext* s) { return s->nb_streams; } +// Durée du conteneur en AV_TIME_BASE (µs) ; AV_NOPTS_VALUE (négatif) si inconnue. +int64_t sn_fmt_duration(AVFormatContext* s) { return s->duration; } AVIOContext* sn_fmt_get_pb(AVFormatContext* s) { return s->pb; } void sn_fmt_set_pb(AVFormatContext* s, AVIOContext* p) { s->pb = p; } diff --git a/crates/compositor/src/compositor_linux.rs b/crates/compositor/src/compositor_linux.rs index 3e8003b434..d79d85ab6a 100644 --- a/crates/compositor/src/compositor_linux.rs +++ b/crates/compositor/src/compositor_linux.rs @@ -2630,12 +2630,13 @@ impl Compositor { // Calque ecran : mode 0 (rect droit, NV12 -> RGB) quand la rotation est // neutre, mode 8 (warp bilineaire inverse dans la bbox du quad projete) // sinon. Place par plan_frame (cover-fit + coins arrondis) ; - // `src = g.cut` (crop utilisateur + zoom en UV texture) dans les deux cas. + // `src = g.screen_src(g.cut)` (crop utilisateur + zoom en UV texture, bornes + // inversees pour un clip tourne) dans les deux cas. // // FLOU DE VELOCITE. Au mode 0, `src_prev`/`dst_prev` decrivent le MEME // calque a la frame precedente ; le shader remappe chaque pixel de sortie // par ce couple pour retrouver l'UV qu'il occupait alors, et floute le long - // du segment. `src_prev = g.cut` et non un `cut` d'avant : la coupe est + // du segment. `src_prev = g.screen_src(g.cut)` et non un `cut` d'avant : la coupe est // identique aux deux frames (`plan_frame` ne fait varier que le rect de // DESTINATION entre `s_dst` et `s_dst_prev`), ce que Windows documente // aussi. Le mouvement vient donc entierement de `dst_prev`. @@ -2652,11 +2653,13 @@ impl Compositor { // Et la remontee du contour interieur du chrome au-dessus de l'ecran (`mb.z` au mode 0, // `color.z` au mode 8) : l'arrondi du haut se fait par le cadre. 0 sans fenetre. let top_lift = g.screen_top_lift_px([rw, rh]); + // The screen texture's source rect: a turned clip samples it with reversed bounds. + let src_uv = g.screen_src(g.cut); let screen_layer = match tilt.as_ref() { None => { // Sous le masque d'un layout en bloc, rogne au slot (`mask_flat_screen`). let (dst, src, quad_px, radius_px) = - g.mask_flat_screen(g.s_dst, g.cut, s_px, g.s_radius, [rw, rh]); + g.mask_flat_screen(g.s_dst, src_uv, s_px, g.s_radius, [rw, rh]); LayerCB { dst, src, @@ -2664,7 +2667,7 @@ impl Compositor { radius_px, mode: 0.0, color: [1.0, 1.0, 1.0, 1.0], - src_prev: g.cut, + src_prev: src_uv, dst_prev: g.s_dst_prev, mb: [g.screen_pixel_taps([rw, rh]), g.mb_amount, top_lift, square_top], ..Default::default() @@ -2675,7 +2678,7 @@ impl Compositor { quad, s_px, quad_center_px, - g.cut, + src_uv, g.focus_plane, g.s_radius, top_lift, @@ -3537,7 +3540,7 @@ impl Compositor { self.generate_mips(&mut encoder, &p.mips); } // Passe 0 bis : l'ecran cadre, isole sur transparent, quand il est floute en bloc. - if trail.is_some() { + if g.show_screen && trail.is_some() { let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { label: Some("screen-trail-pass"), color_attachments: &[Some(wgpu::RenderPassColorAttachment { @@ -3623,7 +3626,7 @@ impl Compositor { // Passe 2 : l'ecran, compose par-dessus le fond (eventuellement floute) avec // `LoadOp::Load`. Puis les flous de confidentialite, le curseur, la camera et les autres // annotations, dans cet ordre : celui de Windows et macOS. - { + if g.show_screen { let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { label: Some("fg-pass"), color_attachments: &[Some(wgpu::RenderPassColorAttachment { @@ -3665,7 +3668,7 @@ impl Compositor { // Passe 2 bis : les flous de confidentialite, sur le metrage, AVANT le curseur (parite // Windows et macOS). Le curseur reste net par-dessus, et le cristal de Prism Glow, qui // refracte l'image composee, n'y voit que des pixels deja floutes. - if !privacy_draws.is_empty() { + if g.show_screen && !privacy_draws.is_empty() { let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { label: Some("privacy-pass"), color_attachments: &[Some(wgpu::RenderPassColorAttachment { @@ -3844,7 +3847,7 @@ impl Compositor { let footage = effects .iter() .any(|e| crate::shader_fx::reads_footage(e)) - .then(|| self.encode_footage_copy(&mut encoder, (&sy, &su, &sv), g.content_cut, stw, sth)); + .then(|| self.encode_footage_copy(&mut encoder, (&sy, &su, &sv), g.screen_src(g.content_cut), stw, sth)); for e in with_target(ShaderEffectTarget::Image) { self.encode_shader_effect(&mut encoder, e, (&self.rt, &self.rt_view), &fx_context(e), footage.as_ref()); } @@ -4549,13 +4552,77 @@ mod tests { /// ubuntu-latest) n'a pas de GPU et rend sur lavapipe. Avec la creation /// hardware-stricte, tous ces tests s'y sauteraient en silence — c'est-a-dire /// que le seul endroit ou ils tournent automatiquement ne les executerait pas. - fn gpu() -> Option { - match Gpu::create_auto(false) { - Ok(g) => Some(g), - Err(e) => { - eprintln!("pas d'adaptateur Vulkan ({e:#}) — test saute"); - None - } + /// Partage entre les tests, jamais un device par test : cf. `Gpu::shared_for_tests`. + fn gpu() -> Option<&'static Gpu> { + let gpu = Gpu::shared_for_tests(); + if gpu.is_none() { + eprintln!("pas d'adaptateur Vulkan — test saute"); + } + gpu + } + + /// Vrai si le device a active les deux extensions dont depend l'export dmabuf — + /// la meme condition que la garde de `create_exportable_staging`. + fn has_external_memory(device: &wgpu::Device) -> bool { + unsafe { + device.as_hal::(|hal| { + hal.is_some_and(|hal| { + let enabled = hal.enabled_device_extensions(); + enabled.contains(&ash::khr::external_memory_fd::NAME) + && enabled.contains(&ash::ext::external_memory_dma_buf::NAME) + }) + }) + } + } + + /// Vrai si le device sait exporter en dmabuf un buffer `TRANSFER_DST` de `size` + /// octets dans une memoire HOST_VISIBLE | HOST_COHERENT, ce qu'exige + /// `create_exportable_staging` apres sa garde d'extensions. Les extensions seules + /// ne le promettent pas : Vulkan rend ces capacites par parametres de buffer. + fn supports_exportable_staging(device: &wgpu::Device, size: u64) -> bool { + use ash::vk; + if !has_external_memory(device) { + return false; + } + unsafe { + device.as_hal::(|hal| { + let Some(hal) = hal else { return false }; + let dev = hal.raw_device(); + let phys = hal.raw_physical_device(); + let instance = hal.shared_instance().raw_instance(); + let usage = vk::BufferUsageFlags::TRANSFER_DST; + let handle = vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT; + + let info = vk::PhysicalDeviceExternalBufferInfo::default() + .usage(usage) + .handle_type(handle); + let mut props = vk::ExternalBufferProperties::default(); + instance.get_physical_device_external_buffer_properties(phys, &info, &mut props); + if !props + .external_memory_properties + .external_memory_features + .contains(vk::ExternalMemoryFeatureFlags::EXPORTABLE) + { + return false; + } + + let mut ext_info = vk::ExternalMemoryBufferCreateInfo::default().handle_types(handle); + let bci = vk::BufferCreateInfo::default() + .push_next(&mut ext_info) + .size(size) + .usage(usage) + .sharing_mode(vk::SharingMode::EXCLUSIVE); + let Ok(raw) = dev.create_buffer(&bci, None) else { return false }; + let req = dev.get_buffer_memory_requirements(raw); + dev.destroy_buffer(raw, None); + let mem = instance.get_physical_device_memory_properties(phys); + let want = + vk::MemoryPropertyFlags::HOST_VISIBLE | vk::MemoryPropertyFlags::HOST_COHERENT; + (0..mem.memory_type_count).any(|i| { + req.memory_type_bits & (1 << i) != 0 + && mem.memory_types[i as usize].property_flags.contains(want) + }) + }) } } @@ -4653,10 +4720,16 @@ mod tests { let Some(gpu) = gpu() else { return }; let comp = Compositor::new_sized(&gpu, 320, 180).expect("Compositor::new_sized"); const N: u64 = 4096; - let Some(st) = comp.create_exportable_staging(N) else { - eprintln!("pas d'extensions de memoire externe — test saute"); + // On ne saute QUE si le device ne sait pas faire ce buffer. Sauter sur tout + // `None` laisserait passer en silence une garde de `create_exportable_staging` + // qui refuserait a tort un device capable. + if !supports_exportable_staging(&gpu.device, N) { + eprintln!("pas de buffer dmabuf exportable et coherent sur ce device — test saute"); return; - }; + } + let st = comp + .create_exportable_staging(N) + .expect("staging exportable attendu sur un device qui le supporte"); assert!(st.fd >= 0, "descripteur dmabuf invalide"); assert_eq!(st.size, N); @@ -4672,6 +4745,47 @@ mod tests { assert_eq!(got, pattern, "la memoire exportee ne porte pas ce que wgpu y a ecrit"); } + /// Un device ouvert SANS les extensions de memoire externe doit rendre `None`, pas + /// avorter le processus. C'est le cas d'un portable hybride AMD + NVIDIA : wgpu y + /// prend la NVIDIA sans export dmabuf, `VaapiEncoder::open` reussit sur l'AMD, et + /// l'appel suivant paniquait dans ash (« Unable to load get_memory_fd_khr ») au + /// clic sur Exporter (#1622). + /// + /// Le device est ouvert par `request_device`, le chemin standard de wgpu, qui + /// n'active pas ces extensions — exactement ce que `create_async` garde quand + /// `open_device_with_dmabuf_export` echoue. + #[test] + fn exportable_staging_without_external_memory_is_none_not_an_abort() { + let instance = wgpu::Instance::new(&wgpu::InstanceDescriptor { + backends: wgpu::Backends::VULKAN, + ..Default::default() + }); + let Some(adapter) = pollster::block_on( + instance.request_adapter(&wgpu::RequestAdapterOptions::default()), + ) else { + eprintln!("pas d'adaptateur Vulkan — test saute"); + return; + }; + let (device, queue) = + pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor::default(), None)) + .expect("request_device"); + if has_external_memory(&device) { + eprintln!("wgpu active deja la memoire externe sur ce pilote — test saute"); + return; + } + let gpu = Gpu { + device, + context: queue, + backend: crate::d3d::Backend::Hardware, + feature_level: 0, + }; + let comp = Compositor::new_sized(&gpu, 320, 180).expect("Compositor::new_sized"); + assert!( + comp.create_exportable_staging(4096).is_none(), + "sans extensions de memoire externe, le staging exportable doit etre refuse", + ); + } + /// Un cran de padding en format Auto change la taille de rendu de la preview. `resized` /// realloue les cibles et garde le reste : le masque webcam, et la boite aux lettres ou le /// worker de segmentation depose les suivants. @@ -5284,6 +5398,29 @@ mod tests { const CUTOUT: &str = r#"{"mode":"transparent","blurIntensity":0,"background":null,"modelPath":null}"#; + #[test] + fn camera_only_cutout_reveals_wallpaper_instead_of_screen() { + let Some(gpu) = gpu() else { return }; + let comp = Compositor::new_sized(&gpu, 320, 180).expect("compositor"); + for legacy in [false, true] { + let mut scene = Scene::from_json(&pip_scene_json(CUTOUT)).expect("scene"); + scene.layout.preset = if legacy { "picture-in-picture" } else { "camera-only" }.to_owned(); + if legacy { + scene.camera_fullscreen_regions = serde_json::from_str(r#"[{"startSec":-10,"endSec":10}]"#).expect("legacy camera span"); + } + scene.layout.webcam_rect = Some(crate::scene::SceneRect { + x: 0.0, y: 0.0, width: 1.0, height: 1.0, + }); + comp.set_webcam_mask(&half_mask(8, 8), 8, 8).expect("mask"); + let rgba = compose_pip_scene(&comp, &gpu, scene, true); + assert!(camera_pixels(&rgba) > 10_000, "the foreground camera must remain visible"); + let blue = rgba.chunks_exact(4).filter(|px| px[0] < 10 && (px[1] as i16 - 128).abs() < 10 && px[2] > 245).count(); + assert!(blue > 20_000, "cutout must reveal the blue wallpaper, not the gray screen: {blue}"); + let gray = rgba.chunks_exact(4).filter(|px| (110..150).contains(&px[0]) && (110..150).contains(&px[1]) && (110..150).contains(&px[2])).count(); + assert_eq!(gray, 0, "camera-only leaked hidden screen pixels"); + } + } + /// Le piege que le brief nomme : un mode SANS masque ne doit rien changer. /// /// `effect_code` doit rester a 0 tant que rien n'a ete segmente, sinon le @@ -7215,7 +7352,7 @@ mod tests { eprintln!("OPENSCREEN_FX_BENCH absent — saute"); return; } - let gpu = Gpu::create_auto(false).expect("device Vulkan"); + let gpu = gpu().expect("device Vulkan"); eprintln!("backend : {:?}", gpu.backend); let screen = FakeFrame::new(&gpu, 1920, 1080, |x, y| ((x ^ y) & 0xff) as u8); let cfg = fx_cfg(); @@ -7360,6 +7497,20 @@ impl Compositor { unsafe { self.gpu.device.as_hal::(|hal| { let hal = hal?; + // LE `None` PROMIS PLUS HAUT DOIT ETRE TENU ICI, AVANT TOUT APPEL. Sans + // les deux extensions, `vkGetMemoryFdKHR` n'est pas charge : ash panique + // (« Unable to load get_memory_fd_khr ») dans un contexte qui ne deroule + // pas, et c'est tout le processus qui avorte au clic sur Exporter. + // Arrive sur un portable hybride AMD + NVIDIA : wgpu prend la NVIDIA, + // ouverte sans export dmabuf, tandis que `VaapiEncoder::open` reussit + // sur `renderD128`, l'AMD. Rendre `None` fait retomber l'export sur + // l'encodeur software, comme le prevoit `run_multi_inner`. + let enabled = hal.enabled_device_extensions(); + if !enabled.contains(&ash::khr::external_memory_fd::NAME) + || !enabled.contains(&ash::ext::external_memory_dma_buf::NAME) + { + return None; + } let dev = hal.raw_device().clone(); let phys = hal.raw_physical_device(); let instance = hal.shared_instance().raw_instance(); diff --git a/crates/compositor/src/compositor_macos.rs b/crates/compositor/src/compositor_macos.rs index c1072cbf1b..4e7e98cd6c 100644 --- a/crates/compositor/src/compositor_macos.rs +++ b/crates/compositor/src/compositor_macos.rs @@ -29,7 +29,7 @@ use crate::config::Cfg; use crate::d3d::Gpu; -use crate::ffi::AVFrame; +use crate::ffi::{AVColorSpace, AVFrame}; // Le constant buffer est le MÊME struct des deux côtés — cf. `frame_geometry`. // Constant buffer, params runtime et constantes de sortie : une seule définition pour // les deux backends — cf. `frame_geometry`, qui documente les divergences que @@ -99,8 +99,15 @@ extern "C" { fn CVPixelBufferGetHeightOfPlane(p: *mut std::ffi::c_void, plane_index: usize) -> usize; fn CVPixelBufferGetWidth(p: *mut std::ffi::c_void) -> usize; fn CVPixelBufferGetHeight(p: *mut std::ffi::c_void) -> usize; + fn CVPixelBufferGetPixelFormatType(p: *mut std::ffi::c_void) -> u32; } +/// `kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange` ('x420') et sa variante pleine +/// excursion ('xf20') : le P010 que VideoToolbox sort pour un flux 10 bits (HEVC Main 10). +const CV_420_10_BIPLANAR_VIDEO: u32 = u32::from_be_bytes(*b"x420"); +const CV_420_10_BIPLANAR_FULL: u32 = u32::from_be_bytes(*b"xf20"); +const CV_420_8_BIPLANAR_FULL: u32 = u32::from_be_bytes(*b"420f"); + /// `retain` ObjC sur un `id`. `CVMetalTextureGetTexture` rend une référence /// *empruntée* au `CVMetalTextureRef` qui la porte : relâcher ce dernier sans /// retenir la texture donne un `id` mort. Et ne jamais le relâcher — @@ -198,6 +205,50 @@ impl CVMetalTextureCache { } } +// `MTLTextureSwizzle` ; absent des bindings `metal` 0.29. +const SWZ_ZERO: u8 = 0; +const SWZ_ONE: u8 = 1; +const SWZ_RED: u8 = 2; +const SWZ_GREEN: u8 = 3; +const SWZ_ALPHA: u8 = 5; + +/// Vue de `tex` qui rend `swizzle` (rouge, vert, bleu, alpha). C'est ainsi que `sample_yuv` +/// (`shaders.metal`) lit le format d'une frame sans rien lier de plus à chaque draw : +/// - bleu à 1 sur le plan CbCr : pleine plage (`420f`, `xf20`) ; +/// - vert ou bleu à 1 sur le plan Y : matrice BT.601 ou BT.2020. +/// +/// Sans vue, `R` et `RG` rendent 0 sur les canaux libres : plage limitée et BT.709 par défaut. +fn swizzled_view(tex: &metal::TextureRef, swizzle: [u8; 4]) -> metal::Texture { + use objc::{msg_send, sel, sel_impl}; + #[repr(C)] + struct Swizzle([u8; 4]); + let one = metal::NSRange::new(0, 1); + unsafe { + msg_send![tex, newTextureViewWithPixelFormat:tex.pixel_format() + textureType:tex.texture_type() + levels:one + slices:one + swizzle:Swizzle(swizzle)] + } +} + +/// Le swizzle du plan Y qui porte la matrice de la frame, ou `None` pour BT.709 (aussi le +/// défaut d'un flux non tagué). La matrice vient de `AVFrame::colorspace`, que libavcodec lit +/// dans le VUI : VideoToolbox ne pose pas `kCVImageBufferYCbCrMatrixKey` sur ses buffers. +/// BT.2020 CL n'est pas une simple matrice : il garde le défaut plutôt qu'un rendu NCL faux. +/// +/// Alpha à 0 : échantillons 10 bits (P010), que `yuv_matrix` convertit avec les bornes 10 bits. +fn matrix_swizzle(colorspace: AVColorSpace::Type, ten_bit: bool) -> Option<[u8; 4]> { + let depth = if ten_bit { SWZ_ZERO } else { SWZ_ONE }; + match colorspace { + AVColorSpace::AVCOL_SPC_BT470BG | AVColorSpace::AVCOL_SPC_SMPTE170M => { + Some([SWZ_RED, SWZ_ONE, SWZ_ZERO, depth]) + } + AVColorSpace::AVCOL_SPC_BT2020_NCL => Some([SWZ_RED, SWZ_ZERO, SWZ_ONE, depth]), + _ => None, + } +} + impl Drop for CVMetalTextureCache { fn drop(&mut self) { unsafe { @@ -1156,6 +1207,11 @@ impl Compositor { /// Crée les `MTLTexture` Y (`R8Unorm`) et UV (`RG8Unorm`) de la frame. Zéro copie : /// les textures Metal partagent l'IOSurface du `CVPixelBuffer`. API symétrique de /// `compositor_windows::nv12_srvs`. + /// + /// Un flux 10 bits arrive en P010 : 16 bits par échantillon, valeur dans les bits de + /// poids fort. En `R16Unorm`/`RG16Unorm` le shader lit la même valeur normalisée qu'en + /// NV12 ; en `R8Unorm` chaque échantillon devenait deux texels, d'où l'image en bandes + /// d'une vidéo iPhone (#1390). pub unsafe fn nv12_srvs( &self, frame: *const AVFrame, @@ -1167,8 +1223,22 @@ impl Compositor { ) })?; let cache = &self.metal_texture_cache; - let y = cache.make_texture_from_pixel_buffer(pb, 0, metal::MTLPixelFormat::R8Unorm)?; - let uv = cache.make_texture_from_pixel_buffer(pb, 1, metal::MTLPixelFormat::RG8Unorm)?; + let pixel_format = CVPixelBufferGetPixelFormatType(pb); + let (y_fmt, uv_fmt) = match pixel_format { + CV_420_10_BIPLANAR_VIDEO | CV_420_10_BIPLANAR_FULL => { + (metal::MTLPixelFormat::R16Unorm, metal::MTLPixelFormat::RG16Unorm) + } + _ => (metal::MTLPixelFormat::R8Unorm, metal::MTLPixelFormat::RG8Unorm), + }; + let mut y = cache.make_texture_from_pixel_buffer(pb, 0, y_fmt)?; + let mut uv = cache.make_texture_from_pixel_buffer(pb, 1, uv_fmt)?; + let ten_bit = matches!(pixel_format, CV_420_10_BIPLANAR_VIDEO | CV_420_10_BIPLANAR_FULL); + if let Some(swizzle) = matrix_swizzle((*frame).colorspace, ten_bit) { + y = swizzled_view(&y, swizzle); + } + if matches!(pixel_format, CV_420_8_BIPLANAR_FULL | CV_420_10_BIPLANAR_FULL) { + uv = swizzled_view(&uv, [SWZ_RED, SWZ_GREEN, SWZ_ONE, SWZ_ALPHA]); + } Ok((y, uv)) } @@ -2739,119 +2809,122 @@ impl Compositor { *self.bg_cache.borrow_mut() = Some(BgCache { key, tex }); } } - // --- écran : ombre puis vidéo --- - let s_px = [g.s_dst[2] * rw, g.s_dst[3] * rh]; - // Géométrie du tilt calculée UNE fois : l'ombre et l'écran doivent porter exactement - // le même quadrilatère, sinon l'ombre se décolle dès que l'un des deux change. - let tilt = g.screen_tilt(s_px); - // Flou de mouvement de l'écran CADRÉ (`FrameGeometry::screen_trail`) : ombre, cadre, - // métrage et appareil se dessinent dans un rendu isolé et transparent, que le mode 18 - // recompose ensuite sur le fond le long de la trajectoire de la boîte, ou du plan incliné. - let trail = g.screen_trail([rw, rh]); - let mut enc = if trail { - self.begin_pass( - cmd_buf, - &self.trail, - Some(metal::MTLClearColor::new(0.0, 0.0, 0.0, 0.0)), - &self.pipeline_main, - )? - } else { - self.begin_pass(cmd_buf, &self.rt, None, &self.pipeline_main)? - }; - enc.set_fragment_texture(0, Some(&sy)); - enc.set_fragment_texture(1, Some(&suv)); - let quad_center_px = [ - (g.s_dst[0] + g.s_dst[2] * 0.5) * rw, - (g.s_dst[1] + g.s_dst[3] * 0.5) * rh, - ]; - if cfg.shadow { - let spread = SCREEN_SHADOW_SPREAD_FRAC * g.screen_unit_px; - let offset = g.screen_shadow_offset(); - let opacity = 0.45 * lp.shadow_scale; - // L'ombre suit la silhouette réellement affichée : rect arrondi quand l'écran est - // droit, quadrilatère projeté quand il est penché. Un rect droit derrière un écran - // incliné se lit comme une seconde surface, pas comme son ombre. Avec un cadre de - // fenêtre, c'est le CADRE qui la porte (`shadow_caster`) ; un appareil porte celle de - // sa silhouette 3D (mode 17), pas celle d'un quad. - if let Some(cb) = g.device_shadow_cb([rw, rh], spread, offset, opacity) { - self.draw_solid(enc, &cb); + if g.show_screen { + // --- écran : ombre puis vidéo --- + let s_px = [g.s_dst[2] * rw, g.s_dst[3] * rh]; + // Géométrie du tilt calculée UNE fois : l'ombre et l'écran doivent porter exactement + // le même quadrilatère, sinon l'ombre se décolle dès que l'un des deux change. + let tilt = g.screen_tilt(s_px); + // Flou de mouvement de l'écran CADRÉ (`FrameGeometry::screen_trail`) : ombre, cadre, + // métrage et appareil se dessinent dans un rendu isolé et transparent, que le mode 18 + // recompose ensuite sur le fond le long de la trajectoire de la boîte, ou du plan incliné. + let trail = g.screen_trail([rw, rh]); + let mut enc = if trail { + self.begin_pass( + cmd_buf, + &self.trail, + Some(metal::MTLClearColor::new(0.0, 0.0, 0.0, 0.0)), + &self.pipeline_main, + )? } else { - match g.shadow_caster([rw, rh]) { - ShadowCaster::Upright { dst, size_px, radius } => { - self.draw_shadow(enc, dst, size_px, radius, spread, offset, opacity) + self.begin_pass(cmd_buf, &self.rt, None, &self.pipeline_main)? + }; + enc.set_fragment_texture(0, Some(&sy)); + enc.set_fragment_texture(1, Some(&suv)); + let quad_center_px = [ + (g.s_dst[0] + g.s_dst[2] * 0.5) * rw, + (g.s_dst[1] + g.s_dst[3] * 0.5) * rh, + ]; + if cfg.shadow { + let spread = SCREEN_SHADOW_SPREAD_FRAC * g.screen_unit_px; + let offset = g.screen_shadow_offset(); + let opacity = 0.45 * lp.shadow_scale; + // L'ombre suit la silhouette réellement affichée : rect arrondi quand l'écran est + // droit, quadrilatère projeté quand il est penché. Un rect droit derrière un écran + // incliné se lit comme une seconde surface, pas comme son ombre. Avec un cadre de + // fenêtre, c'est le CADRE qui la porte (`shadow_caster`) ; un appareil porte celle de + // sa silhouette 3D (mode 17), pas celle d'un quad. + if let Some(cb) = g.device_shadow_cb([rw, rh], spread, offset, opacity) { + self.draw_solid(enc, &cb); + } else { + match g.shadow_caster([rw, rh]) { + ShadowCaster::Upright { dst, size_px, radius } => { + self.draw_shadow(enc, dst, size_px, radius, spread, offset, opacity) + } + ShadowCaster::Tilted { corners, center_px, radius, mask } => self.draw_quad_shadow( + enc, &corners, center_px, radius, mask, spread, offset, opacity, + ), } - ShadowCaster::Tilted { corners, center_px, radius, mask } => self.draw_quad_shadow( - enc, &corners, center_px, radius, mask, spread, offset, opacity, - ), } } - } - // Le cadre (mode 14) passe SOUS l'écran, qui ne laisse voir que la barre et le filet. - if let Some(cb) = g.window_frame_cb([rw, rh]) { - self.draw_solid(enc, &cb); - } - let square_top = g.screen_square_top(); - let top_lift = g.screen_top_lift_px([rw, rh]); - let [su0, sv0, su1, sv1] = g.cut; - // Sous le masque d'un layout en bloc, rogné au slot (`FrameGeometry::mask_flat_screen`). - let (dst, src, quad_px, radius_px) = - g.mask_flat_screen(g.s_dst, [su0, sv0, su1, sv1], s_px, g.s_radius, [rw, rh]); - match tilt.as_ref() { - None => self.draw_video( - enc, - &LayerCB { - dst, - src, - quad_px, - radius_px, - mode: 0.0, - color: [0.0, 0.0, 0.0, 1.0], - src_prev: [su0, sv0, su1, sv1], - dst_prev: g.s_dst_prev, - mb: [g.screen_pixel_taps([rw, rh]), g.mb_amount, top_lift, square_top], - ..Default::default() - }, - &sy, - &suv, - ), - Some(quad) => self.draw_tilted_screen( - enc, - quad, - s_px, - quad_center_px, - g.cut, - g.focus_plane, - g.s_radius, - top_lift, - g.screen_mask, - g.tilt_pixel_trail([rw, rh]), - &sy, - &suv, - dof_pyramid.as_ref(), - ), - } - // L'appareil modelé (mode 17) passe APRÈS l'écran, et non sous lui comme le chrome plat : - // son socle vient DEVANT le plan du métrage et sa lunette mord dessus. Le shader s'arrête - // au plan du métrage dans l'ouverture, donc il ne recouvre jamais l'image. - if let Some(cb) = g.device_frame_cb([rw, rh]) { - self.draw_solid(enc, &cb); - } - if trail { - enc.end_encoding(); - enc = self.begin_pass(cmd_buf, &self.rt, None, &self.pipeline_main)?; - // texture(2) = l'écran cadré isolé ; texture(5) = la pyramide, avec laquelle le repli - // relit le métrage hors de la sortie comme le mode 8 l'a dessiné. - enc.set_fragment_texture(2, Some(&self.trail)); - enc.set_fragment_texture(5, dof_pyramid.as_ref().map(|t| &**t)); - self.draw_video(enc, &g.screen_trail_cb([rw, rh], dof_pyramid.is_some()), &sy, &suv); - } + // Le cadre (mode 14) passe SOUS l'écran, qui ne laisse voir que la barre et le filet. + if let Some(cb) = g.window_frame_cb([rw, rh]) { + self.draw_solid(enc, &cb); + } + let square_top = g.screen_square_top(); + let top_lift = g.screen_top_lift_px([rw, rh]); + // The screen texture's source rect: a turned clip samples it with reversed bounds. + let [su0, sv0, su1, sv1] = g.screen_src(g.cut); + // Sous le masque d'un layout en bloc, rogné au slot (`FrameGeometry::mask_flat_screen`). + let (dst, src, quad_px, radius_px) = + g.mask_flat_screen(g.s_dst, [su0, sv0, su1, sv1], s_px, g.s_radius, [rw, rh]); + match tilt.as_ref() { + None => self.draw_video( + enc, + &LayerCB { + dst, + src, + quad_px, + radius_px, + mode: 0.0, + color: [0.0, 0.0, 0.0, 1.0], + src_prev: [su0, sv0, su1, sv1], + dst_prev: g.s_dst_prev, + mb: [g.screen_pixel_taps([rw, rh]), g.mb_amount, top_lift, square_top], + ..Default::default() + }, + &sy, + &suv, + ), + Some(quad) => self.draw_tilted_screen( + enc, + quad, + s_px, + quad_center_px, + g.screen_src(g.cut), + g.focus_plane, + g.s_radius, + top_lift, + g.screen_mask, + g.tilt_pixel_trail([rw, rh]), + &sy, + &suv, + dof_pyramid.as_ref(), + ), + } + // L'appareil modelé (mode 17) passe APRÈS l'écran, et non sous lui comme le chrome plat : + // son socle vient DEVANT le plan du métrage et sa lunette mord dessus. Le shader s'arrête + // au plan du métrage dans l'ouverture, donc il ne recouvre jamais l'image. + if let Some(cb) = g.device_frame_cb([rw, rh]) { + self.draw_solid(enc, &cb); + } + if trail { + enc.end_encoding(); + enc = self.begin_pass(cmd_buf, &self.rt, None, &self.pipeline_main)?; + // texture(2) = l'écran cadré isolé ; texture(5) = la pyramide, avec laquelle le repli + // relit le métrage hors de la sortie comme le mode 8 l'a dessiné. + enc.set_fragment_texture(2, Some(&self.trail)); + enc.set_fragment_texture(5, dof_pyramid.as_ref().map(|t| &**t)); + self.draw_video(enc, &g.screen_trail_cb([rw, rh], dof_pyramid.is_some()), &sy, &suv); + } - enc.end_encoding(); + enc.end_encoding(); - // --- flous de confidentialité : sur le métrage, AVANT le curseur (parité Windows). Le - // curseur reste net par-dessus, et le cristal de Prism Glow, qui réfracte l'image - // composée, n'y voit que des pixels déjà floutés. - self.draw_annotations(cmd_buf, &self.rt, scene_ref.as_ref(), g.source_t, g.s_ann, &g, true)?; + // --- flous de confidentialité : sur le métrage, AVANT le curseur (parité Windows). Le + // curseur reste net par-dessus, et le cristal de Prism Glow, qui réfracte l'image + // composée, n'y voit que des pixels déjà floutés. + self.draw_annotations(cmd_buf, &self.rt, scene_ref.as_ref(), g.source_t, g.s_ann, &g, true)?; + } // La pointe et les clics récents, tels que les effets shader les lisent : ceux de // l'enregistrement, que le curseur soit dessiné ou non (`effect_pointer`). @@ -3090,7 +3163,7 @@ impl Compositor { }; // L'enregistrement à sa résolution, peint une fois pour la frame si un effet le lit. let footage = if effects.iter().any(|e| crate::shader_fx::reads_footage(e)) { - Some(self.fill_footage_copy(cmd_buf, &sy, &suv, g.content_cut, stw, sth)?) + Some(self.fill_footage_copy(cmd_buf, &sy, &suv, g.screen_src(g.content_cut), stw, sth)?) } else { None }; @@ -3816,6 +3889,10 @@ mod tests { fn from_planes(w: u32, h: u32, y: &[u8], uv: &[u8]) -> FakeFrame { let pb = crate::mac_frames::nv12_pixel_buffer_from_planes(w, h, y, uv) .expect("CVPixelBuffer NV12"); + FakeFrame::from_pixel_buffer(w, h, pb) + } + + fn from_pixel_buffer(w: u32, h: u32, pb: crate::mac_frames::CVPixelBufferRef) -> FakeFrame { let mut frame: Box = Box::new(unsafe { std::mem::zeroed() }); frame.format = crate::ffi::AVPixelFormat::AV_PIX_FMT_VIDEOTOOLBOX as i32; frame.data[3] = pb.as_ptr() as *mut u8; @@ -3829,6 +3906,44 @@ mod tests { } } + /// Les vues que `nv12_srvs` tire de chaque format biplanaire de VideoToolbox, sans + /// décoder ni fichier média. Un flux 10 bits (`x420`, `xf20`) se lit en plans 16 bits, + /// sinon l'image part en bandes (#1390). Une frame pleine plage (`420f`, `xf20`) porte un + /// 1 dans le bleu de sa vue CbCr : c'est le drapeau que `sample_yuv` lit. + #[test] + fn nv12_srvs_reads_bit_depth_and_range_from_the_pixel_format() { + use metal::MTLPixelFormat::{R16Unorm, R8Unorm, RG16Unorm, RG8Unorm}; + use objc::{msg_send, sel, sel_impl}; + #[repr(C)] + struct Swizzle([u8; 4]); + const SWIZZLE_ONE: u8 = 1; + + let Ok(gpu) = crate::d3d::Gpu::create(false) else { + eprintln!("pas de device Metal — test sauté"); + return; + }; + let comp = super::Compositor::new_sized(&gpu, 64, 64).expect("Compositor::new_sized"); + for (fourcc, y_fmt, uv_fmt, full_range) in [ + (b"420v", R8Unorm, RG8Unorm, false), + (b"420f", R8Unorm, RG8Unorm, true), + (b"x420", R16Unorm, RG16Unorm, false), + (b"xf20", R16Unorm, RG16Unorm, true), + ] { + let name = std::str::from_utf8(fourcc).unwrap(); + let pb = unsafe { + crate::mac_frames::create_nv12_pixel_buffer(64, 64, u32::from_be_bytes(*fourcc)) + } + .unwrap_or_else(|e| panic!("{name} : {e}")); + let frame = FakeFrame::from_pixel_buffer(64, 64, pb); + let (y, uv) = unsafe { comp.nv12_srvs(frame.as_ptr()) } + .unwrap_or_else(|e| panic!("{name} : {e}")); + assert_eq!(y.pixel_format(), y_fmt, "{name} : plan Y"); + assert_eq!(uv.pixel_format(), uv_fmt, "{name} : plan CbCr"); + let swizzle: Swizzle = unsafe { msg_send![uv, swizzle] }; + assert_eq!(swizzle.0[2] == SWIZZLE_ONE, full_range, "{name} : drapeau pleine plage"); + } + } + /// Scène PiP minimale. `effect` est le JSON de `webcamEffect` (`"null"` pour aucun). /// /// `effects.shadow` vaut 0 À DESSEIN : ce curseur ne pilote plus que l'ombre de l'écran, @@ -3860,6 +3975,10 @@ mod tests { /// d'ombre passerait alors pour une suppression réussie. fn compose_pip(comp: &super::Compositor, effect: &str, shadow: bool) -> Vec { let scene = crate::scene::Scene::from_json(&pip_scene_json(effect)).expect("scene json"); + compose_pip_scene(comp, scene, shadow) + } + + fn compose_pip_scene(comp: &super::Compositor, scene: crate::scene::Scene, shadow: bool) -> Vec { comp.set_live_params(live_params_from_scene(&scene)); comp.set_has_webcam(true); comp.set_scene(Some(scene)); @@ -3891,6 +4010,32 @@ mod tests { const NO_EFFECT: &str = "null"; const CUTOUT: &str = r#"{"mode":"transparent","blurIntensity":0,"background":null,"modelPath":null}"#; + #[test] + fn camera_only_cutout_reveals_wallpaper_instead_of_screen() { + let Ok(gpu) = crate::d3d::Gpu::create(false) else { + eprintln!("no Metal device — test skipped"); + return; + }; + let comp = Compositor::new_sized(&gpu, 320, 180).expect("compositor"); + for legacy in [false, true] { + let mut scene = crate::scene::Scene::from_json(&pip_scene_json(CUTOUT)).expect("scene"); + scene.layout.preset = if legacy { "picture-in-picture" } else { "camera-only" }.to_owned(); + if legacy { + scene.camera_fullscreen_regions = serde_json::from_str(r#"[{"startSec":-10,"endSec":10}]"#).expect("legacy camera span"); + } + scene.layout.webcam_rect = Some(crate::scene::SceneRect { + x: 0.0, y: 0.0, width: 1.0, height: 1.0, + }); + comp.set_webcam_mask(&half_mask(8, 8), 8, 8).expect("mask"); + let rgba = compose_pip_scene(&comp, scene, true); + assert!(camera_pixels(&rgba) > 10_000, "the foreground camera must remain visible"); + let blue = rgba.chunks_exact(4).filter(|px| px[0] < 10 && (px[1] as i16 - 128).abs() < 10 && px[2] > 245).count(); + assert!(blue > 20_000, "cutout must reveal the blue wallpaper, not the gray screen: {blue}"); + let gray = rgba.chunks_exact(4).filter(|px| (110..150).contains(&px[0]) && (110..150).contains(&px[1]) && (110..150).contains(&px[2])).count(); + assert_eq!(gray, 0, "camera-only leaked hidden screen pixels"); + } + } + /// Le piège que le brief nomme : un mode SANS masque ne doit rien changer. /// /// `effect_code` doit rester à 0 tant que rien n'a été segmenté, sinon le détourage rend @@ -5342,4 +5487,19 @@ mod tests { crate::shader_fx::compile_metal(&gpu.device, &code, ¶ms).expect("compile"); eprintln!("recompilation d'une source déjà vue : {:.1} ms", t0.elapsed().as_secs_f64() * 1000.0); } + + #[test] + fn matrix_swizzle_marks_matrix_and_depth() { + use AVColorSpace::*; + assert_eq!( + matrix_swizzle(AVCOL_SPC_SMPTE170M, false), + Some([SWZ_RED, SWZ_ONE, SWZ_ZERO, SWZ_ONE]) + ); + assert_eq!( + matrix_swizzle(AVCOL_SPC_BT2020_NCL, true), + Some([SWZ_RED, SWZ_ZERO, SWZ_ONE, SWZ_ZERO]) + ); + assert_eq!(matrix_swizzle(AVCOL_SPC_BT709, false), None); + assert_eq!(matrix_swizzle(AVCOL_SPC_BT2020_CL, true), None); + } } diff --git a/crates/compositor/src/compositor_windows.rs b/crates/compositor/src/compositor_windows.rs index 586cfc19b3..3231bf5d6a 100644 --- a/crates/compositor/src/compositor_windows.rs +++ b/crates/compositor/src/compositor_windows.rs @@ -202,6 +202,13 @@ pub struct Compositor { /// the copy's size and format on every hit, so a new texture landing on an old address /// is never handed a copy sized for the old one. `clear_srv_cache` frees stale entries. srv_cache: RefCell>, + /// The Y views (`srv_cache`) whose last frame was full range (`AVCOL_RANGE_JPEG`): + /// `draw_video` binds `range_cb[1]` for them, so `sample_yuv` expands them to limited + /// range first (#1531). Software frames never land here: `CpuFrames` normalises them. + full_range_srvs: RefCell>, + /// The shader's `Source` cbuffer (b1), one immutable buffer per range: `[0]` limited, + /// `[1]` full. + range_cb: [ID3D11Buffer; 2], live_params: RefCell, /// Scène pilotée par l'app (contrat) : quand présente, remplace le layout fixture de /// `timeline()`. Voir `scene.rs` / `SceneDescription` (TS). @@ -715,6 +722,20 @@ impl Compositor { }; let mut shader_fx_params_cb: Option = None; dev.CreateBuffer(&fx_params_bd, None, Some(&mut shader_fx_params_cb))?; + let range_buffer = |full: f32| -> Result { + let data = [full, 0.0, 0.0, 0.0]; + let desc = D3D11_BUFFER_DESC { + ByteWidth: std::mem::size_of_val(&data) as u32, + Usage: D3D11_USAGE_IMMUTABLE, + BindFlags: D3D11_BIND_CONSTANT_BUFFER.0 as u32, + ..Default::default() + }; + let init = D3D11_SUBRESOURCE_DATA { pSysMem: data.as_ptr().cast(), ..Default::default() }; + let mut buf: Option = None; + dev.CreateBuffer(&desc, Some(&init), Some(&mut buf))?; + Ok(buf.unwrap()) + }; + let range_cb = [range_buffer(0.0)?, range_buffer(1.0)?]; // --- blend alpha prémultiplié --- let mut bl = D3D11_BLEND_DESC::default(); @@ -856,6 +877,8 @@ impl Compositor { shader_fx: RefCell::new(HashMap::new()), shader_fx_tick: Cell::new(0), shader_fx_cb: shader_fx_cb.unwrap(), + full_range_srvs: RefCell::new(std::collections::HashSet::new()), + range_cb, shader_fx_params_cb: shader_fx_params_cb.unwrap(), shader_fx_blocking: Cell::new(true), text_layer: RefCell::new(None), @@ -1211,6 +1234,12 @@ impl Compositor { }; self.ctx.CopySubresourceRegion(&dst, 0, 0, 0, 0, &src, slice, None); + let y_key = y.as_raw() as usize; + if (*frame).color_range == crate::ffi::AVColorRange::AVCOL_RANGE_JPEG { + self.full_range_srvs.borrow_mut().insert(y_key); + } else { + self.full_range_srvs.borrow_mut().remove(&y_key); + } Ok((y, uv)) } @@ -1506,6 +1535,8 @@ impl Compositor { ) { self.ctx .PSSetShaderResources(0, Some(&[Some(srv_y.clone()), Some(srv_uv.clone())])); + let full = self.full_range_srvs.borrow().contains(&(srv_y.as_raw() as usize)); + self.ctx.PSSetConstantBuffers(1, Some(&[Some(self.range_cb[full as usize].clone())])); self.draw_layer(cb); } @@ -2383,7 +2414,8 @@ impl Compositor { let mb_amount = g.mb_amount; let source_t = g.source_t; let _padding_scale = g.padding_scale; - let cut = g.cut; + // The screen texture's source rect: a turned clip samples it with reversed bounds. + let cut = g.screen_src(g.cut); let s_dst = g.s_dst; let s_dst_prev = g.s_dst_prev; let s_ann = g.s_ann; @@ -2485,7 +2517,7 @@ impl Compositor { self.draw_video( &LayerCB { dst: [-over, -over, 1.0 + 2.0 * over, 1.0 + 2.0 * over], - src: [0.0, 0.0, u_max, v_max], + src: g.screen_src([0.0, 0.0, u_max, v_max]), quad_px: [self.rw(), self.rh()], mode: 0.0, color: [1.0, 1.0, 1.0, 1.0], @@ -2524,141 +2556,143 @@ impl Compositor { } } - // --- screen : crop du clip actif, puis zoom appliqué dans ce rect source (§8) --- - // `for_clip_window` conserve l'index pour distinguer plusieurs clips du même asset. - // `active_crop` déjà résolu plus haut (utilisé pour dimensionner `s_dst`) — une seule - // source de vérité pour ce lookup. - let s_px = [s_dst[2] * self.rw(), s_dst[3] * self.rh()]; - // Coupes calculées plus haut (elles dimensionnent `s_dst`) : le zoom vit désormais - // dans la boîte, la coupe ne porte que le crop. `dst_prev` porte la vélocité du - // motion blur — la coupe, elle, est la même aux deux frames. - let [su0, sv0, su1, sv1] = cut; - let (hu, hv) = ((su1 - su0) * 0.5, (sv1 - sv0) * 0.5); - let [su0_p, sv0_p, su1_p, sv1_p] = cut; - let (hu_p, hv_p) = ((su1_p - su0_p) * 0.5, (sv1_p - sv0_p) * 0.5); - // Géométrie du tilt, calculée UNE fois : l'ombre et l'écran doivent porter exactement le - // même quadrilatère. Deux calculs séparés, c'est une ombre qui se décolle dès qu'un des - // deux change. - let tilt = g.screen_tilt(s_px); - let quad_center_px = - [(s_dst[0] + s_dst[2] * 0.5) * self.rw(), (s_dst[1] + s_dst[3] * 0.5) * self.rh()]; - // L'ombre suit la silhouette réellement affichée : le rect arrondi quand l'écran est - // droit, le quadrilatère projeté quand il est incliné. Un rect droit derrière un écran - // penché ne se lisait pas comme son ombre mais comme une seconde surface. Elle suit - // aussi la croissance de la boîte pendant un zoom (issue #179) : quand la boîte sort - // du cadre, l'ombre en sort avec elle, sans jamais se lire comme une bande noire. - // Avec un cadre de fenêtre, c'est le CADRE qui porte l'ombre (`shadow_caster`), sinon - // elle tomberait sous l'écran seul et la barre de titre flotterait au-dessus. - let render_px = [self.rw(), self.rh()]; - // Flou de mouvement de l'écran CADRÉ (`FrameGeometry::screen_trail`) : ombre, cadre, - // métrage et appareil se dessinent dans un rendu isolé et transparent, que le mode 18 - // recompose ensuite sur le fond le long de la trajectoire de la boîte, ou du plan incliné. - let trail = g.screen_trail(render_px); - if trail { - self.ctx.OMSetRenderTargets(Some(&[Some(self.trail_rtv.clone())]), None); - self.ctx.ClearRenderTargetView(&self.trail_rtv, &[0.0, 0.0, 0.0, 0.0]); - } - if cfg.shadow { - let spread = SCREEN_SHADOW_SPREAD_FRAC * g.screen_unit_px; - let offset = g.screen_shadow_offset(); - let opacity = 0.45 * lp.shadow_scale; - // Un appareil porte l'ombre de sa silhouette 3D (mode 17), pas celle d'un quad. - if let Some(cb) = g.device_shadow_cb(render_px, spread, offset, opacity) { - self.draw_solid(&cb); - } else { - match g.shadow_caster(render_px) { - ShadowCaster::Upright { dst, size_px, radius } => { - self.draw_shadow(dst, size_px, radius, spread, offset, opacity) + if g.show_screen { + // --- screen : crop du clip actif, puis zoom appliqué dans ce rect source (§8) --- + // `for_clip_window` conserve l'index pour distinguer plusieurs clips du même asset. + // `active_crop` déjà résolu plus haut (utilisé pour dimensionner `s_dst`) — une seule + // source de vérité pour ce lookup. + let s_px = [s_dst[2] * self.rw(), s_dst[3] * self.rh()]; + // Coupes calculées plus haut (elles dimensionnent `s_dst`) : le zoom vit désormais + // dans la boîte, la coupe ne porte que le crop. `dst_prev` porte la vélocité du + // motion blur — la coupe, elle, est la même aux deux frames. + let [su0, sv0, su1, sv1] = cut; + let (hu, hv) = ((su1 - su0) * 0.5, (sv1 - sv0) * 0.5); + let [su0_p, sv0_p, su1_p, sv1_p] = cut; + let (hu_p, hv_p) = ((su1_p - su0_p) * 0.5, (sv1_p - sv0_p) * 0.5); + // Géométrie du tilt, calculée UNE fois : l'ombre et l'écran doivent porter exactement le + // même quadrilatère. Deux calculs séparés, c'est une ombre qui se décolle dès qu'un des + // deux change. + let tilt = g.screen_tilt(s_px); + let quad_center_px = + [(s_dst[0] + s_dst[2] * 0.5) * self.rw(), (s_dst[1] + s_dst[3] * 0.5) * self.rh()]; + // L'ombre suit la silhouette réellement affichée : le rect arrondi quand l'écran est + // droit, le quadrilatère projeté quand il est incliné. Un rect droit derrière un écran + // penché ne se lisait pas comme son ombre mais comme une seconde surface. Elle suit + // aussi la croissance de la boîte pendant un zoom (issue #179) : quand la boîte sort + // du cadre, l'ombre en sort avec elle, sans jamais se lire comme une bande noire. + // Avec un cadre de fenêtre, c'est le CADRE qui porte l'ombre (`shadow_caster`), sinon + // elle tomberait sous l'écran seul et la barre de titre flotterait au-dessus. + let render_px = [self.rw(), self.rh()]; + // Flou de mouvement de l'écran CADRÉ (`FrameGeometry::screen_trail`) : ombre, cadre, + // métrage et appareil se dessinent dans un rendu isolé et transparent, que le mode 18 + // recompose ensuite sur le fond le long de la trajectoire de la boîte, ou du plan incliné. + let trail = g.screen_trail(render_px); + if trail { + self.ctx.OMSetRenderTargets(Some(&[Some(self.trail_rtv.clone())]), None); + self.ctx.ClearRenderTargetView(&self.trail_rtv, &[0.0, 0.0, 0.0, 0.0]); + } + if cfg.shadow { + let spread = SCREEN_SHADOW_SPREAD_FRAC * g.screen_unit_px; + let offset = g.screen_shadow_offset(); + let opacity = 0.45 * lp.shadow_scale; + // Un appareil porte l'ombre de sa silhouette 3D (mode 17), pas celle d'un quad. + if let Some(cb) = g.device_shadow_cb(render_px, spread, offset, opacity) { + self.draw_solid(&cb); + } else { + match g.shadow_caster(render_px) { + ShadowCaster::Upright { dst, size_px, radius } => { + self.draw_shadow(dst, size_px, radius, spread, offset, opacity) + } + // Même rayon que le plan incliné lui-même (cf. le dessin du mode 8). + ShadowCaster::Tilted { corners, center_px, radius, mask } => self + .draw_quad_shadow(&corners, center_px, radius, mask, spread, offset, opacity), } - // Même rayon que le plan incliné lui-même (cf. le dessin du mode 8). - ShadowCaster::Tilted { corners, center_px, radius, mask } => self - .draw_quad_shadow(&corners, center_px, radius, mask, spread, offset, opacity), } } - } - // Le cadre (mode 14) passe SOUS l'écran : l'écran le recouvre, ne laissant voir que la - // barre de titre, le filet et les coins bas entre les deux arrondis. - if let Some(cb) = g.window_frame_cb(render_px) { - self.draw_solid(&cb); - } - let square_top = g.screen_square_top(); - // Sous le chrome de fenetre, la remontee de son contour interieur au-dessus de l'ecran : - // l'arrondi du haut se fait par le cadre (`screen_top_lift_px`). 0 sans fenetre. - let top_lift = g.screen_top_lift_px(render_px); - if let Some(quad) = tilt { - // Écran incliné (angle fixe) ou vu par la caméra réelle : warp inverse du mode 8 dans - // la bbox du quad projeté (`tilted_screen_cb`, partagé). Les coins arrondis y sont - // rendus dans le repère DU PLAN : sans eux le plan a des arêtes de couteau qui - // tranchent le contenu en pleine phrase, et l'œil lit une découpe là où il devrait - // lire une inclinaison. - // La pyramide de profondeur de champ en t5, que `draw_video` ne lie pas. `None` quand - // l'effet est coupé : `k = 0`, le shader n'y lit rien. - self.ctx.PSSetShaderResources(5, Some(&[dof_srv.clone()])); - self.draw_video( - &crate::frame_geometry::tilted_screen_cb( - &quad, - s_px, - quad_center_px, + // Le cadre (mode 14) passe SOUS l'écran : l'écran le recouvre, ne laissant voir que la + // barre de titre, le filet et les coins bas entre les deux arrondis. + if let Some(cb) = g.window_frame_cb(render_px) { + self.draw_solid(&cb); + } + let square_top = g.screen_square_top(); + // Sous le chrome de fenetre, la remontee de son contour interieur au-dessus de l'ecran : + // l'arrondi du haut se fait par le cadre (`screen_top_lift_px`). 0 sans fenetre. + let top_lift = g.screen_top_lift_px(render_px); + if let Some(quad) = tilt { + // Écran incliné (angle fixe) ou vu par la caméra réelle : warp inverse du mode 8 dans + // la bbox du quad projeté (`tilted_screen_cb`, partagé). Les coins arrondis y sont + // rendus dans le repère DU PLAN : sans eux le plan a des arêtes de couteau qui + // tranchent le contenu en pleine phrase, et l'œil lit une découpe là où il devrait + // lire une inclinaison. + // La pyramide de profondeur de champ en t5, que `draw_video` ne lie pas. `None` quand + // l'effet est coupé : `k = 0`, le shader n'y lit rien. + self.ctx.PSSetShaderResources(5, Some(&[dof_srv.clone()])); + self.draw_video( + &crate::frame_geometry::tilted_screen_cb( + &quad, + s_px, + quad_center_px, + [su0, sv0, su0 + 2.0 * hu, sv0 + 2.0 * hv], + g.focus_plane, + s_radius, + top_lift, + dof, + render_px, + g.screen_mask, + g.tilt_pixel_trail(render_px), + ), + &sy, + &suv, + ); + self.ctx.PSSetShaderResources(5, Some(&[None])); + } else { + // Sous le masque d'un layout en bloc, rogné au slot (`FrameGeometry::mask_flat_screen`). + let (dst, src, quad_px, radius_px) = g.mask_flat_screen( + s_dst, [su0, sv0, su0 + 2.0 * hu, sv0 + 2.0 * hv], - g.focus_plane, + s_px, s_radius, - top_lift, - dof, render_px, - g.screen_mask, - g.tilt_pixel_trail(render_px), - ), - &sy, - &suv, - ); - self.ctx.PSSetShaderResources(5, Some(&[None])); - } else { - // Sous le masque d'un layout en bloc, rogné au slot (`FrameGeometry::mask_flat_screen`). - let (dst, src, quad_px, radius_px) = g.mask_flat_screen( - s_dst, - [su0, sv0, su0 + 2.0 * hu, sv0 + 2.0 * hv], - s_px, - s_radius, - render_px, - ); - self.draw_video( - &LayerCB { - dst, - src, - quad_px, - radius_px, - mode: 0.0, - color: [0.0, 0.0, 0.0, 1.0], - src_prev: [su0_p, sv0_p, su0_p + 2.0 * hu_p, sv0_p + 2.0 * hv_p], - dst_prev: s_dst_prev, - mb: [g.screen_pixel_taps(render_px), mb_amount, top_lift, square_top], - ..Default::default() - }, - &sy, - &suv, - ); - } - // L'appareil modelé (mode 17) passe APRÈS l'écran, et non sous lui comme le chrome plat : - // son socle vient DEVANT le plan du métrage et sa lunette mord dessus. Le shader s'arrête - // au plan du métrage dans l'ouverture, donc il ne recouvre jamais l'image. - if let Some(cb) = g.device_frame_cb(render_px) { - self.draw_solid(&cb); - } - if trail { - self.ctx.OMSetRenderTargets(Some(&[Some(self.rtv.clone())]), None); - // t2 = l'écran cadré isolé ; t5 = la pyramide, avec laquelle le repli relit le - // métrage hors de la sortie comme le mode 8 l'a dessiné. - self.ctx.PSSetShaderResources(2, Some(&[Some(self.trail_srv.clone())])); - self.ctx.PSSetShaderResources(5, Some(&[dof_srv.clone()])); - self.draw_video(&g.screen_trail_cb(render_px, dof), &sy, &suv); - self.ctx.PSSetShaderResources(2, Some(&[None])); - self.ctx.PSSetShaderResources(5, Some(&[None])); - } + ); + self.draw_video( + &LayerCB { + dst, + src, + quad_px, + radius_px, + mode: 0.0, + color: [0.0, 0.0, 0.0, 1.0], + src_prev: [su0_p, sv0_p, su0_p + 2.0 * hu_p, sv0_p + 2.0 * hv_p], + dst_prev: s_dst_prev, + mb: [g.screen_pixel_taps(render_px), mb_amount, top_lift, square_top], + ..Default::default() + }, + &sy, + &suv, + ); + } + // L'appareil modelé (mode 17) passe APRÈS l'écran, et non sous lui comme le chrome plat : + // son socle vient DEVANT le plan du métrage et sa lunette mord dessus. Le shader s'arrête + // au plan du métrage dans l'ouverture, donc il ne recouvre jamais l'image. + if let Some(cb) = g.device_frame_cb(render_px) { + self.draw_solid(&cb); + } + if trail { + self.ctx.OMSetRenderTargets(Some(&[Some(self.rtv.clone())]), None); + // t2 = l'écran cadré isolé ; t5 = la pyramide, avec laquelle le repli relit le + // métrage hors de la sortie comme le mode 8 l'a dessiné. + self.ctx.PSSetShaderResources(2, Some(&[Some(self.trail_srv.clone())])); + self.ctx.PSSetShaderResources(5, Some(&[dof_srv.clone()])); + self.draw_video(&g.screen_trail_cb(render_px, dof), &sy, &suv); + self.ctx.PSSetShaderResources(2, Some(&[None])); + self.ctx.PSSetShaderResources(5, Some(&[None])); + } - // --- flous de confidentialité : sur le métrage, AVANT le curseur. Le curseur reste net - // par-dessus, et le cristal de Prism Glow, qui réfracte l'image composée, n'y voit que des - // pixels déjà floutés. Les autres annotations restent le calque le plus haut (plus bas). - self.draw_annotations(scene_ref.as_ref(), source_t, s_ann, &g, true); + // --- flous de confidentialité : sur le métrage, AVANT le curseur. Le curseur reste net + // par-dessus, et le cristal de Prism Glow, qui réfracte l'image composée, n'y voit que des + // pixels déjà floutés. Les autres annotations restent le calque le plus haut (plus bas). + self.draw_annotations(scene_ref.as_ref(), source_t, s_ann, &g, true); + } // La pointe et les clics récents, tels que les effets shader les lisent : ceux de // l'enregistrement, que le curseur soit dessiné ou non (`effect_pointer`). @@ -2808,8 +2842,8 @@ impl Compositor { if lp.has_webcam { // L'ombre portée appartient à la bulle flottante PiP : elle se retire avec elle // (`shape_fade`), pour qu'au plein écran plus rien n'encadre la caméra. C'est une - // ombre légère NON paramétrable — indépendante du slider Shadow, qui ne pilote plus - // que l'écran (`WEBCAM_SHADOW_OPACITY`, pas `shadow_scale`) — et propre au PiP : les + // ombre indépendante du slider Shadow de l'écran, réglable via `g.webcam_shadow`, + // et propre au PiP : les // blocs side-by-side / top-bottom soudent la caméra à l'écran et n'en portent aucune // (parité `preset.shadow` web, `null` hors PiP dans `compositeLayout.ts`). // L'ombre appartient à la bulle PiP. En détourage il n'y a plus de bulle — une @@ -2907,7 +2941,7 @@ impl Compositor { }; // L'enregistrement à sa résolution, peint une fois pour la frame si un effet le lit. let footage = if effects.iter().any(|e| crate::shader_fx::reads_footage(e)) { - Some(self.fill_footage_copy(&sy, &suv, g.content_cut, stw, sth)?) + Some(self.fill_footage_copy(&sy, &suv, g.screen_src(g.content_cut), stw, sth)?) } else { None }; @@ -3788,6 +3822,7 @@ impl Compositor { /// (p.ex. après un export) pour ne pas retenir indéfiniment des textures de pool. pub fn clear_srv_cache(&self) { self.srv_cache.borrow_mut().clear(); + self.full_range_srvs.borrow_mut().clear(); } } @@ -3904,6 +3939,24 @@ mod tests { } } + /// #1531 : une surface pleine plage (D3D11VA, `yuvj420p`) marque sa vue Y pour que + /// `draw_video` lie `range_cb[1]` ; la même texture redevenue limitée la démarque. + #[test] + fn full_range_frames_mark_their_y_view() { + let Ok(gpu) = crate::d3d::Gpu::create_auto(false) else { + eprintln!("pas de device D3D11 — test sauté"); + return; + }; + let comp = Compositor::new_sized(&gpu, 320, 180).expect("compositeur"); + let (mut frame, _tex) = nv12_frame(&gpu, 64, 64); + frame.color_range = crate::ffi::AVColorRange::AVCOL_RANGE_JPEG; + let (y, _) = unsafe { comp.nv12_srvs(&*frame) }.expect("source pleine"); + assert!(comp.full_range_srvs.borrow().contains(&(y.as_raw() as usize))); + frame.color_range = crate::ffi::AVColorRange::AVCOL_RANGE_MPEG; + let (y, _) = unsafe { comp.nv12_srvs(&*frame) }.expect("source limitée"); + assert!(!comp.full_range_srvs.borrow().contains(&(y.as_raw() as usize))); + } + /// Un décodeur remplacé sans `clear_srv_cache` (changement de résolution en cours de flux, /// `CpuFrames::ensure_tex`) peut donner à une texture NEUVE l'adresse d'une ancienne. La /// copie privée gardée pour l'ancienne est alors trop petite, `CopySubresourceRegion` la @@ -4122,6 +4175,33 @@ mod tests { assert!(left[0].abs_diff(right[0]) < 128, "l'image composée sépare les colonnes : {left:?} {right:?}"); } + /// Sur un clip retourné, `footage()` lit l'enregistrement tel qu'on le voit, retourné : la + /// colonne 10 affichée est la colonne 501 de la texture, noire. + #[test] + fn footage_reads_a_turned_clip_turned() { + let Ok(gpu) = crate::d3d::Gpu::create_auto(false) else { + eprintln!("pas de device D3D11 — test sauté"); + return; + }; + let pixel = |turned: bool| { + let comp = Compositor::new_sized(&gpu, 320, 180).expect("compositeur"); + let code = "fn effect(uv: vec2) -> vec4 { return footage(vec2(10.5 / 512.0, 0.5)); }"; + let mut json = shader_fx_json(serde_json::json!([fx(code, "image")])); + if turned { + json["rotationByClip"] = serde_json::json!([180]); + } + comp.set_scene(Some(Scene::from_json(&json.to_string()).expect("scène"))); + let (screen, _tex) = nv12_columns(&gpu, 512, 512); + comp.set_timeline_time(Some(2.0)); + comp.set_programme_time(Some(2.0)); + unsafe { comp.compose_frame(&*screen, &*screen, 0.0, &fx_cfg()) }.expect("compose"); + let (_, _, rgba) = unsafe { comp.readback_direct() }.expect("readback"); + rgba[0] + }; + assert!(pixel(false) >= 253, "droit : colonne 10 blanche"); + assert!(pixel(true) <= 2, "retourné : colonne 501 noire"); + } + /// La copie de l'enregistrement coûte une passe : un effet qui ne lit pas `footage()` ne la /// fait jamais faire. #[test] diff --git a/crates/compositor/src/cpu_frames_windows.rs b/crates/compositor/src/cpu_frames_windows.rs index ea0e921fbe..cd96e12c61 100644 --- a/crates/compositor/src/cpu_frames_windows.rs +++ b/crates/compositor/src/cpu_frames_windows.rs @@ -24,9 +24,9 @@ use windows::Win32::Graphics::Dxgi::Common::{DXGI_FORMAT_NV12, DXGI_SAMPLE_DESC} /// Le flag d'algorithme de swscale. Bindgen ne génère pas les `SWS_*` d'algorithme (des /// macros), et leurs valeurs sont figées par l'ABI de libswscale. `POINT` (plus proche -/// voisin) est le choix honnête : la conversion se fait à dimensions ÉGALES, donc aucun -/// rééchantillonnage n'a lieu — seul le convertisseur de format travaille, et le filtre -/// choisi n'a aucun effet sur la sortie. +/// voisin) est le choix honnête : la conversion se fait à dimensions ÉGALES, donc la luma +/// n'est jamais rééchantillonnée. Seule exception : une source 4:2:2 ou 4:4:4 (H.264 +/// importé) perd sa chroma pleine en passant en NV12, et POINT la décime par voisin. const SWS_POINT: i32 = 0x10; /// Source de frames du backend CPU, attachée à un `Decoder` quand `Backend::Cpu`. @@ -34,10 +34,10 @@ pub(crate) struct CpuFrames { dev: d3d11::ID3D11Device, ctx: d3d11::ID3D11DeviceContext, sws: *mut SwsContext, - /// `(w, h, format source)` du contexte swscale courant. Un flux qui change de - /// résolution en cours de route (rare mais légal) le reconstruit au lieu de + /// `(w, h, format source, plage pleine)` du contexte swscale courant. Un flux qui change + /// de résolution en cours de route (rare mais légal) le reconstruit au lieu de /// convertir de travers. - sws_key: (i32, i32, i32), + sws_key: (i32, i32, i32, bool), /// NV12 en mémoire système : la cible de swscale, la source de l'upload. nv12: *mut AVFrame, /// La texture NV12 échantillonnée par les shaders. UNE seule, réécrite à chaque @@ -65,7 +65,7 @@ impl CpuFrames { dev: gpu.device.clone(), ctx: gpu.context.clone(), sws: ptr::null_mut(), - sws_key: (0, 0, -1), + sws_key: (0, 0, -1, false), nv12, tex: None, tex_dims: (0, 0), @@ -81,7 +81,8 @@ impl CpuFrames { if w <= 0 || h <= 0 { bail!("frame décodée sans dimensions ({w}x{h})"); } - self.ensure_sws(w, h, (*src).format)?; + let full = (*src).color_range == AVColorRange::AVCOL_RANGE_JPEG; + self.ensure_sws(w, h, (*src).format, full)?; self.ensure_nv12(w, h)?; // Les plans NV12 de destination sont ceux de `self.nv12` : swscale écrit @@ -109,26 +110,53 @@ impl CpuFrames { Ok(self.present) } - unsafe fn ensure_sws(&mut self, w: i32, h: i32, src_fmt: i32) -> Result<()> { - let key = (w, h, src_fmt); + /// `full` : la frame est en plage pleine (`AVCOL_RANGE_JPEG`). swscale ne le déduit que + /// des formats `yuvj*` ; un `yuv420p` étiqueté plein serait copié tel quel, et le shader, + /// qui lit en plage limitée, écraserait ses ombres et ses hautes lumières (#1531). On le + /// lui dit donc : source pleine, destination limitée. + unsafe fn ensure_sws(&mut self, w: i32, h: i32, src_fmt: i32, full: bool) -> Result<()> { + let key = (w, h, src_fmt, full); if self.sws_key == key && !self.sws.is_null() { return Ok(()); } if !self.sws.is_null() { sws_freeContext(self.sws); + self.sws = ptr::null_mut(); } - self.sws = sws_getContext( - w, - h, - src_fmt as AVPixelFormat::Type, - w, - h, - AVPixelFormat::AV_PIX_FMT_NV12, - SWS_POINT, - ptr::null_mut(), - ptr::null_mut(), - ptr::null(), - ); + self.sws = if full { + // La plage se pose AVANT `sws_init_context` : c'est elle qui choisit le + // convertisseur, et le raccourci yuv420p → NV12 copie sans la regarder. + let c = sws_alloc_context(); + if !c.is_null() { + (*c).src_w = w; + (*c).src_h = h; + (*c).src_format = src_fmt; + (*c).dst_w = w; + (*c).dst_h = h; + (*c).dst_format = AVPixelFormat::AV_PIX_FMT_NV12 as i32; + (*c).flags = SWS_POINT as u32; + (*c).src_range = 1; + (*c).dst_range = 0; + if sws_init_context(c, ptr::null_mut(), ptr::null_mut()) < 0 { + sws_freeContext(c); + bail!("sws_init_context plage pleine {w}x{h} fmt {src_fmt} → NV12"); + } + } + c + } else { + sws_getContext( + w, + h, + src_fmt as AVPixelFormat::Type, + w, + h, + AVPixelFormat::AV_PIX_FMT_NV12, + SWS_POINT, + ptr::null_mut(), + ptr::null_mut(), + ptr::null(), + ) + }; if self.sws.is_null() { bail!("sws_getContext {w}x{h} fmt {src_fmt} → NV12"); } @@ -245,3 +273,45 @@ impl Drop for CpuFrames { } } } + +#[cfg(test)] +mod tests { + use super::*; + + /// La luma et la chroma (Cb, Cr) NV12 que `present` produit d'un yuv420p uni de luma `y` + /// et de chroma neutre, étiqueté `range`. + unsafe fn nv12_luma(frames: &mut CpuFrames, y: u8, range: AVColorRange::Type) -> (u8, u8, u8) { + let mut src = av_frame_alloc(); + (*src).width = 16; + (*src).height = 16; + (*src).format = AVPixelFormat::AV_PIX_FMT_YUV420P as i32; + (*src).color_range = range; + assert!(av_frame_get_buffer(src, 32) >= 0); + ptr::write_bytes((*src).data[0], y, (*src).linesize[0] as usize * 16); + ptr::write_bytes((*src).data[1], 128, (*src).linesize[1] as usize * 8); + ptr::write_bytes((*src).data[2], 128, (*src).linesize[2] as usize * 8); + frames.present(src).expect("present"); + av_frame_free(&mut src); + let n = &*frames.nv12; + let uv = n.data[1].add(n.linesize[1] as usize * 4 + 8); + (*n.data[0].add(n.linesize[0] as usize * 8 + 8), *uv, *uv.add(1)) + } + + /// #1531 : un yuv420p plein est ramené en plage limitée, un limité passe tel quel ; la + /// chroma neutre reste à 128 (un gris plein ne se teinte pas). + #[test] + fn full_range_yuv_is_normalised_to_limited() { + let Ok(gpu) = crate::d3d::Gpu::create_auto(false) else { + eprintln!("pas de device D3D11 — test sauté"); + return; + }; + let mut frames = CpuFrames::new(&gpu).expect("CpuFrames"); + unsafe { + use AVColorRange::*; + assert_eq!(nv12_luma(&mut frames, 0, AVCOL_RANGE_JPEG), (16, 128, 128)); + assert_eq!(nv12_luma(&mut frames, 255, AVCOL_RANGE_JPEG), (235, 128, 128)); + assert_eq!(nv12_luma(&mut frames, 0, AVCOL_RANGE_UNSPECIFIED), (0, 128, 128)); + assert_eq!(nv12_luma(&mut frames, 255, AVCOL_RANGE_MPEG), (255, 128, 128)); + } + } +} diff --git a/crates/compositor/src/cursor.rs b/crates/compositor/src/cursor.rs index 4eb981919c..cec1353112 100644 --- a/crates/compositor/src/cursor.rs +++ b/crates/compositor/src/cursor.rs @@ -57,7 +57,9 @@ pub struct CursorTrack { std::sync::Arc>)>>>, } -type ViewKey = (u32, u32, u32); +type ViewKey = (u32, u32, u32, [u32; 4]); +/// L'image entière comme recadrage `[x0, y0, x1, y1]` : le repère de la piste elle-même. +pub const WHOLE_FRAME: [f32; 4] = [0.0, 0.0, 1.0, 1.0]; /// Assez pour toutes les régions auto d'un clip ; au-delà, la plus ancienne est rejouée. const VIEW_CACHE_LEN: usize = 32; @@ -119,6 +121,14 @@ fn smooth_follow_samples(samples: &[(f32, f32, f32)]) -> Vec<(f32, f32, f32)> { smoothed } +/// Un point de la piste dans le recadrage `[x0, y0, x1, y1]`, en 0..1 de celui-ci. Sans borne : +/// un pointeur hors du recadrage tire la vue vers ce bord, comme hors de l'écran. L'image +/// entière rend le point tel quel, au bit près. +fn in_crop(crop: [f32; 4], (x, y): (f32, f32)) -> (f32, f32) { + let [x0, y0, x1, y1] = crop; + ((x - x0) / (x1 - x0).max(1e-6), (y - y0) / (y1 - y0).max(1e-6)) +} + /// Un pas du lissage de suivi : `from` avance vers `to` d'un facteur qui croît avec la distance /// (loin = rattrape vite, près = décélère), corrigé en temps sur `dt_ms`. fn follow_step(from: (f32, f32), to: (f32, f32), dt_ms: f32) -> (f32, f32) { @@ -164,6 +174,23 @@ impl CursorTrack { } } + /// The same track seen on a picture turned by 180°: every position mirrored, times, + /// clicks, visibility and cursor types unchanged. A turned clip is planned entirely in + /// what the viewer sees, so its telemetry is turned once here rather than at every reader. + pub fn turned(&self) -> CursorTrack { + let flip = + |v: &[(f32, f32, f32)]| v.iter().map(|&(t, x, y)| (t, 1.0 - x, 1.0 - y)).collect(); + CursorTrack { + samples: flip(&self.samples), + follow_samples: flip(&self.follow_samples), + clicks: self.clicks.clone(), + click_points: self.click_points.iter().map(|&(x, y)| (1.0 - x, 1.0 - y)).collect(), + visibility: self.visibility.clone(), + types: self.types.clone(), + view_cache: Default::default(), + } + } + /// Visibilité enregistrée au temps `t`. Les anciens sidecars qui n'ont pas ce champ sont /// visibles par défaut, comme les lecteurs TypeScript. pub fn visible_at(&self, t: f32) -> bool { @@ -269,13 +296,25 @@ impl CursorTrack { /// rattrape avec le même lissage, juste assez pour le ramener au bord de la zone. Bornée à ce /// qu'une vue de `1/scale` peut atteindre, pour que la zone se mesure contre la vue affichée. /// + /// Tout se joue dans le recadrage `crop` du clip (`[x0, y0, x1, y1]` dans le repère de la + /// piste) : la vue zoomée en est une part, et le focus rendu s'y exprime en 0..1, comme le + /// focus manuel. Lu dans l'image entière, le pointeur sortait de la vue dès qu'un recadrage + /// la décentrait. + /// /// Rejouée depuis `t0` sur les échantillons BRUTS, puis mise en cache jusqu'à `t_end` : une /// fonction pure de `(t0, t, t_end)`, donc la même en preview et à l'export, après un seek /// comme en lecture. Le cache ne garde rien après la fenêtre de la région. - pub fn follow_in_view(&self, t0: f32, t: f32, t_end: f32, scale: f32) -> Option<(f32, f32)> { + pub fn follow_in_view( + &self, + t0: f32, + t: f32, + t_end: f32, + scale: f32, + crop: [f32; 4], + ) -> Option<(f32, f32)> { let t_end = t_end.max(t0); let t = t.clamp(t0, t_end); - let states = self.view_states(t0, t_end, scale)?; + let states = self.view_states(t0, t_end, scale, crop)?; let half = 0.5 / scale.max(1.0); let reach = |v: f32| v.clamp(half, 1.0 - half); // Après le dernier échantillon, la vue continue de rattraper le pointeur resté immobile @@ -283,6 +322,7 @@ impl CursorTrack { let &(at, vx, vy) = states.last()?; if t > at { let (_, x, y) = *self.samples.last()?; + let (x, y) = in_crop(crop, (x, y)); let dead = half * FOLLOW_DEAD_ZONE; let pull = |cursor: f32, view: f32| cursor - (cursor - view).clamp(-dead, dead); let (nx, ny) = follow_step((vx, vy), (pull(x, vx), pull(y, vy)), (t - at) * 1000.0); @@ -309,9 +349,10 @@ impl CursorTrack { t0: f32, t_end: f32, scale: f32, + crop: [f32; 4], ) -> Option>> { let t_end = t_end.max(t0); - let key = (t0.to_bits(), t_end.to_bits(), scale.to_bits()); + let key = (t0.to_bits(), t_end.to_bits(), scale.to_bits(), crop.map(f32::to_bits)); if let Some((_, v)) = self.view_cache.lock().ok()?.iter().find(|(k, _)| *k == key) { return Some(v.clone()); } @@ -320,12 +361,13 @@ impl CursorTrack { let reach = |v: f32| v.clamp(half, 1.0 - half); // La cible : le point qui ramène le pointeur au bord de la zone, la vue elle-même dedans. let pull = |cursor: f32, view: f32| cursor - (cursor - view).clamp(-dead, dead); - let (x0, y0) = self.follow_at(t0)?; + let (x0, y0) = in_crop(crop, self.follow_at(t0)?); let mut view = (reach(x0), reach(y0)); let mut states = vec![(t0, view.0, view.1)]; let first = self.samples.partition_point(|s| s.0 <= t0); let end = self.samples.partition_point(|s| s.0 <= t_end); for &(ts, x, y) in &self.samples[first..end] { + let (x, y) = in_crop(crop, (x, y)); let at = states.last().map_or(t0, |s| s.0); let (nx, ny) = follow_step(view, (pull(x, view.0), pull(y, view.1)), (ts - at) * 1000.0); @@ -335,6 +377,7 @@ impl CursorTrack { let at = states.last().map_or(t0, |s| s.0); if at < t_end { if let Some(&(ts, x, y)) = self.samples.get(end) { + let (x, y) = in_crop(crop, (x, y)); let (nx, ny) = follow_step(view, (pull(x, view.0), pull(y, view.1)), (ts - at) * 1000.0); let f = ((t_end - at) / (ts - at)).clamp(0.0, 1.0); @@ -672,6 +715,26 @@ fn cursor_spring_config(smoothing_factor: f32) -> (f32, f32, f32) { mod tests { use super::*; + #[test] + fn turned_track_mirrors_positions_and_keeps_times() { + let t = CursorTrack::new( + vec![(0.0, 0.2, 0.3), (1.0, 0.6, 0.9)], + vec![0.5], + vec![(0.0, "arrow".to_string())], + ); + let r = t.turned(); + let (x, y) = r.at(0.0).unwrap(); + assert!((x - 0.8).abs() < 1e-6 && (y - 0.7).abs() < 1e-6, "({x}, {y})"); + assert_eq!(r.clicks_between(0.0, 1.0), t.clicks_between(0.0, 1.0)); + assert_eq!(r.type_at(0.0), t.type_at(0.0)); + let (fx, fy) = r.follow_at(1.0).unwrap(); + let (gx, gy) = t.follow_at(1.0).unwrap(); + assert!((fx - (1.0 - gx)).abs() < 1e-5 && (fy - (1.0 - gy)).abs() < 1e-5); + // The click point turns with the picture: the click at 0.5 s was at (0.4, 0.6). + let (px, py) = r.click_points[0]; + assert!((px - 0.6).abs() < 1e-6 && (py - 0.4).abs() < 1e-6, "({px}, {py})"); + } + /// Zone morte du suivi auto : un pointeur qui tremble au centre de la vue ne la bouge pas ; /// un pointeur qui part loin est rattrapé, jusqu'au bord de la zone et pas au-delà. /// Les vues d'une région sont rejouées une fois puis relues : même réponse dans n'importe quel @@ -692,19 +755,19 @@ mod tests { }; let (cached, times) = (make(), [4.9f32, 0.2, 3.3, 0.0, 2.71, 4.0]); for t in times { - let _ = cached.follow_in_view(0.4, t, 5.0, 2.0); + let _ = cached.follow_in_view(0.4, t, 5.0, 2.0, WHOLE_FRAME); } for t in times { assert_eq!( - cached.follow_in_view(0.4, t, 5.0, 2.0), - make().follow_in_view(0.4, t, 5.0, 2.0), + cached.follow_in_view(0.4, t, 5.0, 2.0, WHOLE_FRAME), + make().follow_in_view(0.4, t, 5.0, 2.0, WHOLE_FRAME), "t {t}" ); } // Une autre région (autre départ, autre échelle) a ses propres vues. assert_ne!( - cached.follow_in_view(1.0, 3.0, 5.0, 3.0), - cached.follow_in_view(0.4, 3.0, 5.0, 2.0) + cached.follow_in_view(1.0, 3.0, 5.0, 3.0, WHOLE_FRAME), + cached.follow_in_view(0.4, 3.0, 5.0, 2.0, WHOLE_FRAME) ); } @@ -721,8 +784,8 @@ mod tests { vec![], ); - let at_end = track.follow_in_view(0.0, 2.0, 2.0, 2.0).unwrap(); - assert_eq!(track.follow_in_view(0.0, 8.0, 2.0, 2.0), Some(at_end)); + let at_end = track.follow_in_view(0.0, 2.0, 2.0, 2.0, WHOLE_FRAME).unwrap(); + assert_eq!(track.follow_in_view(0.0, 8.0, 2.0, 2.0, WHOLE_FRAME), Some(at_end)); let cache = track.view_cache.lock().unwrap(); let states = &cache[0].1; @@ -742,7 +805,7 @@ mod tests { vec![], vec![], ); - let at = |t: f32| track.follow_in_view(0.0, t, 9.0, 2.0).unwrap().0; + let at = |t: f32| track.follow_in_view(0.0, t, 9.0, 2.0, WHOLE_FRAME).unwrap().0; assert!( at(2.0) < at(2.5) && at(2.5) < at(4.0), "{} {} {}", @@ -771,7 +834,7 @@ mod tests { let jitter = track(|t| 0.5 + 0.08 * (t * 7.0).sin()); for i in 0..=90 { let (x, y) = jitter - .follow_in_view(0.0, i as f32 / 10.0, 9.0, 2.0) + .follow_in_view(0.0, i as f32 / 10.0, 9.0, 2.0, WHOLE_FRAME) .unwrap(); assert!( (x - 0.5).abs() < 1e-6 && (y - 0.5).abs() < 1e-6, @@ -787,15 +850,15 @@ mod tests { // Un saut à 0,8 à 2 s : la vue rattrape jusqu'à laisser le pointeur au bord de la zone. let jump = track(|t| if t < 2.0 { 0.5 } else { 0.8 }); - let settled = jump.follow_in_view(0.0, 9.0, 9.0, 2.0).unwrap().0; + let settled = jump.follow_in_view(0.0, 9.0, 9.0, 2.0, WHOLE_FRAME).unwrap().0; assert!((settled - (0.8 - 0.125)).abs() < 1e-3, "{settled}"); - let early = jump.follow_in_view(0.0, 2.1, 9.0, 2.0).unwrap().0; + let early = jump.follow_in_view(0.0, 2.1, 9.0, 2.0, WHOLE_FRAME).unwrap().0; assert!( early > 0.5 && early < settled, "rattrapage progressif : {early}" ); // Une pure fonction de (t0, t, t_end) : l'ordre des appels ne change rien. - assert_eq!(jump.follow_in_view(0.0, 2.1, 9.0, 2.0).unwrap().0, early); + assert_eq!(jump.follow_in_view(0.0, 2.1, 9.0, 2.0, WHOLE_FRAME).unwrap().0, early); } /// L'état du curseur est une fonction en escalier : il tient jusqu'à la transition diff --git a/crates/compositor/src/d3d_linux.rs b/crates/compositor/src/d3d_linux.rs index 3d74edf652..1b11fadb64 100644 --- a/crates/compositor/src/d3d_linux.rs +++ b/crates/compositor/src/d3d_linux.rs @@ -270,6 +270,29 @@ impl Gpu { pub fn probe() -> Option { probe() } + + /// Le device de TOUS les tests unitaires du crate : cree une fois par `create_auto`, + /// jamais detruit. `None` (et le test se saute) sans adaptateur Vulkan. + /// + /// Un device par test faisait planter la suite en SIGSEGV, environ une fois sur deux + /// sur la CI. Le Vulkan-Loader 1.4.341 (celui d'Ubuntu 26.04, donc d'`ubuntu-latest` + /// depuis le 30/10/2026) parcourt la liste globale des devices a chaque + /// `vkSetDebugUtilsObjectNameEXT` (wgpu nomme chaque objet en debug) et a chaque + /// `vkGetDeviceProcAddr`, pendant que `vkDestroyDevice` d'un autre thread libere un + /// noeud de cette liste sous un autre verrou : `loader_get_icd_and_device` lit alors + /// un `loader_device` libere. Corrige en amont par KhronosGroup/Vulkan-Loader#1866, + /// pas dans 1.4.341. Un seul device jamais detruit ne laisse aucune destruction + /// courir pendant que les tests travaillent, sans les serialiser. + #[cfg(test)] + pub fn shared_for_tests() -> Option<&'static Gpu> { + static GPU: OnceLock> = OnceLock::new(); + GPU.get_or_init(|| { + Gpu::create_auto(false) + .map_err(|e| eprintln!("pas d'adaptateur Vulkan ({e:#})")) + .ok() + }) + .as_ref() + } } /// Message d'echec ACTIONNABLE (symetrie d'API avec `d3d_windows::diagnose`, qui diff --git a/crates/compositor/src/d3d_windows.rs b/crates/compositor/src/d3d_windows.rs index 9733b86e02..95c8bec157 100644 --- a/crates/compositor/src/d3d_windows.rs +++ b/crates/compositor/src/d3d_windows.rs @@ -1,5 +1,5 @@ //! Le device D3D11 unique du POC (§2). -//! Un seul `ID3D11Device`, feature level 11_1, flag VIDEO_SUPPORT (décodeur), +//! Un seul `ID3D11Device`, feature level 11_1 ou 11_0, flag VIDEO_SUPPORT (décodeur), //! et `ID3D10Multithread::SetMultithreadProtected(TRUE)` — parce que le décodeur //! ffmpeg et notre boucle de rendu toucheront le device depuis des threads distincts. @@ -9,6 +9,7 @@ use windows::core::Interface; use windows::Win32::Foundation::HMODULE; use windows::Win32::Graphics::Direct3D::{ D3D_DRIVER_TYPE, D3D_DRIVER_TYPE_HARDWARE, D3D_DRIVER_TYPE_WARP, D3D_FEATURE_LEVEL, + D3D_FEATURE_LEVEL_11_0, D3D_FEATURE_LEVEL_11_1, }; use windows::Win32::Graphics::Direct3D11::{ @@ -65,14 +66,14 @@ pub struct Gpu { pub backend: Backend, } -/// Une tentative `D3D11CreateDevice` à FL 11_1. Extraite pour que le chemin d'échec -/// puisse re-sonder avec d'autres flags/driver et dire POURQUOI la vraie tentative +/// Une tentative `D3D11CreateDevice` à FL 11_1, puis 11_0 en repli. Extraite pour que le +/// chemin d'échec puisse re-sonder avec d'autres flags/driver et dire POURQUOI la vraie tentative /// a échoué (voir `diagnose`), au lieu de remonter un HRESULT nu. fn try_create( driver: D3D_DRIVER_TYPE, flags: D3D11_CREATE_DEVICE_FLAG, ) -> windows::core::Result<(ID3D11Device, ID3D11DeviceContext, D3D_FEATURE_LEVEL)> { - let levels = [D3D_FEATURE_LEVEL_11_1]; + let levels = [D3D_FEATURE_LEVEL_11_1, D3D_FEATURE_LEVEL_11_0]; let mut device: Option = None; let mut context: Option = None; let mut got = D3D_FEATURE_LEVEL::default(); @@ -130,9 +131,9 @@ fn diagnose(err: &windows::core::Error) -> String { ); } format!( - "no Direct3D 11 feature level 11_1 display adapter ({err}). OpenScreen's compositor \ + "no Direct3D 11 feature level 11_0 display adapter ({err}). OpenScreen's compositor \ requires one for both preview and export. Update the display driver, or run on a \ - machine with a GPU that reaches feature level 11_1." + machine with a GPU that reaches feature level 11_0." ) } @@ -201,8 +202,8 @@ impl Gpu { Err(err) => bail!("{}", diagnose(&err)), }; - if got != D3D_FEATURE_LEVEL_11_1 { - bail!("feature level obtenu {:?} != 11_1", got); + if got.0 < D3D_FEATURE_LEVEL_11_0.0 { + bail!("feature level obtenu {:?} < 11_0", got); } // §2 : multithread-protected. Le décodeur ffmpeg soumet depuis son thread, diff --git a/crates/compositor/src/export_control.rs b/crates/compositor/src/export_control.rs index 472fdad913..0ef79f338e 100644 --- a/crates/compositor/src/export_control.rs +++ b/crates/compositor/src/export_control.rs @@ -82,14 +82,30 @@ pub(crate) fn with_staged_output( let (mut staged, file) = loop { let nonce = NEXT_OUTPUT.fetch_add(1, Ordering::Relaxed); let path = parent.join(format!(".openscreen-{ext}-{}-{nonce}.partial.{ext}", std::process::id())); - match OpenOptions::new().write(true).create_new(true).open(&path) { + let mut options = OpenOptions::new(); + options.write(true).create_new(true); + // Private while it renders when it will replace a file, which may be private too. + #[cfg(unix)] + if target.is_file() { + std::os::unix::fs::OpenOptionsExt::mode(&mut options, 0o600); + } + match options.open(&path) { Ok(file) => break (StagedOutput { path, published: false }, file), Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue, Err(e) => return Err(e).context("creating temporary export output"), } }; + // Mode and owner go through a handle on the inode we created: in a shared folder the + // staged name could be swapped for a symlink while it renders. + #[cfg(unix)] + let handle = file.try_clone().ok(); + #[cfg(not(unix))] + let handle: Option = None; let result = render(file, &staged.path).and_then(|stats| { control.begin_commit()?; + if let Some(handle) = handle { + keep_permissions_of(target, &handle); + } fs::rename(&staged.path, target).context("publishing completed export output")?; staged.published = true; Ok(stats) @@ -106,6 +122,25 @@ pub(crate) fn with_staged_output( result } +/// The published file keeps the mode of the file it replaces, and its owner and group as +/// far as we may set them, as an in-place write would; the staged file otherwise has the +/// umask's default. Best effort: a filesystem without POSIX modes (exFAT, SMB) must not +/// fail the export. Windows ACLs are not copied: the new file inherits the folder's. +#[cfg(unix)] +fn keep_permissions_of(target: &Path, staged: &File) { + use std::os::unix::fs::{fchown, MetadataExt, PermissionsExt}; + let Ok(old) = fs::metadata(target) else { return }; + if !old.is_file() { + return; + } + // Only root may give a file away; the group alone is allowed to any of our groups. + let _ = fchown(staged, Some(old.uid()), Some(old.gid())).or_else(|_| fchown(staged, None, Some(old.gid()))); + let _ = staged.set_permissions(fs::Permissions::from_mode(old.mode() & 0o777)); +} + +#[cfg(not(unix))] +fn keep_permissions_of(_target: &Path, _staged: &File) {} + /// Render and close an MP4, then verify a displayable decoded frame before /// publishing it. A positive encoder submission count alone is insufficient: /// a zero-duration MP4 sample can be flagged discard by the demuxer. @@ -407,6 +442,28 @@ mod tests { dir.assert_only("destination-directory"); } + /// v2.3.0-rc.4 D5: an MP4 or GIF replacing a 600 or 640 file came back 644/664. + /// 664 and 666 also check the umask is not applied to the kept mode. + #[cfg(unix)] + #[test] + fn publication_keeps_the_mode_of_the_replaced_file() { + use std::os::unix::fs::PermissionsExt; + for mode in [0o600, 0o640, 0o664, 0o666, 0o755] { + let dir = TestDir::new(); + let target = dir.0.join("out.gif"); + fs::write(&target, b"original").unwrap(); + fs::set_permissions(&target, fs::Permissions::from_mode(mode)).unwrap(); + with_staged_output(&target, &ExportControl::default(), |mut file, _| { + file.write_all(b"finished GIF")?; + Ok(()) + }) + .unwrap(); + assert_eq!(fs::read(&target).unwrap(), b"finished GIF"); + let kept = fs::metadata(&target).unwrap().permissions().mode() & 0o777; + assert_eq!(kept, mode, "replaced a {mode:o} file, got {kept:o}"); + } + } + #[test] fn cancellation_and_commit_have_exactly_one_winner() { for _ in 0..64 { diff --git a/crates/compositor/src/ffi.rs b/crates/compositor/src/ffi.rs index 0df8963a4a..670b6ba551 100644 --- a/crates/compositor/src/ffi.rs +++ b/crates/compositor/src/ffi.rs @@ -42,6 +42,8 @@ extern "C" { pub fn sn_fmt_stream(s: *mut AVFormatContext, i: i32) -> *mut AVStream; /// `s->nb_streams` (cf. `shim.c`). pub fn sn_fmt_nb_streams(s: *mut AVFormatContext) -> u32; + /// `s->duration`, en µs (`AV_TIME_BASE`), négatif si inconnue (cf. `shim.c`). + pub fn sn_fmt_duration(s: *mut AVFormatContext) -> i64; /// `s->pb` (cf. `shim.c`). pub fn sn_fmt_get_pb(s: *mut AVFormatContext) -> *mut AVIOContext; /// `s->pb = p` (cf. `shim.c`). @@ -65,6 +67,20 @@ pub fn averr(ret: i32, ctx: &str) -> anyhow::Result<()> { Ok(()) } +/// Étiquette couleur d'un encodeur d'export, à poser avant `avcodec_open2` : l'encodeur +/// l'écrit dans le VUI, `avcodec_parameters_from_context` la recopie et le muxer MP4 en +/// fait l'atome `colr`. Le compositeur rend du sRGB, d'où primaires et transfert BT.709 ; +/// `matrix` est celle de la conversion RGB → YUV de la plateforme, en plage limitée. +/// +/// Sans elle le fichier ne dit rien et chaque lecteur devine : ffmpeg suppose BT.601 et +/// décalait jusqu'à 22 niveaux les couleurs saturées d'un export macOS en BT.709. +pub unsafe fn tag_export_color(ctx: *mut AVCodecContext, matrix: AVColorSpace::Type) { + (*ctx).color_primaries = AVColorPrimaries::AVCOL_PRI_BT709; + (*ctx).color_trc = AVColorTransferCharacteristic::AVCOL_TRC_BT709; + (*ctx).colorspace = matrix; + (*ctx).color_range = AVColorRange::AVCOL_RANGE_MPEG; +} + #[cfg(test)] mod tests { /// Les constantes Rust doivent valoir EXACTEMENT ce que les macros ffmpeg valent diff --git a/crates/compositor/src/frame_geometry.rs b/crates/compositor/src/frame_geometry.rs index e7410ad558..82a67f955c 100644 --- a/crates/compositor/src/frame_geometry.rs +++ b/crates/compositor/src/frame_geometry.rs @@ -1906,6 +1906,8 @@ pub struct FrameGeometryInput<'a> { /// qui traversent. pub struct FrameGeometry { pub scene_preset: Option, + /// A transparent camera must reveal the wallpaper, not hidden screen footage. + pub show_screen: bool, /// Continuous PiP shadow weight through a layout change. pub webcam_shadow: f32, pub mb_taps: f32, @@ -1981,6 +1983,11 @@ pub struct FrameGeometry { /// Layouts en bloc : le slot de l'écran, un conteneur qui le MASQUE (cf. `ScreenMask`). /// `None` ailleurs, et le rendu est celui d'avant, à l'octet. pub screen_mask: Option, + /// The clip being composed is turned by 180°. Everything above is planned in display + /// space; only the screen layer's texture sampling turns, through `screen_src`. + pub screen_turn: bool, + /// `[u_max, v_max]` of the screen texture: the valid area `screen_src` mirrors within. + pub screen_uv_max: [f32; 2], } /// Le conteneur d'un layout en bloc (côte à côte, haut/bas) : un masque fixe, overflow hidden. @@ -2062,6 +2069,18 @@ pub fn annotation_dst_in(anchor: [f32; 4], x: f32, y: f32, w: f32, h: f32) -> [f } impl FrameGeometry { + /// The texture rect to sample for a display-space screen rect. Upright: unchanged. + /// Turned: mirrored inside the valid area, which reverses both bounds — the shaders + /// interpolate linearly from `src.xy` to `src.zw`, so that IS the half turn. `cut` and + /// everything planned from it stay in display space. + pub fn screen_src(&self, uv: [f32; 4]) -> [f32; 4] { + if !self.screen_turn { + return uv; + } + let [um, vm] = self.screen_uv_max; + [um - uv[0], vm - uv[1], um - uv[2], vm - uv[3]] + } + /// Le quad de l'écran incliné pour une boîte de `s_px` px, `None` quand l'écran est droit. /// LE point de passage de l'écran, de son ombre, du curseur et du masque de flou : tous /// doivent porter la même séparation base / dynamique, sinon ils se décollent. @@ -2232,7 +2251,8 @@ impl FrameGeometry { let s_px = [self.s_dst[2] * render_px[0], self.s_dst[3] * render_px[1]]; let flat = LayerCB { dst: [0.0, 0.0, 1.0, 1.0], - src: self.cut, + // Reread from the screen texture: a turned clip samples it with reversed bounds. + src: self.screen_src(self.cut), quad_px: s_px, radius_px: self.s_radius, mode: 18.0, @@ -3159,9 +3179,19 @@ pub fn plan_frame(input: &FrameGeometryInput) -> FrameGeometry { // focus auto) : un plan ne bouge que sous un pointeur qu'on voit. Preview et export // chargent la piste dans tous les cas (le focus auto la suit), cette porte fait le reste. let parallax_track = cursor_for_zoom.filter(|_| scene.is_some_and(|s| s.cursor.show)); - let active_crop = scene.and_then(|scene| { - scene.crop_by_clip.get(scene.active_clip_index).copied().flatten() - }); + // A turned clip is planned in what the viewer sees: its crop (stored in source + // coordinates) is mirrored here, its cursor track is mirrored where it is loaded, and + // only the screen layer's sampling maps back to the texture (`screen_src`). + let screen_turn = scene.is_some_and(|s| s.screen_turned()); + let active_crop = scene + .and_then(|scene| scene.crop_by_clip.get(scene.active_clip_index).copied().flatten()) + .map(|c| { + if screen_turn { + SceneCrop { x: 1.0 - c.x - c.width, y: 1.0 - c.y - c.height, ..c } + } else { + c + } + }); if !zoom_regions.is_empty() { // L'orbite en focus auto lit le curseur dans l'image SOURCE recadrée — pas dans la // coupe zoomée, qu'un focus auto recentre sur lui — et ignore ce qu'une coupe retire. @@ -3675,9 +3705,17 @@ pub fn plan_frame(input: &FrameGeometryInput) -> FrameGeometry { }, }; + // An opaque full-frame camera used to cover the screen by accident. Cutout makes + // that footage visible again, so stop drawing it once either camera-only path settles. + let show_screen = !lp.has_webcam || (cam_progress < 1.0 && match sampled_layout.as_ref() { + Some(sample) => sample.preset != "camera-only" || sample.progress < 1.0, + None => scene_preset.as_deref() != Some("camera-only"), + }); FrameGeometry { + show_screen, webcam_shadow: sampled_layout.as_ref().map(|s| s.webcam_shadow) - .unwrap_or_else(|| if matches!(scene_preset.as_deref(), Some("dual-frame") | Some("vertical-stack") | Some("camera-only")) { 0.0 } else { 1.0 }), + .unwrap_or_else(|| if matches!(scene_preset.as_deref(), Some("dual-frame") | Some("vertical-stack") | Some("camera-only")) { 0.0 } else { 1.0 }) + * input.scene.and_then(|scene| scene.effects.webcam_shadow).unwrap_or(0.6).clamp(0.0, 1.0) / 0.6, scene_preset, mb_taps, mb_amount, @@ -3711,6 +3749,8 @@ pub fn plan_frame(input: &FrameGeometryInput) -> FrameGeometry { shape_fade, window_frame, screen_mask, + screen_turn, + screen_uv_max: [u_max, v_max], } } @@ -3973,6 +4013,9 @@ pub fn effect_pointer(g: &FrameGeometry, input: &CursorPlanInput) -> EffectPoint /// courant (zoom serré, hors écran), ou sprite réduit à rien au creux d'un click bounce /// extrême — un état normal en lecture, pas une erreur. pub fn plan_cursor(g: &FrameGeometry, input: &CursorPlanInput) -> Option { + if !g.show_screen { + return None; + } let (rw, rh) = (input.render_px[0], input.render_px[1]); // Caché par l'application (`visible: false`) : rien de ce plan n'est dessiné, ni le sprite, @@ -4879,17 +4922,23 @@ mod tests { layout_region("no-webcam", 5.0, 10.0, None)]; let cfg = crate::config::all().pop().expect("compositor config"); let mut input = golden_input(&scene, &cfg); + input.live.has_webcam = true; input.timeline_t_override = Some(2.0); let camera = plan_frame(&input); assert_eq!(camera.scene_preset.as_deref(), Some("camera-only")); assert!((camera.w_dst[2] - 1.0).abs() < 0.001); assert!((camera.w_dst[3] - 1.0).abs() < 0.001); assert_eq!(camera.w_radius, 0.0); + assert!(!camera.show_screen, "camera-only must hide the footage even with a transparent camera"); + input.live.has_webcam = false; + assert!(plan_frame(&input).show_screen, "footage without a camera must keep its screen"); + input.live.has_webcam = true; input.timeline_t_override = Some(5.0); let screen = plan_frame(&input); assert_eq!(screen.scene_preset.as_deref(), Some("no-webcam")); assert_eq!(screen.w_dst[2], 0.0); assert_eq!(screen.w_dst[3], 0.0); + assert!(screen.show_screen); } #[test] @@ -4934,6 +4983,13 @@ mod tests { scene.layout_regions = vec![region]; assert!((scene.layout_at(3.1).unwrap().layout.webcam_rect.unwrap().width - 0.6).abs() < 0.001); assert_eq!(scene.layout_at(3.5).unwrap().layout.webcam_rect.unwrap().width, 1.0); + let cfg = crate::config::all().pop().expect("compositor config"); + let mut input = golden_input(&scene, &cfg); + input.live.has_webcam = true; + input.timeline_t_override = Some(3.1); + assert!(plan_frame(&input).show_screen, "the incoming camera has not covered the screen yet"); + input.timeline_t_override = Some(3.5); + assert!(!plan_frame(&input).show_screen, "the settled camera-only passage hides the footage"); assert!(scene.layout_at(10.0).is_none()); let other = scene.for_clip_window(1, 0.0, 10.0); assert!(other.layout_at(3.1).is_none()); @@ -5187,6 +5243,19 @@ mod tests { } } + #[test] + fn webcam_shadow_preserves_old_scenes_and_scales_independently() { + let mut scene = golden_scene(); + let cfg = crate::config::all().pop().expect("compositor config"); + let baseline = plan_frame(&golden_input(&scene, &cfg)); + for (strength, scale) in [(0.0, 0.0), (0.3, 0.5), (0.6, 1.0), (0.9, 1.5)] { + scene.effects.webcam_shadow = Some(strength); + let frame = plan_frame(&golden_input(&scene, &cfg)); + assert!((frame.webcam_shadow - baseline.webcam_shadow * scale).abs() < 1e-5); + assert_eq!(frame.w_dst, baseline.w_dst); + } + } + /// Le JSON de la scène zoomée, brut : `tilted_golden_scene` n'en change QUE la /// rotation, et le faire par substitution garantit que les deux scènes ne diffèrent /// pas ailleurs sans qu'on s'en aperçoive. @@ -7220,6 +7289,124 @@ mod tests { } } + /// A single-clip scene at zoom 1 with the cursor shown, its clip `rotation` (0 or 180) + /// and its crop (`null` or a crop object, verbatim JSON). + fn turn_scene(rotation: u16, crop: &str) -> Scene { + let mut scene = Scene::from_json(&format!( + r##"{{ + "clips":[{{"screenPath":"/s.mp4","webcamPath":"","sourceStartSec":0,"sourceEndSec":10,"webcamOffsetSec":0,"hasAudio":false}}], + "layout":{{"preset":"no-webcam","webcamSize":1,"webcamShape":"rounded","webcamMirror":false,"webcamPosition":null, + "webcamReactiveZoom":false}}, + "effects":{{"padding":0.5,"blur":false,"shadow":0.5,"roundnessFrac":0.02,"motionBlur":0}}, + "background":{{"kind":"color","color":"#1e1e2e"}}, + "zoomRegions":[], + "cursor":{{"show":true,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":1,"clipToBounds":false,"theme":"default"}}, + "cropByClip":[{crop}], + "output":{{"width":1920,"height":1080,"fps":60}} + }}"## + )) + .expect("turn scene"); + scene.rotation_by_clip = vec![rotation]; + scene + } + + #[test] + fn upright_clip_keeps_screen_src_identity() { + let g = framed_plan(&turn_scene(0, "null")); + assert!(!g.screen_turn); + assert_eq!(g.screen_src(g.cut), g.cut); + // An absent field is upright too. + let mut absent = turn_scene(0, "null"); + absent.rotation_by_clip.clear(); + let g = framed_plan(&absent); + assert!(!g.screen_turn); + assert_eq!(g.screen_src(g.cut), g.cut); + } + + #[test] + fn turned_clip_reverses_the_source_bounds_inside_the_valid_area() { + let g = framed_plan(&turn_scene(180, "null")); + assert!(g.screen_turn); + assert_eq!(g.screen_uv_max, [1920.0 / 1920.0, 1080.0 / 1088.0]); + // The display-space cut is the upright one: only the sampling turns. + assert_eq!(g.cut, framed_plan(&turn_scene(0, "null")).cut); + let [a0, b0, a1, b1] = g.screen_src(g.cut); + assert!(a0 > a1 && b0 > b1, "bounds reversed: {:?}", [a0, b0, a1, b1]); + for v in [a0, a1] { + assert!((0.0..=g.screen_uv_max[0]).contains(&v), "u {v}"); + } + for v in [b0, b1] { + assert!((0.0..=g.screen_uv_max[1]).contains(&v), "v {v}"); + } + // Full frame: the whole valid area, corner to opposite corner. + assert!((a0 - 1.0).abs() < 1e-6 && a1.abs() < 1e-6, "u {a0}..{a1}"); + assert!((b0 - 1080.0 / 1088.0).abs() < 1e-6 && b1.abs() < 1e-6, "v {b0}..{b1}"); + } + + #[test] + fn turned_crop_selects_the_same_source_pixels() { + let crop = r#"{"x":0.1,"y":0.2,"width":0.5,"height":0.3}"#; + let g = framed_plan(&turn_scene(180, crop)); + // Planned in display space: the crop is mirrored, x 0.4..0.9, y 0.5..0.8. + let (um, vm) = (g.screen_uv_max[0], g.screen_uv_max[1]); + assert!((g.cut[0] / um - 0.4).abs() < 1e-5 && (g.cut[2] / um - 0.9).abs() < 1e-5, "cut {:?}", g.cut); + assert!((g.cut[1] / vm - 0.5).abs() < 1e-5 && (g.cut[3] / vm - 0.8).abs() < 1e-5, "cut {:?}", g.cut); + // ...and the texture rect sampled is exactly the stored crop, reversed. + let [a0, b0, a1, b1] = g.screen_src(g.cut); + assert!((a1 / um - 0.1).abs() < 1e-5 && (a0 / um - 0.6).abs() < 1e-5, "u {a0}..{a1}"); + assert!((b1 / vm - 0.2).abs() < 1e-5 && (b0 / vm - 0.5).abs() < 1e-5, "v {b0}..{b1}"); + // The screen box keeps the crop's shape either way. + assert_eq!(g.s_dst, framed_plan(&turn_scene(0, crop)).s_dst); + } + + #[test] + fn screen_trail_rereads_a_turned_clip_turned() { + let g = framed_plan(&turn_scene(180, "null")); + assert_eq!(g.screen_trail_cb(RENDER, false).src, g.screen_src(g.cut)); + let upright = framed_plan(&turn_scene(0, "null")); + assert_eq!(upright.screen_trail_cb(RENDER, false).src, upright.cut); + } + + #[test] + fn cursor_of_a_turned_clip_lands_on_the_turned_content() { + let cfg = crate::config::all().pop().expect("au moins une config"); + let center = |rotation: u16, track: &'static crate::cursor::CursorTrack| -> ([f32; 2], [f32; 4]) { + let scene = turn_scene(rotation, "null"); + let g = framed_plan(&scene); + let mut live = live_params_from_scene(&scene); + live.cursor_model3d = false; + let plan = plan_cursor( + &g, + &CursorPlanInput { + render_px: RENDER, + u_max: 1.0, + v_max: 1080.0 / 1088.0, + cfg: &cfg, + live, + scene: Some(&scene), + track, + t: 1.5, + }, + ) + .expect("cursor in view"); + match plan.placement { + CursorPlacement::Upright { center } => (center, g.s_dst), + _ => panic!("upright screen, flat sprite"), + } + }; + let (x, y) = (0.2f32, 0.3f32); + // Upright: the cursor sits at (x, y) of the screen box. + let (c, d) = center(0, parked_track(x, y)); + let want = [d[0] + x * d[2], d[1] + y * d[3]]; + assert!((c[0] - want[0]).abs() < 1e-5 && (c[1] - want[1]).abs() < 1e-5, "{c:?} vs {want:?}"); + // Turned: the track is turned where it is loaded, so the cursor sits where the turned + // picture shows that source point, (1 - x, 1 - y) of the screen box. + let turned: &'static crate::cursor::CursorTrack = Box::leak(Box::new(parked_track(x, y).turned())); + let (c, d) = center(180, turned); + let want = [d[0] + (1.0 - x) * d[2], d[1] + (1.0 - y) * d[3]]; + assert!((c[0] - want[0]).abs() < 1e-5 && (c[1] - want[1]).abs() < 1e-5, "{c:?} vs {want:?}"); + } + /// Le curseur est sur le métrage : le slot le rogne aussi. #[test] fn the_cursor_is_clipped_to_the_slot() { @@ -8484,6 +8671,7 @@ mod tests { /// Écran droit plein cadre, sans zoom : le curseur y tombe toujours dans le rect source. fn full_frame_geometry() -> FrameGeometry { FrameGeometry { + show_screen: true, scene_preset: None, webcam_shadow: 1.0, mb_taps: 1.0, @@ -8514,6 +8702,8 @@ mod tests { shape_fade: 0.0, window_frame: None, screen_mask: None, + screen_turn: false, + screen_uv_max: [1.0, 1.0], } } diff --git a/crates/compositor/src/gif_export.rs b/crates/compositor/src/gif_export.rs index 54293a1766..cd261960bd 100644 --- a/crates/compositor/src/gif_export.rs +++ b/crates/compositor/src/gif_export.rs @@ -40,6 +40,19 @@ //! one SOURCE frame per OUTPUT frame — which made a 30 s/60 fps recording //! export as 6 s of content stretched over 30 s, and could not decode files //! the MP4 path handled. `tests/export_timing.rs` pins the behaviour. +//! - Frame differencing. After the first frame, a frame is only the rectangle +//! around the pixels that moved past `UNCHANGED_TOLERANCE` since they were +//! last drawn (`take_change`), with the others in it set to +//! `TRANSPARENT_INDEX`, so what the screen already shows stays. With the +//! dither on, a change is redrawn by whole `DITHER_TILE` tiles, so error +//! diffusion only runs across pixels drawn together. A frame where +//! nothing moved is not written: its delay goes to the image before it. The +//! palette is rebuilt on its schedule and as soon as most of the picture moves +//! (a cut, a zoom); one that changed redraws the whole frame, so nothing an +//! older palette drew stays on screen. +//! Without this, every frame was written whole, and the dither turned any +//! change, a ticking clock or the pointer, into noise across the rest of the +//! frame: 57 MB for a 17 s demo at 720p, 5.3 MB with it. //! - `GifWriter` — GIF89a format writer: header, optional Netscape 2.0 //! loop extension, per-frame Graphics Control Extension + Image //! Descriptor + LZW image data, trailer. Pure std `Write`. @@ -110,6 +123,119 @@ const PALETTE_REQUANTIZE_EVERY: u64 = 30; /// recordings and the cost difference is tiny. const PALETTE_COLORS: usize = 256; +/// The palette entry kept for "unchanged since the last frame". It repeats +/// entry 0, and a tie in the nearest-colour search goes to the lower index, so +/// no pixel is ever mapped to it: it only appears where a frame lets the +/// previous one show through. +const TRANSPARENT_INDEX: u8 = (PALETTE_COLORS - 1) as u8; + +/// The part of the frame a GIF image covers, in pixels. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct Rect { + left: u32, + top: u32, + width: u32, + height: u32, +} + +impl Rect { + fn full(width: u32, height: u32) -> Self { + Rect { left: 0, top: 0, width, height } + } +} + +/// How far a channel may move and still count as unchanged. A still picture +/// comes out of decoding, scaling and the compositor's anti-aliased edges with +/// up to ±3 of noise from one frame to the next (measured on an H.264 export and +/// on the export test fixture). Compared exactly, every frame of a still stretch +/// "changed" and was redrawn. +const UNCHANGED_TOLERANCE: u8 = 4; + +/// Whether an RGBA pixel moved past `UNCHANGED_TOLERANCE` on any colour channel. +fn moved(drawn: &[u8], now: &[u8]) -> bool { + (0..3).any(|c| drawn[c].abs_diff(now[c]) > UNCHANGED_TOLERANCE) +} + +/// With the dither on, a change is redrawn by whole tiles this many pixels a +/// side. Error diffusion hands each pixel's error to its neighbours: a pixel +/// redrawn among ones left as they were would round on its own, against +/// neighbours the screen does not show, and a dithered area would drift. A +/// whole tile is dithered together; a tile left as it was neither takes nor +/// passes on any error. +const DITHER_TILE: usize = 8; + +/// What a frame redraws: the rectangle around the pixels that moved, for each +/// pixel in it whether to draw it (row by row) or let the GIF screen show what +/// is already there, and how many pixels moved. +struct Change { + rect: Rect, + draw: Vec, + moved: usize, +} + +/// The change from `canvas` to `frame`, both RGBA `width`×`height`, or `None` +/// when nothing moved. It redraws by `tile`×`tile` blocks: whole blocks around +/// what moved, or, with 1, the moved pixels alone. `canvas` holds each pixel as +/// it was last drawn, not the previous frame, so a slow drift is redrawn once it +/// has moved past the tolerance and the picture never strays further than that. +/// The pixels drawn take their new value in it. +fn take_change( + canvas: &mut [u8], + frame: &[u8], + width: u32, + height: u32, + tile: usize, +) -> Option { + let (w, h) = (width as usize, height as usize); + debug_assert!(canvas.len() == w * h * 4 && frame.len() == canvas.len()); + let px = |i: usize| i * 4..i * 4 + 4; + let tiles_across = w.div_ceil(tile); + let tile_of = |x: usize, y: usize| (y / tile) * tiles_across + x / tile; + let mut dirty = vec![false; tiles_across * h.div_ceil(tile)]; + let (mut left, mut top, mut right, mut bottom) = (w, h, 0, 0); + let mut count = 0; + for y in 0..h { + for x in 0..w { + if moved(&canvas[px(y * w + x)], &frame[px(y * w + x)]) { + dirty[tile_of(x, y)] = true; + count += 1; + (left, right) = (left.min(x / tile * tile), right.max((x / tile + 1) * tile)); + (top, bottom) = (top.min(y / tile * tile), bottom.max((y / tile + 1) * tile)); + } + } + } + if count == 0 { + return None; + } + let (right, bottom) = (right.min(w), bottom.min(h)); + let mut draw = Vec::with_capacity((right - left) * (bottom - top)); + for y in top..bottom { + for x in left..right { + let redraw = dirty[tile_of(x, y)]; + if redraw { + canvas[px(y * w + x)].copy_from_slice(&frame[px(y * w + x)]); + } + draw.push(redraw); + } + } + let rect = Rect { + left: left as u32, + top: top as u32, + width: (right - left) as u32, + height: (bottom - top) as u32, + }; + Some(Change { rect, draw, moved: count }) +} + +/// The palette for `rgba`: 255 colours by median cut, then `TRANSPARENT_INDEX`. +fn build_palette(rgba: &[u8]) -> Vec { + let colours = (PALETTE_COLORS - 1) * 3; + let mut palette = vec![0u8; PALETTE_COLORS * 3]; + build_palette_median_cut(rgba, PALETTE_COLORS - 1, &mut palette[..colours]); + palette.copy_within(0..3, colours); + palette +} + /// Wall-time / size / fps summary for a GIF export, shaped like /// `pipeline::Stats` so the bench and the napi binding can pass it /// through without a second struct. @@ -241,9 +367,12 @@ fn export_gif_inner( // delays actually written is kept for `video_duration_s`. let mut total_delay_cs: u64 = 0; - // Cached palette: rebuilt on a schedule (`PALETTE_REQUANTIZE_EVERY`), - // shared with the workers mapping the frames that use it. + // Cached palette, shared with the workers mapping the frames that use it, + // and the frame it was last built on. let mut palette_rgb: Arc> = Arc::new(vec![0u8; PALETTE_COLORS * 3]); + let mut palette_at: u64 = 0; + // Each pixel as it was last drawn, which the next frame is compared with. + let mut canvas: Option> = None; let t0 = Instant::now(); let scene = comp.scene_snapshot(); @@ -290,12 +419,17 @@ fn export_gif_inner( let mut pending: BTreeMap = BTreeMap::new(); let in_flight_max = 2 * workers as u64; let mut written: u64 = 0; - // Writes what is next in line, returns how many frames are written. + let mut hold = DelayHold::default(); + // Takes what is next in line, returns how many frames are taken. An image + // is written once the frames after it that changed nothing have added + // their delays to its own. let mut write_ready = |pending: &mut BTreeMap| -> Result { while let Some(f) = pending.remove(&written) { let delay_cs = frame_delay_cs(written, fps); total_delay_cs += delay_cs as u64; - gw.write_frame(&f.lzw, &f.palette_rgb, delay_cs)?; + if let Some((image, delay)) = hold.push(f.image, delay_cs) { + gw.write_frame(&image, delay)?; + } written += 1; progress(written); } @@ -323,13 +457,34 @@ fn export_gif_inner( debug_assert_eq!(rw, width); debug_assert_eq!(rh, height); - // Refresh the palette on a schedule. Building the histogram - // and running median-cut is O(unique colors) — fast enough at - // 480p on our 30-frame cadence. - if frame_index % PALETTE_REQUANTIZE_EVERY == 0 { - let mut p = vec![0u8; PALETTE_COLORS * 3]; - build_palette_median_cut(&rgba, PALETTE_COLORS, &mut p); - palette_rgb = Arc::new(p); + // What this frame redraws: what moved since it was last drawn. + // Dithered, by whole tiles (`DITHER_TILE`); plain, by the pixel. + let tile = if dither { DITHER_TILE } else { 1 }; + let change = canvas + .as_mut() + .and_then(|canvas| take_change(canvas, &rgba, width, height, tile)); + let moved = change.as_ref().map_or(0, |c| c.moved); + let mut redraw = change.map(|Change { rect, draw, .. }| (rect, Some(draw))); + // The palette is rebuilt on a schedule, and at once when most of + // the picture moved (a cut, a zoom): a new picture mapped to the + // old one's palette gets the wrong colours. A palette that changed + // redraws the whole frame, opaque, or what an older one drew would + // stay on screen wherever the picture then stands still. Building + // it is O(unique colors), cheap next to the mapping. + if canvas.is_none() + || frame_index - palette_at >= PALETTE_REQUANTIZE_EVERY + || moved * 2 >= (width * height) as usize + { + palette_at = frame_index; + let palette = build_palette(&rgba); + if canvas.is_none() || palette != *palette_rgb { + palette_rgb = Arc::new(palette); + match canvas.as_mut() { + Some(canvas) => canvas.copy_from_slice(&rgba), + None => canvas = Some(rgba.clone()), + } + redraw = Some((Rect::full(width, height), None)); + } } // Bound what is submitted but not yet written: behind one slow // frame the rest would pile up in `pending`, each holding its @@ -341,9 +496,24 @@ fn export_gif_inner( .map_err(|_| anyhow!("export_gif: encoder worker died"))?; pending.insert(f.index, f); } - let job = - FrameJob { index: frame_index, rgba, palette_rgb: palette_rgb.clone() }; - job_tx.send(job).map_err(|_| anyhow!("export_gif: encoder worker died"))?; + match redraw { + // Nothing moved: no image, only the frame's delay. + None => { + pending.insert(frame_index, EncodedFrame { index: frame_index, image: None }); + } + Some((rect, draw)) => { + let job = FrameJob { + index: frame_index, + rgba, + rect, + draw, + palette_rgb: palette_rgb.clone(), + }; + job_tx + .send(job) + .map_err(|_| anyhow!("export_gif: encoder worker died"))?; + } + } for f in done_rx.try_iter() { pending.insert(f.index, f); } @@ -359,6 +529,9 @@ fn export_gif_inner( let f = done_rx.recv().map_err(|_| anyhow!("export_gif: encoder worker died"))?; pending.insert(f.index, f); } + if let Some((image, delay)) = hold.finish() { + gw.write_frame(&image, delay)?; + } Ok(frames) })?; @@ -380,25 +553,83 @@ fn export_gif_inner( Ok(GifStats { frames, wall_s, fps: fps_actual, video_duration_s, file_bytes }) } -/// One frame for an encoder worker: its pixels and the palette to map them to. +/// One frame for an encoder worker: its pixels, the rectangle it redraws, and +/// the palette to map it to. struct FrameJob { index: u64, rgba: Vec, + rect: Rect, + /// Which pixels of `rect` to draw, row by row; the rest are left + /// transparent. `None` for the first frame, drawn whole and opaque. + draw: Option>, palette_rgb: Arc>, } -/// A frame ready to write: its indices, LZW-encoded, and its palette. +/// A frame in output order: the image it draws, or `None` when it changed nothing. struct EncodedFrame { index: u64, + image: Option, +} + +/// One GIF image: its indices, LZW-encoded, its palette, and where it sits. +struct EncodedImage { lzw: Vec, palette_rgb: Arc>, + rect: Rect, + /// Whether `TRANSPARENT_INDEX` marks pixels the previous frame shows through. + transparent: bool, +} + +impl EncodedImage { + /// One transparent pixel in the corner: it changes nothing on screen, and + /// only carries a delay. + fn unchanged(palette_rgb: Arc>) -> Self { + let mut lzw = Vec::new(); + lzw_compress(&[TRANSPARENT_INDEX], 8, &mut lzw); + EncodedImage { lzw, palette_rgb, rect: Rect::full(1, 1), transparent: true } + } +} + +/// Holds the last image until the frames after it that changed nothing have +/// added their delays to its own: a still stretch is one image, not many. +#[derive(Default)] +struct DelayHold(Option<(EncodedImage, u16)>); + +impl DelayHold { + /// Takes the next frame in output order, with its delay; returns the image + /// before it once that one's delay is complete. + fn push(&mut self, image: Option, delay_cs: u16) -> Option<(EncodedImage, u16)> { + let Some(image) = image else { + let (held, delay) = self.0.as_mut()?; + match delay.checked_add(delay_cs) { + Some(sum) => { + *delay = sum; + return None; + } + // A GIF delay tops out at 655.35 s: past it, the held image goes + // out with what it has, and a pixel that changes nothing carries + // the rest of the stretch. + None => { + let carrier = EncodedImage::unchanged(held.palette_rgb.clone()); + return self.0.replace((carrier, delay_cs)); + } + } + }; + self.0.replace((image, delay_cs)) + } + + /// The last image, with every delay it gathered. + fn finish(self) -> Option<(EncodedImage, u16)> { + self.0 + } } /// A worker's scratch buffers, reused across the frames it encodes. struct FrameEncoder { width: u32, - height: u32, dither: bool, + /// The changed rectangle's pixels, contiguous. + rect_rgba: Vec, indices: Vec, // Two rows of Floyd-Steinberg error, 3 channels per pixel. err_cur: Vec, @@ -410,8 +641,8 @@ impl FrameEncoder { let w = width as usize; FrameEncoder { width, - height, dither, + rect_rgba: Vec::with_capacity(w * height as usize * 4), indices: vec![0u8; w * height as usize], err_cur: vec![0.0f32; w * 3], err_next: vec![0.0f32; w * 3], @@ -419,25 +650,49 @@ impl FrameEncoder { } fn encode(&mut self, job: FrameJob) -> EncodedFrame { + let Rect { left, top, width, height } = job.rect; + let (stride, x0, y0) = (self.width as usize, left as usize, top as usize); + let (w, h) = (width as usize, height as usize); + self.rect_rgba.clear(); + for y in y0..y0 + h { + let start = (y * stride + x0) * 4; + self.rect_rgba.extend_from_slice(&job.rgba[start..start + w * 4]); + } + let indices = &mut self.indices[..w * h]; // The dither pass diffuses the error against the CHOSEN palette // entry, so it runs fused with the mapping — see - // `map_to_indices_dithered`. + // `map_to_indices_dithered`. What is not drawn takes no part in it. if self.dither { map_to_indices_dithered( &job.palette_rgb, - &job.rgba, - self.width, - self.height, - &mut self.err_cur, - &mut self.err_next, - &mut self.indices, + &self.rect_rgba, + width, + height, + &mut self.err_cur[..w * 3], + &mut self.err_next[..w * 3], + indices, + job.draw.as_deref(), ); } else { - map_to_indices(&job.palette_rgb, &job.rgba, &mut self.indices); + map_to_indices(&job.palette_rgb, &self.rect_rgba, indices); + } + // In the rectangle, what did not move shows what the screen already has. + if let Some(draw) = &job.draw { + for (index, &drawn) in indices.iter_mut().zip(draw) { + if !drawn { + *index = TRANSPARENT_INDEX; + } + } } let mut lzw = Vec::new(); - lzw_compress(&self.indices, 8, &mut lzw); - EncodedFrame { index: job.index, lzw, palette_rgb: job.palette_rgb } + lzw_compress(indices, 8, &mut lzw); + let image = EncodedImage { + lzw, + palette_rgb: job.palette_rgb, + rect: job.rect, + transparent: job.draw.is_some(), + }; + EncodedFrame { index: job.index, image: Some(image) } } } @@ -508,47 +763,45 @@ impl GifWriter { Ok(()) } - /// Write one animated frame: Graphics Control Extension (delay - /// only — no transparency, no disposal), Image Descriptor, local - /// color table, and `lzw`, the frame's indices as `lzw_compress` - /// encodes them with a minimum code size of 8. - fn write_frame(&mut self, lzw: &[u8], palette_rgb: &[u8], delay_cs: u16) -> Result<()> { - debug_assert_eq!(palette_rgb.len(), PALETTE_COLORS * 3); - - // Graphics Control Extension: delay only. The disposal - // method is "leave in place" (0) and the transparent flag - // is off, so a frame with overlapping dimensions is - // composited on top of the previous frame. + /// Write one image: Graphics Control Extension (delay, disposal, + /// transparency), Image Descriptor placing it on the logical screen, + /// local color table, and its indices as `lzw_compress` encodes them + /// with a minimum code size of 8. + fn write_frame(&mut self, image: &EncodedImage, delay_cs: u16) -> Result<()> { + debug_assert_eq!(image.palette_rgb.len(), PALETTE_COLORS * 3); + let Rect { left, top, width, height } = image.rect; + debug_assert!(left + width <= self.width as u32 && top + height <= self.height as u32); + + // Graphics Control Extension. Disposal is "do not dispose", so the + // next image draws over this one: what a later frame leaves + // transparent, or outside its rectangle, still shows this frame. self.w.write_all(&[0x21, 0xF9, 0x04])?; - // Packed: 3 reserved (0) | 3 disposal (0 = leave in - // place) | 1 user input (0) | 1 transparent (0). 0x00 - // throughout. - self.w.write_all(&[0x00])?; + // Packed: 3 reserved (0) | 3 disposal (1 = do not dispose) | + // 1 user input (0) | 1 transparent flag. + self.w.write_all(&[0b000_001_0_0 | image.transparent as u8])?; self.w.write_all(&delay_cs.to_le_bytes())?; - // Transparent color index (unused — `0` is the conventional - // "no transparency" sentinel). - self.w.write_all(&[0x00])?; + // Transparent color index, read only when the flag is set. + self.w.write_all(&[if image.transparent { TRANSPARENT_INDEX } else { 0 }])?; // Block terminator. self.w.write_all(&[0x00])?; // Image Descriptor. self.w.write_all(&[0x2C])?; - self.w.write_all(&0u16.to_le_bytes())?; // left - self.w.write_all(&0u16.to_le_bytes())?; // top - self.w.write_all(&self.width.to_le_bytes())?; - self.w.write_all(&self.height.to_le_bytes())?; + for value in [left, top, width, height] { + self.w.write_all(&(value as u16).to_le_bytes())?; + } // Packed: LCT flag (bit 7) = 1, interlace (bit 6) = 0, // sort (bit 5) = 0, reserved (bits 3-4) = 0, LCT size // (bits 0-2) = 7 (i.e. 2^(7+1) = 256 entries). self.w.write_all(&[0b1_0_0_00_111])?; // Local color table. - self.w.write_all(palette_rgb)?; + self.w.write_all(&image.palette_rgb)?; // LZW image data: `LZW minimum code size` byte, then // sub-blocks of compressed bytes, then a 0x00 terminator. // LZW min code size is 8 for a 256-color palette. self.w.write_all(&[8])?; - write_sub_blocks(&mut self.w, lzw)?; + write_sub_blocks(&mut self.w, &image.lzw)?; self.w.write_all(&[0x00])?; // image data terminator Ok(()) } @@ -1046,6 +1299,10 @@ impl SortedPalette { /// /// Kernel is the standard 7/16 right, 3/16 below-left, 5/16 below, /// 1/16 below-right. Two row buffers keep the working set at O(width). +/// +/// `drawn`, when given, marks the pixels to map: the others are left as the +/// screen already shows them, so they neither take the error spread into them +/// nor pass any on, and their index is left for the caller to set. #[allow(clippy::too_many_arguments)] fn map_to_indices_dithered( palette_rgb: &[u8], @@ -1055,6 +1312,7 @@ fn map_to_indices_dithered( err_cur: &mut [f32], err_next: &mut [f32], indices: &mut [u8], + drawn: Option<&[bool]>, ) { let w = width as usize; let h = height as usize; @@ -1068,6 +1326,9 @@ fn map_to_indices_dithered( for y in 0..h { for x in 0..w { + if drawn.is_some_and(|drawn| !drawn[y * w + x]) { + continue; + } let base = (y * w + x) * 4; let e = x * 3; // Channel value carrying the error diffused into this pixel. @@ -1195,10 +1456,9 @@ mod tests { let mut gw = GifWriter::new(&mut buf, 2, 2).unwrap(); gw.write_header().unwrap(); gw.write_netscape_loop(0).unwrap(); - let palette = vec![0u8; PALETTE_COLORS * 3]; let mut lzw = Vec::new(); lzw_compress(&[0u8; 4], 8, &mut lzw); - gw.write_frame(&lzw, &palette, 10).unwrap(); + gw.write_frame(&opaque_image(lzw, 2, 2), 10).unwrap(); gw.finish().unwrap(); } // Magic. @@ -1411,7 +1671,6 @@ mod tests { // the sub-block chunking is exercised. let (w, h) = (64usize, 64usize); let indices: Vec = (0..w * h).map(|i| ((i / 7 + i % 13) % 251) as u8).collect(); - let palette = vec![0u8; PALETTE_COLORS * 3]; let mut buf = Vec::new(); { @@ -1420,7 +1679,7 @@ mod tests { gw.write_netscape_loop(0).unwrap(); let mut lzw = Vec::new(); lzw_compress(&indices, 8, &mut lzw); - gw.write_frame(&lzw, &palette, 8).unwrap(); + gw.write_frame(&opaque_image(lzw, w as u32, h as u32), 8).unwrap(); gw.finish().unwrap(); } @@ -1540,6 +1799,7 @@ mod tests { &mut err_cur, &mut err_next, &mut indices, + None, ); assert!((indices[0] as usize) < PALETTE_COLORS); } @@ -1571,6 +1831,7 @@ mod tests { &mut err_cur, &mut err_next, &mut dithered, + None, ); // RMS distance between the ramp and a 16-px moving average of what a row maps to: the @@ -1646,6 +1907,7 @@ mod tests { &mut err_cur, &mut err_next, &mut dithered, + None, ); let (mut cur, mut nxt) = (vec![0.0f32; w * 3], vec![0.0f32; w * 3]); for y in 0..h { @@ -1705,4 +1967,251 @@ mod tests { // last byte is the last data byte of the third block. assert_eq!(*buf.last().unwrap(), 0xAA); } + + /// A whole opaque `width`×`height` image over a black palette. + fn opaque_image(lzw: Vec, width: u32, height: u32) -> EncodedImage { + EncodedImage { + lzw, + palette_rgb: Arc::new(vec![0u8; PALETTE_COLORS * 3]), + rect: Rect::full(width, height), + transparent: false, + } + } + + /// An RGBA frame where pixel `i` is `(i * 9, 0, 0)`. + fn ramp(width: u32, height: u32) -> Vec { + (0..width * height).flat_map(|i| [(i * 9) as u8, 0, 0, 255]).collect() + } + + #[test] + fn a_change_is_the_box_around_what_moved_past_the_tolerance() { + let (w, h) = (6u32, 4u32); + let px = |x: u32, y: u32| ((y * w + x) * 4) as usize; + let first = ramp(w, h); + let mut canvas = first.clone(); + assert!(take_change(&mut canvas, &first, w, h, 1).is_none()); + + // Codec noise, within the tolerance: nothing to redraw. + let mut frame = first.clone(); + frame[px(2, 2) + 1] = UNCHANGED_TOLERANCE; + assert!(take_change(&mut canvas, &frame, w, h, 1).is_none()); + + // Two pixels moved for real: the box takes both in, and only they are drawn. + frame[px(4, 1) + 2] = 200; + frame[px(1, 3)] = 200; + let change = take_change(&mut canvas, &frame, w, h, 1).expect("a change"); + assert_eq!(change.rect, Rect { left: 1, top: 1, width: 4, height: 3 }); + assert_eq!(change.draw.iter().filter(|&&d| d).count(), 2); + assert!(change.draw[3] && change.draw[8]); // (4, 1) and (1, 3) in the box + assert_eq!(canvas[px(4, 1)..px(4, 1) + 4], frame[px(4, 1)..px(4, 1) + 4]); + assert!(take_change(&mut canvas, &frame, w, h, 1).is_none(), "drawn, so settled"); + } + + #[test] + fn a_slow_drift_is_redrawn_once_it_passes_the_tolerance() { + let (w, h) = (2u32, 1u32); + let mut canvas = vec![100u8, 100, 100, 255, 0, 0, 0, 255]; + let step = |v: u8| vec![v, 100, 100, 255, 0, 0, 0, 255]; + // Each step is within the tolerance of the last frame, not of what was drawn. + assert!(take_change(&mut canvas, &step(103), w, h, 1).is_none()); + let change = take_change(&mut canvas, &step(106), w, h, 1).expect("drifted too far"); + assert_eq!(change.rect, Rect { left: 0, top: 0, width: 1, height: 1 }); + assert_eq!(canvas[0], 106); + } + + #[test] + fn no_pixel_is_mapped_to_the_transparent_entry() { + let (w, h) = (16u32, 16u32); + let mut rgba = ramp(w, h); + let palette = build_palette(&rgba); + assert_eq!(palette[TRANSPARENT_INDEX as usize * 3..], palette[..3]); + // A pixel exactly the colour of entry 0, which the transparent entry repeats. + rgba[..3].copy_from_slice(&palette[..3]); + let mut indices = vec![0u8; (w * h) as usize]; + map_to_indices(&palette, &rgba, &mut indices); + assert!(!indices.contains(&TRANSPARENT_INDEX)); + let (mut cur, mut next) = (vec![0f32; w as usize * 3], vec![0f32; w as usize * 3]); + map_to_indices_dithered(&palette, &rgba, w, h, &mut cur, &mut next, &mut indices, None); + assert!(!indices.contains(&TRANSPARENT_INDEX)); + } + + #[test] + fn a_later_frame_draws_only_what_changed() { + let (w, h) = (5u32, 3u32); + let prev = ramp(w, h); + let mut cur = prev.clone(); + // Two pixels of the middle row change; the one between them does not. + let px = |x: u32, y: u32| ((y * w + x) * 4) as usize; + cur[px(1, 1)] = 250; + cur[px(3, 1)] = 250; + let Change { rect, draw, .. } = take_change(&mut prev.clone(), &cur, w, h, 1).expect("a change"); + assert_eq!(rect, Rect { left: 1, top: 1, width: 3, height: 1 }); + let palette = Arc::new(build_palette(&cur)); + for dither in [false, true] { + let job = FrameJob { + index: 1, + rgba: cur.clone(), + rect, + draw: Some(draw.clone()), + palette_rgb: palette.clone(), + }; + let image = FrameEncoder::new(w, h, dither).encode(job).image.expect("an image"); + assert_eq!(image.rect, rect); + assert!(image.transparent); + let indices = lzw_decompress(&image.lzw, 8); + assert_eq!(indices.len(), 3, "dither {dither}"); + assert_ne!(indices[0], TRANSPARENT_INDEX, "dither {dither}"); + assert_eq!(indices[1], TRANSPARENT_INDEX, "dither {dither}"); + assert_ne!(indices[2], TRANSPARENT_INDEX, "dither {dither}"); + } + } + + #[test] + fn the_first_frame_is_whole_and_opaque() { + let (w, h) = (4u32, 2u32); + let rgba = ramp(w, h); + let job = FrameJob { + index: 0, + rgba: rgba.clone(), + rect: Rect::full(w, h), + draw: None, + palette_rgb: Arc::new(build_palette(&rgba)), + }; + let image = FrameEncoder::new(w, h, true).encode(job).image.expect("an image"); + assert!(!image.transparent); + let indices = lzw_decompress(&image.lzw, 8); + assert_eq!(indices.len(), 8); + assert!(!indices.contains(&TRANSPARENT_INDEX)); + } + + #[test] + fn a_still_stretch_lengthens_the_image_before_it() { + let image = |n: u8| Some(opaque_image(vec![n], 1, 1)); + let mut hold = DelayHold::default(); + assert!(hold.push(image(1), 7).is_none()); + assert!(hold.push(None, 7).is_none()); + assert!(hold.push(None, 6).is_none()); + let (first, delay) = hold.push(image(2), 7).expect("the first image, complete"); + assert_eq!((first.lzw, delay), (vec![1], 20)); + let (last, delay) = hold.finish().expect("the last image"); + assert_eq!((last.lzw, delay), (vec![2], 7)); + } + + #[test] + fn a_dithered_change_redraws_whole_tiles() { + let (w, h) = (40u32, 20u32); + let first = ramp(w, h); + let mut canvas = first.clone(); + let mut frame = first.clone(); + let px = |x: u32, y: u32| ((y * w + x) * 4) as usize; + frame[px(20, 5)] = frame[px(20, 5)].wrapping_add(100); + frame[px(22, 5) + 1] = UNCHANGED_TOLERANCE - 1; // too small to move on its own + let change = take_change(&mut canvas, &frame, w, h, DITHER_TILE).expect("a change"); + // The pixel's tile, drawn whole. + let tile = DITHER_TILE as u32; + let tile_of_pixel = Rect { left: 20 / tile * tile, top: 5 / tile * tile, width: tile, height: tile }; + assert_eq!(change.rect, tile_of_pixel); + assert!(change.draw.iter().all(|&d| d)); + assert_eq!(change.moved, 1); + assert_eq!(canvas[px(22, 5) + 1], UNCHANGED_TOLERANCE - 1, "the whole tile takes its new value"); + // A tile clipped by the frame's edge. + frame[px(39, 19)] = frame[px(39, 19)].wrapping_add(100); + let change = take_change(&mut canvas, &frame, w, h, DITHER_TILE).expect("a change"); + let (left, top) = (39 / tile * tile, 19 / tile * tile); + assert_eq!(change.rect, Rect { left, top, width: w - left, height: h - top }); + } + + #[test] + fn pixels_not_drawn_pass_no_error_to_those_drawn() { + // A grey row against a black and white palette: what is drawn must be + // dithered as if what is not drawn were not there. + let mut palette = vec![0u8; PALETTE_COLORS * 3]; + palette[3..6].copy_from_slice(&[255, 255, 255]); + let grey: Vec = (0..8).flat_map(|_| [128u8, 128, 128, 255]).collect(); + let drawn: Vec = (0..8).map(|i| i >= 4).collect(); + let (mut cur, mut next) = (vec![0f32; 8 * 3], vec![0f32; 8 * 3]); + let mut indices = vec![7u8; 8]; + map_to_indices_dithered(&palette, &grey, 8, 1, &mut cur, &mut next, &mut indices, Some(&drawn)); + let mut alone = vec![0u8; 4]; + map_to_indices_dithered(&palette, &grey[16..], 4, 1, &mut cur[..12], &mut next[..12], &mut alone, None); + assert_eq!(indices[4..], alone[..]); + assert_eq!(indices[..4], [7, 7, 7, 7], "the caller sets what is not drawn"); + } + + #[test] + fn a_dithered_patch_keeps_its_balance_when_part_of_it_moves() { + // Grey 128 dithers to white, black, white, black. Pixels 1 and 3 then move + // to 133. Redrawn alone, they came out white beside the white the screen + // still showed at 0 and 2: four whites for a grey. Their tile is redrawn + // together instead. + let mut palette = vec![0u8; PALETTE_COLORS * 3]; + palette[3..6].copy_from_slice(&[255, 255, 255]); + let palette = Arc::new(palette); + let row = |values: [u8; 4]| -> Vec { values.iter().flat_map(|&v| [v, v, v, 255]).collect() }; + let first = row([128, 128, 128, 128]); + let mut canvas = first.clone(); + let job = |rgba: Vec, rect: Rect, draw: Option>| FrameJob { + index: 0, + rgba, + rect, + draw, + palette_rgb: palette.clone(), + }; + let shown = FrameEncoder::new(4, 1, true) + .encode(job(first, Rect::full(4, 1), None)) + .image + .expect("an image"); + assert_eq!(lzw_decompress(&shown.lzw, 8), vec![1, 0, 1, 0]); + + let moved = row([128, 133, 128, 133]); + let Change { rect, draw, .. } = + take_change(&mut canvas, &moved, 4, 1, DITHER_TILE).expect("a change"); + let redrawn = FrameEncoder::new(4, 1, true) + .encode(job(moved, rect, Some(draw))) + .image + .expect("an image"); + assert_eq!(redrawn.rect, Rect::full(4, 1)); + let whites = lzw_decompress(&redrawn.lzw, 8).iter().filter(|&&i| i == 1).count(); + assert_eq!(whites, 2); + } + + #[test] + fn a_still_stretch_past_the_longest_gif_delay_loses_none_of_it() { + let mut hold = DelayHold::default(); + assert!(hold.push(Some(opaque_image(vec![1], 1, 1)), 60_000).is_none()); + assert!(hold.push(None, 5_000).is_none()); + let (first, delay) = hold.push(None, 1_000).expect("the image, with what it can hold"); + assert_eq!((first.lzw, delay), (vec![1], 65_000)); + let (carrier, rest) = hold.finish().expect("the rest of the stretch"); + assert!(carrier.transparent); + assert_eq!(carrier.rect, Rect::full(1, 1)); + assert_eq!(lzw_decompress(&carrier.lzw, 8), vec![TRANSPARENT_INDEX]); + assert_eq!(delay as u32 + rest as u32, 66_000); + } + + #[test] + fn a_partial_frame_is_placed_on_the_screen_and_marks_its_transparency() { + let mut buf = Vec::new(); + { + let mut gw = GifWriter::new(&mut buf, 8, 8).unwrap(); + gw.write_header().unwrap(); + let mut lzw = Vec::new(); + lzw_compress(&[0, TRANSPARENT_INDEX, 0], 8, &mut lzw); + let image = EncodedImage { + rect: Rect { left: 1, top: 2, width: 3, height: 1 }, + transparent: true, + ..opaque_image(lzw, 3, 1) + }; + gw.write_frame(&image, 9).unwrap(); + gw.finish().unwrap(); + } + let gce = buf.windows(3).position(|b| *b == [0x21, 0xF9, 0x04]).expect("GCE") + 3; + assert_eq!(buf[gce], 0b000_001_0_1, "do not dispose, transparent"); + assert_eq!(&buf[gce + 1..gce + 3], &9u16.to_le_bytes()); + assert_eq!(buf[gce + 3], TRANSPARENT_INDEX); + let desc = gce + 5; + assert_eq!(buf[desc], 0x2C); + let field = |i: usize| u16::from_le_bytes([buf[desc + 1 + 2 * i], buf[desc + 2 + 2 * i]]); + assert_eq!([field(0), field(1), field(2), field(3)], [1, 2, 3, 1]); + } } diff --git a/crates/compositor/src/linux_decode.rs b/crates/compositor/src/linux_decode.rs index 2e2433d84b..efbf96bfa6 100644 --- a/crates/compositor/src/linux_decode.rs +++ b/crates/compositor/src/linux_decode.rs @@ -34,6 +34,7 @@ use crate::ffi::{ /// crate pour tous les accesseurs du shim. extern "C" { fn sn_fmt_stream(s: *mut AVFormatContext, i: i32) -> *mut AVStream; + fn sn_fmt_duration(s: *mut AVFormatContext) -> i64; } /// `SEEK_SET` constant — la position de seek `av_seek_frame` interprète @@ -646,13 +647,14 @@ impl SwDecoder { return None; } let duration = (*stream).duration; - if duration > 0 && self.stream_timebase > 0.0 { - let s = duration as f64 * self.stream_timebase; - if s.is_finite() && s > 0.0 { - return Some(s); - } - } - None + let stream_sec = if duration > 0 && self.stream_timebase > 0.0 { + Some(duration as f64 * self.stream_timebase).filter(|s| s.is_finite() && *s > 0.0) + } else { + None + }; + // Le plus long du flux et du conteneur : un flux d'une seule frame (écran + // statique) ne doit pas raccourcir le clip (cf. `cover_container_duration`). + crate::timeline_walk::cover_container_duration(stream_sec, sn_fmt_duration(self.fmt)) } } diff --git a/crates/compositor/src/live.rs b/crates/compositor/src/live.rs index 2165d1d357..52a95f905f 100644 --- a/crates/compositor/src/live.rs +++ b/crates/compositor/src/live.rs @@ -880,6 +880,61 @@ fn resolve_scene_clip_index( } } +/// The cache key of a clip's cursor track. A turned clip uses the same telemetry file +/// mirrored (`CursorTrack::turned`), so the two must never share a cache entry. +pub(crate) fn cursor_key(screen_path: &str, turned: bool) -> String { + let path = format!("{screen_path}.cursor.json"); + if turned { format!("{path}#turned") } else { path } +} + +/// A clip's cursor track as the compositor reads it: mirrored for a turned clip, whose frame is +/// planned in what the viewer sees. +fn oriented_cursor(track: CursorTrack, turned: bool) -> CursorTrack { + if turned { track.turned() } else { track } +} + +/// Rereads the active clip's cursor telemetry when its key changed (another file, or the same +/// file turned the other way) and hands it to the compositor already smoothed, so a paused +/// preview never shows one unsmoothed frame. Returns whether it reread. +fn reload_cursor_if_changed( + comp: &Compositor, + raw_cursor: &mut Option, + loaded_cursor_path: &mut String, + screen_path: &str, + turned: bool, + smoothing: f32, +) -> bool { + let key = cursor_key(screen_path, turned); + if key == *loaded_cursor_path { + return false; + } + let cursor_path = format!("{screen_path}.cursor.json"); + *raw_cursor = CursorTrack::load(&cursor_path, 0.0, 24.0 * 3600.0) + .ok() + .map(|t| oriented_cursor(t, turned)); + // Journalisé ICI seulement : sinon la ligne annonce « loaded=ok » à + // chaque changement de segment alors que rien n'a été relu, et le log + // laisse croire à un travail qui n'a plus lieu. + match raw_cursor { + Some(track) => eprintln!( + "[live] cursor: path={} turned={} loaded=ok samples={}", + cursor_path, + turned, + track.sample_count(), + ), + None => eprintln!("[live] cursor: path={} loaded=FAIL — clear_cursor()", cursor_path), + } + *loaded_cursor_path = key; + // Sans relecture, rien à reposer : un changement de taille garde le + // curseur (`Compositor::resized`), et le relisser parcourt tout + // l'enregistrement à chaque bascule d'un scrub. + match raw_cursor { + Some(track) => comp.set_cursor(track.smoothed(smoothing)), + None => comp.clear_cursor(), + } + true +} + fn scene_for_clip(scene: &Scene, clip_index: usize) -> Scene { match scene.clips.get(clip_index) { Some(clip) => scene.for_clip_window( @@ -1519,20 +1574,24 @@ unsafe fn advance_to_next_scene_clip( *active_webcam_path = next_clip.webcam_path.clone(); *active_webcam_offset_sec = next_clip.webcam_offset_sec; *active_clip_index = next_index; - comp.set_scene(Some(scene_for_clip(scene, *active_clip_index))); + let clip_scene = scene_for_clip(scene, *active_clip_index); + let turned = clip_scene.screen_turned(); + comp.set_scene(Some(clip_scene)); player.set_programme_clock(Some(scene), *active_clip_index); // Réutilise le curseur préchargé s'il est disponible (voir plus haut) — sinon // (préchargement pas encore prêt / raté) on retombe sur la lecture synchrone - // habituelle, comme avant cette optimisation. + // habituelle, comme avant cette optimisation. The prefetched track is the raw + // file: it is turned here, where the clip it belongs to is known. let cursor_path = format!("{}.cursor.json", active_screen_path); *raw_cursor = match prefetched_cursor { Some(track) => track, None => CursorTrack::load(&cursor_path, 0.0, 24.0 * 3600.0).ok(), - }; + } + .map(|t| oriented_cursor(t, turned)); // Le fichier est noté avec la piste. Sinon une requête de clip qui revient sur le // fichier d'avant le croit encore chargé, ne relit rien, et dessine sur ce clip la // télémétrie de celui qu'on vient de quitter. - *loaded_cursor_path = cursor_path; + *loaded_cursor_path = cursor_key(active_screen_path, turned); match raw_cursor { Some(track) => comp.set_cursor(track.smoothed(0.0)), None => comp.clear_cursor(), @@ -1595,7 +1654,8 @@ unsafe fn render_thread( // fichier à chaque changement du slider "smoothing" (voir la boucle plus bas). let mut raw_cursor = CursorTrack::load(cursor_json, 0.0, 24.0 * 3600.0).ok(); /// Chemin de la télémétrie curseur actuellement chargée dans `raw_cursor` — évite de - /// relire le même fichier à chaque changement de segment (voir plus bas). + /// relire le même fichier à chaque changement de segment (voir plus bas). Compared with + /// `cursor_key`: no scene has arrived yet, so this first track is upright. let mut loaded_cursor_path = cursor_json.to_string(); if let Some(track) = &raw_cursor { comp.set_cursor(track.smoothed(0.0)); @@ -1725,6 +1785,7 @@ unsafe fn render_thread( active_webcam_offset_sec = request.webcam_offset_sec; let scene = shared.scene.lock().unwrap().clone(); full_scene = scene.clone(); + let mut turned = false; if let Some(base_scene) = scene { if let Some(index) = resolve_scene_clip_index( &base_scene, @@ -1740,7 +1801,9 @@ unsafe fn render_thread( active_screen_path, active_webcam_path ); } - comp.set_scene(Some(scene_for_clip(&base_scene, active_clip_index))); + let clip_scene = scene_for_clip(&base_scene, active_clip_index); + turned = clip_scene.screen_turned(); + comp.set_scene(Some(clip_scene)); scene_applied = true; } // Relire la télémétrie curseur seulement si le FICHIER change. Elle était @@ -1748,32 +1811,16 @@ unsafe fn render_thread( // compris quand seul le segment changeait, où le chemin est par // construction identique. Mesuré : 66 bascules sur un scrub de deux clips, // donc 66 relectures du même fichier. - let cursor_path = format!("{}.cursor.json", active_screen_path); - if cursor_path != loaded_cursor_path { - loaded_cursor_path = cursor_path.clone(); - raw_cursor = CursorTrack::load(&cursor_path, 0.0, 24.0 * 3600.0).ok(); - // Journalisé ICI seulement : sinon la ligne annonce « loaded=ok » à - // chaque changement de segment alors que rien n'a été relu, et le log - // laisse croire à un travail qui n'a plus lieu. - match &raw_cursor { - Some(track) => eprintln!( - "[live] cursor: path={} loaded=ok samples={}", - cursor_path, - track.sample_count(), - ), - None => eprintln!( - "[live] cursor: path={} loaded=FAIL — clear_cursor()", - cursor_path, - ), - } - // Sans relecture, rien à reposer : un changement de taille garde le - // curseur (`Compositor::resized`), et le relisser parcourt tout - // l'enregistrement à chaque bascule d'un scrub. - match &raw_cursor { - Some(track) => comp.set_cursor(track.smoothed(0.0)), - None => comp.clear_cursor(), - } - last_smoothing = -1.0; + // The key also carries the clip's turn: the same file turned is another track. + if reload_cursor_if_changed( + &comp, + &mut raw_cursor, + &mut loaded_cursor_path, + &active_screen_path, + turned, + ip.cursor_smoothing, + ) { + last_smoothing = ip.cursor_smoothing; } clip_changed = true; } @@ -1843,7 +1890,20 @@ unsafe fn render_thread( } scene_for_clip(&base_scene, active_clip_index) }); + // Turning the active clip changes no file, only the scene: its track is reread + // here, mirrored the new way. + let turned = scene.as_ref().is_some_and(Scene::screen_turned); comp.set_scene(scene); + if reload_cursor_if_changed( + &comp, + &mut raw_cursor, + &mut loaded_cursor_path, + &active_screen_path, + turned, + ip.cursor_smoothing, + ) { + last_smoothing = ip.cursor_smoothing; + } } // Le temps programme dépend du clip actif ET de la scène entière (durées des clips // précédents) : à recalculer dès que l'un des deux change. @@ -2394,9 +2454,15 @@ pub fn run_standalone(_screen: &str, _webcam: &str, _cursor_json: &str) -> anyho #[cfg(test)] mod tests { - use super::{open_settle_window, should_settle, SETTLE_STEP, SETTLE_WINDOW}; + use super::{cursor_key, open_settle_window, should_settle, SETTLE_STEP, SETTLE_WINDOW}; use std::time::Instant; + #[test] + fn cursor_key_tells_a_turned_clip_from_an_upright_one() { + assert_eq!(cursor_key("a.mp4", false), "a.mp4.cursor.json"); + assert_ne!(cursor_key("a.mp4", false), cursor_key("a.mp4", true)); + } + /// Le bug d'origine : en pause, un seul recompose par changement, donc le masque de /// segmentation (asynchrone, 3 composes de latence) n'arrivait jamais avant un scrub. #[test] diff --git a/crates/compositor/src/mac_frames.rs b/crates/compositor/src/mac_frames.rs index 0d04b3da54..68459b667a 100644 --- a/crates/compositor/src/mac_frames.rs +++ b/crates/compositor/src/mac_frames.rs @@ -105,11 +105,17 @@ extern "C" { } /// Crée un `CVPixelBufferRef` NV12 IOSurface-backed, dimensions paires `(w, h)`. +/// `format` est le fourcc CoreVideo : `420v` partout, sauf les tests qui fabriquent les +/// autres formats biplanaires que VideoToolbox sort (`420f`, `x420`, `xf20`). /// /// NV12 impose des dimensions paires : on arrondit AU-DESSUS pour le buffer et on /// laisse `present.width/height` aux dimensions visibles — c'est le même écart /// texture/visible que produit l'alignement macrobloc de D3D11VA (1080 → 1088). -unsafe fn create_nv12_pixel_buffer(w: usize, h: usize) -> Result { +pub(crate) unsafe fn create_nv12_pixel_buffer( + w: usize, + h: usize, + format: u32, +) -> Result { // `{ IOSurfaceProperties: {}, MetalCompatibility: true }` — un dictionnaire // IOSurface vide suffit à demander le backing, `MetalCompatibility` fait valider // par CoreVideo que le résultat est utilisable depuis Metal. @@ -130,7 +136,7 @@ unsafe fn create_nv12_pixel_buffer(w: usize, h: usize) -> Result 0 { out.extradata = std::slice::from_raw_parts(ptr, len as usize).to_vec(); @@ -2087,6 +2108,26 @@ mod tests { /// image clé pour tout le fichier — mesuré 1 image I sur 300 avant ce correctif. /// Le MP4 reste lisible, mais tout seek redécode depuis le début et un paquet /// abîmé emporte le reste. C'est le test qui aurait attrapé ça. + /// Le MP4 dit sa matrice : BT.601, celle de `yuv.wgsl`. Sans étiquette un + /// lecteur qui suppose BT.709 décale les couleurs saturées. + #[test] + fn l_export_h264_porte_son_etiquette_couleur() { + let Some(enc) = encode(10, 5) else { + eprintln!("libopenh264 indisponible — test sauté"); + return; + }; + use crate::ffi::*; + assert_eq!( + enc.color, + ( + AVColorPrimaries::AVCOL_PRI_BT709 as i32, + AVColorTransferCharacteristic::AVCOL_TRC_BT709 as i32, + AVColorSpace::AVCOL_SPC_SMPTE170M as i32, + AVColorRange::AVCOL_RANGE_MPEG as i32, + ) + ); + } + #[test] fn l_export_h264_emet_des_images_cles_periodiques() { let Some(enc) = encode(10, 45) else { diff --git a/crates/compositor/src/pipeline_macos.rs b/crates/compositor/src/pipeline_macos.rs index 25638157db..d57407354a 100644 --- a/crates/compositor/src/pipeline_macos.rs +++ b/crates/compositor/src/pipeline_macos.rs @@ -630,9 +630,18 @@ impl Decoder { } } - /// Durée réellement annoncée par le flux vidéo (symétrique de - /// `pipeline_windows::Decoder::available_duration_sec`). + /// Durée disponible de la source : le plus long du flux vidéo et du conteneur (cf. + /// `cover_container_duration`). Symétrique de + /// `pipeline_windows::Decoder::available_duration_sec`. pub unsafe fn available_duration_sec(&self) -> Option { + crate::timeline_walk::cover_container_duration( + self.video_stream_duration_sec(), + crate::ffi::sn_fmt_duration(self.fmt), + ) + } + + /// Durée annoncée par le seul flux vidéo, `nb_frames / fps` en repli. + unsafe fn video_stream_duration_sec(&self) -> Option { let stream = crate::ffi::sn_fmt_stream(self.fmt, self.vidx); let duration = (*stream).duration; let tb_sec = self.tb_sec(); @@ -943,6 +952,8 @@ impl VideoEncoder { (*ctx).framerate = crate::ffi::AVRational { num: fps, den: 1 }; (*ctx).bit_rate = bit_rate; (*ctx).flags |= crate::ffi::AV_CODEC_FLAG_GLOBAL_HEADER as i32; + // `rgb2y` / `rgb2uv` de `shaders.metal` : BT.709 limité. + crate::ffi::tag_export_color(ctx, crate::ffi::AVColorSpace::AVCOL_SPC_BT709); // VT : on attache le hw_frames_ctx. En pratique le call-site (run_composited_multi) // nous passera un `hw_frames_ctx` pré-construit lié au même device VideoToolbox @@ -1474,6 +1485,13 @@ unsafe fn drain_encoder( return Ok(()); } averr(r, "receive_packet")?; + // VideoToolbox hands its packets back without a duration. The MP4 muxer then gives the + // last sample none and ends the edit list on its pts, and a player drops that frame + // whenever its pts falls on a whole millisecond (every third frame index): a 30 fps + // export of 277 frames played 276. The time base is 1/fps, so one tick is one frame. + if (*opkt).duration <= 0 { + (*opkt).duration = 1; + } av_packet_rescale_ts(opkt, (*ectx).time_base, (*ostream).time_base); averr( av_interleaved_write_frame(octx, opkt), diff --git a/crates/compositor/src/pipeline_windows.rs b/crates/compositor/src/pipeline_windows.rs index 39ad180fab..6219b3b18e 100644 --- a/crates/compositor/src/pipeline_windows.rs +++ b/crates/compositor/src/pipeline_windows.rs @@ -188,6 +188,7 @@ const AVSEEK_FLAG_BACKWARD: i32 = 1; // seek vers la keyframe <= ts (macro non g extern "C" { fn sn_fmt_stream(s: *mut AVFormatContext, i: i32) -> *mut AVStream; fn sn_fmt_nb_streams(s: *mut AVFormatContext) -> u32; + fn sn_fmt_duration(s: *mut AVFormatContext) -> i64; fn sn_fmt_get_pb(s: *mut AVFormatContext) -> *mut AVIOContext; fn sn_fmt_set_pb(s: *mut AVFormatContext, p: *mut AVIOContext); } @@ -333,6 +334,14 @@ unsafe fn decode_frame_n_inner(path: &str, gpu: &Gpu, n: u32) -> Result bool { codec_id == AVCodecID::AV_CODEC_ID_H264 } +/// Per-stream refinement of `d3d11va_for_codec`: FFmpeg only offers a D3D11 +/// NV12 surface for 8-bit 4:2:0 H.264. High 10 (yuv420p10le) and 4:2:2 / 4:4:4 +/// get none, so those streams take the software `CpuFrames` path. An unknown pixel +/// format (`AV_PIX_FMT_NONE`) keeps the hardware path; if it then has no D3D11 +/// surface, `get_hw_format` falls back to software (#1532). +fn d3d11va_for_stream(codec_id: AVCodecID::Type, format: AVPixelFormat::Type) -> bool { + d3d11va_for_codec(codec_id) + && matches!( + format, + AVPixelFormat::AV_PIX_FMT_NONE + | AVPixelFormat::AV_PIX_FMT_YUV420P + | AVPixelFormat::AV_PIX_FMT_YUVJ420P + ) +} + unsafe fn require_decoder_id( codec_id: AVCodecID::Type, ) -> Result<(*const AVCodec, *mut AVCodecContext)> { @@ -437,6 +461,7 @@ unsafe extern "C" fn get_hw_format( ctx: *mut AVCodecContext, mut fmts: *const AVPixelFormat::Type, ) -> AVPixelFormat::Type { + let mut software = None; while *fmts != AVPixelFormat::AV_PIX_FMT_NONE { if *fmts == AVPixelFormat::AV_PIX_FMT_D3D11 { // frames-context manuel : impose BindFlags (sinon ffmpeg met BIND_DECODER seul, @@ -460,9 +485,18 @@ unsafe extern "C" fn get_hw_format( (*ctx).hw_frames_ctx = frames; return AVPixelFormat::AV_PIX_FMT_D3D11; } + let desc = av_pix_fmt_desc_get(*fmts); + if software.is_none() + && !desc.is_null() + && (*desc).flags & AV_PIX_FMT_FLAG_HWACCEL as u64 == 0 + { + software = Some(*fmts); + } fmts = fmts.add(1); } - AVPixelFormat::AV_PIX_FMT_NONE + // No D3D11 surface for this stream (High 10, 4:2:2, 4:4:4 whose format was unknown at + // open, #1532): decode in software. `Decoder::present` routes the frame to CpuFrames. + software.unwrap_or(AVPixelFormat::AV_PIX_FMT_NONE) } pub fn run_c0(screen: &str, out: &str, gpu: &Gpu) -> Result { @@ -846,15 +880,23 @@ impl Decoder { )?; allow_d3d11va_h264_baseline(resources.dctx); - // Hardware D3D11VA is the H.264 capture path. WARP has no video decoder - // (`tests/warp_device_cannot_decode.rs`). AV1/VP9 (legacy WebMs, #554) - // take the same software CpuFrames axis as Backend::Cpu. - let want_hw = gpu.backend != Backend::Cpu && d3d11va_for_codec(codec_id); - let cpu = if want_hw { - None - } else { + // Hardware D3D11VA is the H.264 8-bit 4:2:0 capture path. WARP has no video + // decoder (`tests/warp_device_cannot_decode.rs`). AV1/VP9 (legacy WebMs, #554) + // and H.264 High 10 / 4:2:2 / 4:4:4 (no D3D11 surface exists for them) take + // the same software CpuFrames axis as Backend::Cpu. + let format = (*codecpar).format as AVPixelFormat::Type; + let want_hw = gpu.backend != Backend::Cpu && d3d11va_for_stream(codec_id, format); + let cpu = if !want_hw { (*resources.dctx).thread_count = 0; Some(CpuFrames::new(gpu)?) + } else if format == AVPixelFormat::AV_PIX_FMT_NONE { + // An unknown format may still turn out to have no D3D11 surface (#1532): + // `get_hw_format` then falls back to software, and its frames need CpuFrames. + // ponytail: that fallback decodes on one thread (the hardware default, fixed at + // avcodec_open2); reopen in software if a large High 10 file ever plays slowly. + Some(CpuFrames::new(gpu)?) + } else { + None }; if want_hw { @@ -889,9 +931,10 @@ impl Decoder { /// `next`, donc les deux doivent rendre la même chose. Le temps (`cur_time_sec`), lui, /// continue de se lire sur la vraie frame décodée. pub(crate) fn cur_frame(&self) -> *mut AVFrame { + let hw = unsafe { (*self.frame).format } == AVPixelFormat::AV_PIX_FMT_D3D11 as i32; let frame = match &self.cpu { - Some(cpu) => cpu.current(), - None => self.frame, + Some(cpu) if !hw => cpu.current(), + _ => self.frame, }; // `AVFrame*` identifies the reusable container, not whether it currently contains a // presentable frame. It stays allocated before the first decode and can be unreffed by @@ -1046,9 +1089,18 @@ impl Decoder { if r.den != 0 && r.num != 0 { r.num as f64 / r.den as f64 } else { 30.0 } } - /// Durée réellement annoncée par le flux vidéo. La durée du stream est prioritaire ; - /// `nb_frames / fps` sert de repli pour les conteneurs qui omettent `duration`. + /// Durée disponible de la source : le plus long du flux vidéo et du conteneur (cf. + /// `cover_container_duration`). pub(crate) unsafe fn available_duration_sec(&self) -> Option { + crate::timeline_walk::cover_container_duration( + self.video_stream_duration_sec(), + sn_fmt_duration(self.fmt), + ) + } + + /// Durée annoncée par le seul flux vidéo. La durée du stream est prioritaire ; + /// `nb_frames / fps` sert de repli pour les conteneurs qui omettent `duration`. + unsafe fn video_stream_duration_sec(&self) -> Option { let stream = sn_fmt_stream(self.fmt, self.vidx); let duration = (*stream).duration; let tb_sec = self.tb_sec(); @@ -1077,9 +1129,19 @@ impl Decoder { } let pts = (*self.frame).best_effort_timestamp; self.cur_pts = if pts == i64::MIN { None } else { Some(pts) }; + self.present() + } + + /// D3D11VA surfaces go to the compositor as they are; software frames go through + /// CpuFrames. A D3D11VA decoder only yields software frames after `get_hw_format`'s + /// fallback, which `open` prepares CpuFrames for. + unsafe fn present(&mut self) -> Result<*mut AVFrame> { + if (*self.frame).format == AVPixelFormat::AV_PIX_FMT_D3D11 as i32 { + return Ok(self.frame); + } match &mut self.cpu { Some(cpu) => cpu.present(self.frame), - None => Ok(self.frame), + None => bail!("D3D11VA decoder returned a software frame without CpuFrames"), } } @@ -1147,10 +1209,7 @@ impl Decoder { self.has_peek = false; let pts = (*self.frame).best_effort_timestamp; self.cur_pts = if pts == i64::MIN { None } else { Some(pts) }; - match &mut self.cpu { - Some(cpu) => cpu.present(self.frame), - None => Ok(self.frame), - } + self.present() } } @@ -1543,6 +1602,8 @@ impl VideoEncoder { // d'ajouter n'émettent pas d'extradata sans ce drapeau, et personne ici n'a de machine // pour constater le MP4 bancal qui en sortirait. (*ctx).flags |= AV_CODEC_FLAG_GLOBAL_HEADER as i32; + // `rgb2y` / `rgb2uv` de `shaders.hlsl` : BT.709 limité. + crate::ffi::tag_export_color(ctx, AVColorSpace::AVCOL_SPC_BT709); if pix_fmt == AVPixelFormat::AV_PIX_FMT_D3D11 { (*ctx).hw_frames_ctx = av_buffer_ref(hw_frames); } @@ -1942,14 +2003,15 @@ mod tests { fn raw_hardware_prerequisite() -> windows::core::Result<()> { use windows::Win32::Foundation::{E_UNEXPECTED, HMODULE}; use windows::Win32::Graphics::Direct3D::{ - D3D_DRIVER_TYPE_HARDWARE, D3D_FEATURE_LEVEL, D3D_FEATURE_LEVEL_11_1, + D3D_DRIVER_TYPE_HARDWARE, D3D_FEATURE_LEVEL, D3D_FEATURE_LEVEL_11_0, + D3D_FEATURE_LEVEL_11_1, }; use windows::Win32::Graphics::Direct3D11::{ D3D11CreateDevice, ID3D11Device, ID3D11DeviceContext, D3D11_CREATE_DEVICE_BGRA_SUPPORT, D3D11_CREATE_DEVICE_VIDEO_SUPPORT, D3D11_SDK_VERSION, }; - let levels = [D3D_FEATURE_LEVEL_11_1]; + let levels = [D3D_FEATURE_LEVEL_11_1, D3D_FEATURE_LEVEL_11_0]; let mut device: Option = None; let mut context: Option = None; let mut got = D3D_FEATURE_LEVEL::default(); @@ -1966,7 +2028,7 @@ mod tests { Some(&mut context), )?; } - if device.is_none() || context.is_none() || got != D3D_FEATURE_LEVEL_11_1 { + if device.is_none() || context.is_none() || got.0 < D3D_FEATURE_LEVEL_11_0.0 { return Err(windows::core::Error::from(E_UNEXPECTED)); } Ok(()) @@ -1998,6 +2060,40 @@ mod tests { } } + /// Le MP4 dit sa matrice : BT.709, celle de `shaders.hlsl`. Sans étiquette ffmpeg + /// suppose BT.601 et décale les couleurs saturées. Sans pool D3D11 c'est un candidat + /// en mémoire système qui ouvre, par le même `try_open` que l'export. + #[test] + fn export_encoder_tags_its_colour() { + let opened = unsafe { + VideoEncoder::open(&ExportCodec::H264, 320, 180, 30, 1_000_000, ptr::null_mut()) + }; + let Ok(enc) = opened else { + eprintln!("aucun encodeur H.264 en mémoire système ici, test sauté"); + return; + }; + unsafe { + let mut par = avcodec_parameters_alloc(); + avcodec_parameters_from_context(par, enc.ctx); + let color = ( + (*par).color_primaries, + (*par).color_trc, + (*par).color_space, + (*par).color_range, + ); + avcodec_parameters_free(&mut par); + assert_eq!( + color, + ( + AVColorPrimaries::AVCOL_PRI_BT709, + AVColorTransferCharacteristic::AVCOL_TRC_BT709, + AVColorSpace::AVCOL_SPC_BT709, + AVColorRange::AVCOL_RANGE_MPEG, + ) + ); + } + } + #[test] fn hardware_prerequisite_classification() { use windows::Win32::Foundation::{E_INVALIDARG, E_OUTOFMEMORY, E_UNEXPECTED}; @@ -2122,6 +2218,19 @@ mod tests { assert!(!d3d11va_for_codec(AVCodecID::AV_CODEC_ID_VP9)); } + #[test] + fn d3d11va_is_8bit_420_h264_only() { + use AVPixelFormat::*; + let h264 = AVCodecID::AV_CODEC_ID_H264; + assert!(d3d11va_for_stream(h264, AV_PIX_FMT_YUV420P)); + assert!(d3d11va_for_stream(h264, AV_PIX_FMT_YUVJ420P)); + assert!(d3d11va_for_stream(h264, AV_PIX_FMT_NONE)); + assert!(!d3d11va_for_stream(h264, AV_PIX_FMT_YUV420P10LE)); + assert!(!d3d11va_for_stream(h264, AV_PIX_FMT_YUV422P)); + assert!(!d3d11va_for_stream(h264, AV_PIX_FMT_YUV444P)); + assert!(!d3d11va_for_stream(AVCodecID::AV_CODEC_ID_AV1, AV_PIX_FMT_YUV420P)); + } + #[test] fn require_decoder_rejects_none() { let err = unsafe { require_decoder_id(AVCodecID::AV_CODEC_ID_NONE) } @@ -2198,6 +2307,25 @@ mod tests { out } + /// A 64x64, 25 fps, 0.6 s clip whose frames around the two cut points get their own colour: + /// blue before 0.36 s, magenta 0.36-0.399 s (the last frame a 0.4 s cut keeps), lime from + /// 0.4 s, yellow from 0.56 s (the file's last frame). + fn encode_end_frame_colours(codec_args: &[&str], filename: &str) -> std::path::PathBuf { + let boxes = [ + ("magenta", "between(t,0.36,0.399)"), + ("lime", "between(t,0.4,0.559)"), + ("yellow", "gte(t,0.56)"), + ]; + let filter = boxes + .iter() + .map(|(c, en)| format!("drawbox=x=0:y=0:w=iw:h=ih:color={c}:t=fill:enable='{en}'")) + .collect::>() + .join(","); + let mut args = vec!["-vf", filter.as_str()]; + args.extend_from_slice(codec_args); + encode_named_color(&args, filename, "blue", "0.6") + } + unsafe fn allocated_decoder_open_resources(filename: &str) -> DecoderOpenResources { let path = encode_color(&["-c:v", "libopenh264", "-b:v", "200k"], filename); let cpath = CString::new(path.to_string_lossy().as_bytes()).expect("fixture path CString"); @@ -2851,6 +2979,124 @@ mod tests { ); } + /// 64x64 H.264 High 10 (yuv420p10le), 10 frames, Annex B, from libx264, which the + /// bundled LGPL ffmpeg lacks: + /// `ffmpeg -f lavfi -i color=c=red:s=64x64:d=0.4 -c:v libx264 -pix_fmt yuv420p10le + /// -profile:v high10 -bsf:v filter_units=remove_types=6 high10.h264` + const H264_HIGH10_64X64: &[u8] = &[ + 0x00, 0x00, 0x00, 0x01, 0x67, 0x6e, 0x00, 0x0a, 0xa6, 0xcd, 0x94, 0x42, 0x6c, 0x04, 0x40, + 0x00, 0x00, 0x03, 0x00, 0x40, 0x00, 0x00, 0x0c, 0x83, 0xc4, 0x89, 0x65, 0x80, 0x00, 0x00, + 0x00, 0x01, 0x68, 0xeb, 0xe3, 0xcb, 0x22, 0xc0, 0x00, 0x00, 0x01, 0x65, 0x88, 0x84, 0x00, + 0x12, 0xff, 0xfe, 0xe8, 0xc9, 0xfc, 0xcb, 0x2b, 0x2f, 0xe3, 0xd4, 0x38, 0x5c, 0x46, 0x29, + 0xd3, 0xa1, 0xa7, 0x1c, 0xc2, 0x3d, 0x1d, 0x23, 0xfe, 0xb9, 0x07, 0x66, 0x4b, 0xc8, 0x21, + 0xc1, 0x59, 0x07, 0x8b, 0x89, 0xc1, 0x00, 0x00, 0x00, 0x01, 0x41, 0x9a, 0x24, 0x6c, 0x41, + 0x1f, 0xfe, 0xb5, 0x2b, 0xef, 0x00, 0x00, 0x00, 0x01, 0x41, 0x9e, 0x42, 0x78, 0x87, 0xff, + 0x04, 0xc5, 0x00, 0x00, 0x00, 0x01, 0x01, 0x9e, 0x61, 0x74, 0x43, 0xbf, 0x05, 0xbc, 0x00, + 0x00, 0x00, 0x01, 0x01, 0x9e, 0x63, 0x6a, 0x43, 0xbf, 0x05, 0xbd, 0x00, 0x00, 0x00, 0x01, + 0x41, 0x9a, 0x68, 0x34, 0xb4, 0x4c, 0x04, 0x3f, 0xff, 0xfe, 0xa9, 0x9d, 0x35, 0x00, 0x00, + 0x00, 0x01, 0x41, 0x9e, 0x86, 0x45, 0x11, 0x2c, 0x3f, 0xff, 0x04, 0xc5, 0x00, 0x00, 0x00, + 0x01, 0x01, 0x9e, 0xa5, 0x74, 0x43, 0xbf, 0x05, 0xbd, 0x00, 0x00, 0x00, 0x01, 0x01, 0x9e, + 0xa7, 0x6a, 0x43, 0xbf, 0x05, 0xbc, 0x00, 0x00, 0x00, 0x01, 0x41, 0x9a, 0xa9, 0x34, 0xb6, + 0x4c, 0x04, 0x3b, 0xff, 0xfe, 0xa9, 0x9d, 0x34, + ]; + + /// #1532: High 10 whose pixel format is still unknown after `find_stream_info` opens + /// on D3D11VA, which has no surface for it. The video comes after ~7 MB of audio in an + /// MPEG-TS, past the 5 MB probe limit, so the format stays `AV_PIX_FMT_NONE` at open. + /// Decoding must fall back to software, not fail. + #[test] + fn h264_high10_with_unknown_format_at_open_decodes_in_software() { + let Some(gpu) = + strict_hardware_gpu("h264_high10_with_unknown_format_at_open_decodes_in_software") + else { + return; + }; + let dir = std::env::temp_dir().join(format!("openscreen-1532-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("temp dir"); + let es = dir.join("high10.h264"); + std::fs::write(&es, H264_HIGH10_64X64).expect("write High 10 stream"); + let out = dir.join("late-high10.ts"); + let ff = ffmpeg_exe(); + let output = std::process::Command::new(&ff) + .args([ + "-y", + "-f", + "lavfi", + "-i", + "anullsrc=r=48000:cl=stereo:d=150", + ]) + .args(["-f", "h264", "-i"]) + .arg(&es) + .args(["-map", "0:a", "-map", "1:v", "-c:a", "mp2", "-b:a", "384k"]) + .args([ + "-c:v", + "copy", + "-bsf:v", + "setts=ts=140*90000+N*3600:time_base=1/90000", + ]) + .args(["-f", "mpegts"]) + .arg(&out) + .output() + .unwrap_or_else(|e| panic!("spawn {ff:?}: {e}")); + assert!( + output.status.success(), + "ffmpeg mux failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + let mut dec = unsafe { Decoder::open(out.to_str().expect("utf8 path"), &gpu) } + .unwrap_or_else(|e| panic!("High 10 Decoder::open: {e:#}")); + assert_eq!( + unsafe { (*(*sn_fmt_stream(dec.fmt, dec.vidx)).codecpar).format }, + AVPixelFormat::AV_PIX_FMT_NONE as i32, + "fixture must leave the pixel format unknown at open" + ); + assert!( + !dec.hwdev.is_null(), + "unknown format must keep D3D11VA at open" + ); + let frame = unsafe { first_decoded_frame(&mut dec) }; + assert_eq!(unsafe { (*frame).width }, 64); + assert_eq!( + unsafe { (*dec.frame).format }, + AVPixelFormat::AV_PIX_FMT_YUV420P10LE as i32, + "High 10 must be decoded in software" + ); + assert_eq!( + dec.cpu.as_ref().map(|cpu| cpu.current()), + Some(frame), + "the software frame must be presented through CpuFrames" + ); + assert_eq!( + dec.cur_frame(), + frame, + "cur_frame must be the presented frame" + ); + println!( + "CORE_ASSERTIONS_COMPLETED:h264_high10_with_unknown_format_at_open_decodes_in_software" + ); + } + + /// #1532: High 10 has no D3D11 surface, so `get_hw_format` decodes it in software. + /// `decode_frame_n` must refuse that frame rather than hand it to `nv12_srvs`. + #[test] + fn decode_frame_n_rejects_a_software_frame() { + let Some(gpu) = strict_hardware_gpu("decode_frame_n_rejects_a_software_frame") else { + return; + }; + let path = std::env::temp_dir().join(format!( + "openscreen-1532-decode-frame-n-{}.h264", + std::process::id() + )); + std::fs::write(&path, H264_HIGH10_64X64).expect("write High 10 stream"); + let error = decode_frame_error(&path, &gpu, 0); + let _ = std::fs::remove_file(&path); + assert!( + format!("{error:#}").contains("no D3D11 surface"), + "unexpected error: {error:#}" + ); + println!("CORE_ASSERTIONS_COMPLETED:decode_frame_n_rejects_a_software_frame"); + } + #[test] fn h264_opens_on_d3d11va() { let Some(gpu) = strict_hardware_gpu("h264_opens_on_d3d11va") else { @@ -2998,7 +3244,9 @@ mod tests { let h264 = ["-c:v", "libopenh264", "-b:v", "200k"]; // Long enough that a view which wrapped onto it is still there when the test looks. let first = encode_named_color(&h264, "997-first.mp4", "red", "3.0"); - let last = encode_named_color(&h264, "997-last.mp4", "blue", "0.6"); + // The last frame before each end has its own colour (magenta before the 0.4 s cut, yellow + // for the file's last frame), so the held picture says which frame the view stopped on. + let last = encode_end_frame_colours(&h264, "997-last.mp4"); // The paths travel in JSON, and the view matches them against its requests as strings. let first = first.to_str().expect("utf8").replace('\\', "/"); let last = last.to_str().expect("utf8").replace('\\', "/"); @@ -3029,6 +3277,8 @@ mod tests { let first_frame = wait_for_frame(&view, 0, |p| p.clip_index == 2); let mut gen = first_frame.0; let mut held = first_frame.5; + let mut held_rgba = first_frame.3; + let (mut held_w, mut held_h) = (first_frame.1, first_frame.2); // Play to the end of the last clip, then about as long again. view.set_playing(true); @@ -3038,20 +3288,44 @@ mod tests { while let Some(frame) = view.latest_frame_since(gen) { gen = frame.0; held = frame.5; + held_rgba = frame.3; + (held_w, held_h) = (frame.1, frame.2); } - // The frame held at the end is the clip's own last frame, not the first one its cut - // removed (the 25 fps file has a frame at exactly 0.4 s). + // The published time is the hold clock: it runs on to the clip's end even if decoding + // stopped early. The bounds check the clock (reached the end, never past it); the + // colour below checks which frame is held. assert!( - held.source_time_sec < last_end_sec, - "end {last_end_sec}: held a frame past the clip's end, at {:.3} s", + held.source_time_sec <= last_end_sec + 1e-6, + "end {last_end_sec}: the clock ran past the clip's end, at {:.3} s", held.source_time_sec ); - // And it did play up to there: a view stalled at its start would pass the bound above. assert!( held.source_time_sec > last_end_sec - 0.1, - "end {last_end_sec}: playback did not reach the clip's end, held at {:.3} s", + "end {last_end_sec}: the clock did not reach the clip's end, held at {:.3} s", held.source_time_sec ); + // The colour proves which frame is held: the last one the cut keeps (magenta), or the + // file's last frame (yellow). Stuck on an earlier frame shows blue or lime instead. + let center = (((held_h / 2) * held_w + held_w / 2) * 4) as usize; + let rgb = &held_rgba[center..center + 3]; + let (r, g, b) = (i32::from(rgb[0]), i32::from(rgb[1]), i32::from(rgb[2])); + if last_end_sec < 0.5 { + assert!( + r > 150 && b > 150 && g < 100, + "end {last_end_sec}: expected magenta, the last frame the cut keeps: {rgb:?}" + ); + } else { + assert!( + r > 150 && g > 150 && b < 100, + "end {last_end_sec}: expected yellow, the file's last frame: {rgb:?}" + ); + } + + assert_eq!( + held.clip_index, 2, + "end {last_end_sec}: held a frame from clip {} instead of the last clip", + held.clip_index + ); // A seek inside the last clip, as the app sends it while paused. view.set_time(0.2); @@ -3137,6 +3411,11 @@ unsafe fn drain_encoder( } averr(r, "receive_packet")?; (*opkt).stream_index = (*ostream).index; + // A packet without a duration ends the MP4 edit list on its pts, which drops the last + // frame (see `pipeline_macos::drain_encoder`). The time base is 1/fps: one tick, one frame. + if (*opkt).duration <= 0 { + (*opkt).duration = 1; + } av_packet_rescale_ts(opkt, (*ectx).time_base, (*ostream).time_base); averr( av_interleaved_write_frame(octx, opkt), diff --git a/crates/compositor/src/regions.rs b/crates/compositor/src/regions.rs index a923dde780..ad7865f025 100644 --- a/crates/compositor/src/regions.rs +++ b/crates/compositor/src/regions.rs @@ -7,7 +7,7 @@ //! temporel et la géométrie, pas le rendu GPU). use crate::camera::Follow; -use crate::cursor::CursorTrack; +use crate::cursor::{CursorTrack, WHOLE_FRAME}; use crate::scene::{SceneCameraFullscreenRegion, SceneSpeedRegion, SceneZoomRegion}; /// Quantification commune vidéo/audio, sur UNE grille pour toute la timeline : la frame de sortie @@ -568,7 +568,8 @@ fn camera_for(rotation: &Option) -> Camera { /// Ce que l'orbite en focus auto lit de la scène : la piste curseur (celle de la parallaxe, /// donc `None` curseur masqué, comme à l'export), le recadrage du clip actif dans le repère -/// normalisé du curseur (`[x0, y0, x1, y1]`) et sa fenêtre source `[début, fin)`. +/// normalisé du curseur (`[x0, y0, x1, y1]`) et sa fenêtre source `[début, fin)`. Le zoom à plat +/// en focus auto lit le même recadrage (`resolve_focus`), avec sa propre piste. #[derive(Clone, Copy)] pub struct CameraFrame<'a> { pub track: Option<&'a CursorTrack>, @@ -580,7 +581,7 @@ impl CameraFrame<'static> { /// Sans piste : la caméra vise le centre, à sa pose de repos. pub const NONE: CameraFrame<'static> = CameraFrame { track: None, - crop: [0.0, 0.0, 1.0, 1.0], + crop: WHOLE_FRAME, window: [f32::NEG_INFINITY, f32::INFINITY], }; } @@ -760,10 +761,14 @@ pub fn arrow_local_geometry( /// Focus effectif d'une région à `t` : sa position fixe, sauf en mode "auto" où elle suit la /// télémétrie curseur (port de `getResolvedFocus`, sans le clamp — le crop-window de /// `compositor.rs` clampe déjà après coup, cf. `su0.clamp(...)`, donc redondant ici). +/// +/// Le focus se lit dans le recadrage `crop` du clip actif (`screen_source_rect`) : le suivi auto +/// y ramène donc la piste, qui est dans le repère de l'image entière. fn resolve_focus( region: &SceneZoomRegion, t: f32, cursor: Option<&CursorTrack>, + crop: [f32; 4], clock: &ScreenClock, ) -> [f32; 2] { // Sous la caméra réelle, le cadrage vient de sa visée (`orbit_at`), en auto comme en manuel : @@ -786,7 +791,7 @@ fn resolve_focus( } else { clock.source_at(clock.at(region.end_sec as f32) + window) }; - if let Some((cx, cy)) = track.follow_in_view(t0, t, t_end, region.scale) { + if let Some((cx, cy)) = track.follow_in_view(t0, t, t_end, region.scale, crop) { return [cx, cy]; } } @@ -910,8 +915,8 @@ pub fn zoom_state_in( let progress = ease_connected_pan(clamp01((screen_t - t_start) / (t_end - t_start).max(1e-3))); let (cur, next) = (®ions[ci], ®ions[ni]); - let cur_focus = resolve_focus(cur, t, cursor, clock); - let next_focus = resolve_focus(next, t, cursor, clock); + let cur_focus = resolve_focus(cur, t, cursor, frame.crop, clock); + let next_focus = resolve_focus(next, t, cursor, frame.crop, clock); // Entre deux présets DIFFÉRENTS, la base traverse des poses qu'aucun préset ne valide // (left→right passe par Y = 0) : la marge que l'échelle gelée laissait au budget n'y est // plus. La porte se ferme donc en milieu de course et ne se rouvre qu'au ras des bouts — @@ -976,7 +981,7 @@ pub fn zoom_state_in( return ZoomState { scale: next.scale, strength: 1.0, - focus: resolve_focus(next, t, cursor, clock), + focus: resolve_focus(next, t, cursor, frame.crop, clock), rotation: fixed_rotation(next), tilt: fixed_flag(next), camera, @@ -1014,7 +1019,7 @@ pub fn zoom_state_in( match best { Some((i, strength)) => { let r = ®ions[i]; - let focus = resolve_focus(r, t, cursor, clock); + let focus = resolve_focus(r, t, cursor, frame.crop, clock); let scale = scale_lerp(1.0, r.scale, strength); // La référence (`zoomTransform.ts`) fait converger le point de focus vers le centre // de l'écran LINÉAIREMENT : screen(f) = 0.5 + (f - 0.5)(1 - strength). Passer @@ -1812,8 +1817,8 @@ mod zoom_focus_tests { "garde : le ralenti doit déplacer le départ ({t0})" ); let t_end = clock.source_at(clock.at(r.end_sec as f32) + window); - let (x, y) = track.follow_in_view(t0, 3.0, t_end, r.scale).unwrap(); - assert_eq!(resolve_focus(&r, 3.0, Some(&track), &clock), [x, y]); + let (x, y) = track.follow_in_view(t0, 3.0, t_end, r.scale, WHOLE_FRAME).unwrap(); + assert_eq!(resolve_focus(&r, 3.0, Some(&track), WHOLE_FRAME, &clock), [x, y]); } #[test] @@ -1828,21 +1833,48 @@ mod zoom_focus_tests { let t0 = r.start_sec as f32 - zoom_transition_s(r.scale); let clock = ScreenClock::default(); let t_end = clock.source_at(clock.at(r.end_sec as f32) + zoom_transition_s(r.scale)); - let last = track.follow_in_view(t0, t_end, t_end, r.scale).unwrap(); + let last = track.follow_in_view(t0, t_end, t_end, r.scale, WHOLE_FRAME).unwrap(); for t in [4.5f32, 6.0, 7.9] { assert_eq!( - resolve_focus(&r, t, Some(&track), &clock), + resolve_focus(&r, t, Some(&track), WHOLE_FRAME, &clock), [last.0, last.1], "t = {t}" ); } - let first = track.follow_in_view(t0, 2.1, t_end, r.scale).unwrap(); + let first = track.follow_in_view(t0, 2.1, t_end, r.scale, WHOLE_FRAME).unwrap(); assert_eq!( - resolve_focus(&r, 2.1, Some(&track), &clock), + resolve_focus(&r, 2.1, Some(&track), WHOLE_FRAME, &clock), [first.0, first.1] ); } + /// Clip recadré (dialogue « Edit clip ») : le focus auto suit le pointeur dans le recadrage, + /// le repère de la coupe zoomée, pas dans l'image entière. Recadrage [0,4 ; 0,8] en x, + /// pointeur à 0,45 de l'image, donc à 0,125 de ce qu'on voit : au zoom 2 la vue se cale à + /// gauche (focus 0,25) et le montre. Lu dans l'image entière, le focus valait 0,45 et la + /// vue [0,2 ; 0,7] laissait le pointeur dehors (#1409). + #[test] + fn auto_focus_follows_the_pointer_in_the_crop() { + let track = CursorTrack::new( + (0..=300).map(|i| (i as f32 / 30.0, 0.45, 0.5)).collect(), + Vec::new(), + Vec::new(), + ); + let mut r = region(2.0, 0.5); + r.focus_mode = Some("auto".into()); + let clock = ScreenClock::default(); + let at = |crop: [f32; 4]| { + let frame = CameraFrame { crop, ..CameraFrame::NONE }; + zoom_state_in(&[r.clone()], 5.0, Some(&track), &frame, &clock).focus + }; + let cropped = at([0.4, 0.0, 0.8, 1.0]); + assert!((cropped[0] - 0.25).abs() < 1e-4 && (cropped[1] - 0.5).abs() < 1e-4, "{cropped:?}"); + let pointer = (0.45 - 0.4) / 0.4; + assert!((cropped[0] - pointer).abs() < 0.25, "pointeur hors de la vue : {cropped:?}"); + // Sans recadrage, rien ne change : le pointeur est déjà dans le repère de la vue. + assert!((at(WHOLE_FRAME)[0] - 0.45).abs() < 1e-4); + } + #[test] fn zoom_cursor_alpha_hides_during_zoom() { let mut r = region(2.0, 0.5); diff --git a/crates/compositor/src/scene.rs b/crates/compositor/src/scene.rs index 2a479c8dd8..b4a5fb11d9 100644 --- a/crates/compositor/src/scene.rs +++ b/crates/compositor/src/scene.rs @@ -282,6 +282,8 @@ pub struct SceneEffects { pub blur: f32, /// 0..1 force de l'ombre. pub shadow: f32, + #[serde(default)] + pub webcam_shadow: Option, /// Slider Roundness, en FRACTION du petit côté du cadre de sortie. /// /// Toute longueur qui traverse ce contrat est une fraction, jamais un nombre de pixels, et @@ -953,6 +955,11 @@ pub struct Scene { /// Crop écran par clip, dans le même ordre que `clips` (`cropByClip` côté TS). #[serde(default)] pub crop_by_clip: Vec>, + /// Half turn of each clip's main picture, same order as `clips` (`rotationByClip` on + /// the TS side). Values are 0 or 180; any other u16 is treated as upright. Absent in every + /// scene written before it. + #[serde(default)] + pub rotation_by_clip: Vec, /// Effet d'arrière-plan de la webcam. Absent = aucun effet. #[serde(default)] pub webcam_effect: Option, @@ -975,6 +982,11 @@ impl Scene { Ok(serde_json::from_str(json)?) } + /// Whether the clip being composed is turned by 180°. + pub fn screen_turned(&self) -> bool { + self.rotation_by_clip.get(self.active_clip_index) == Some(&180) + } + /// Copie de scène limitée aux régions du clip actif. `clipIndex` est l'identité fiable /// lorsque plusieurs clips réutilisent les mêmes temps source ; son absence retombe sur le /// chevauchement avec la fenêtre source pour accepter les anciens payloads. @@ -1381,6 +1393,25 @@ mod annotation_tests { ); } + #[test] + fn rotation_by_clip_defaults_empty_and_reads_the_active_clip() { + let scene = Scene::from_json(&scene_json("[]")).expect("parse"); + assert!(scene.rotation_by_clip.is_empty()); + assert!(!scene.screen_turned()); + let mut turned = scene.clone(); + turned.rotation_by_clip = vec![0, 180]; + assert!(!turned.for_clip_window(0, 0.0, 1.0).screen_turned()); + assert!(turned.for_clip_window(1, 0.0, 1.0).screen_turned()); + // Only a half turn counts; anything else stays upright. + turned.rotation_by_clip = vec![90]; + assert!(!turned.for_clip_window(0, 0.0, 1.0).screen_turned()); + // The field arrives from the app as `rotationByClip`. + let json = scene_json("[]").replace(r#""cropByClip":[]"#, r#""cropByClip":[],"rotationByClip":[180]"#); + let parsed = Scene::from_json(&json).expect("parse rotationByClip"); + assert_eq!(parsed.rotation_by_clip, vec![180]); + assert!(parsed.screen_turned()); + } + #[test] fn for_clip_window_still_falls_back_to_window_overlap_without_a_clip_index() { // Vieux payload : rien ne dit à quel clip la région appartient, le chevauchement de diff --git a/crates/compositor/src/shaders.hlsl b/crates/compositor/src/shaders.hlsl index 8478823035..eab13c8d96 100644 --- a/crates/compositor/src/shaders.hlsl +++ b/crates/compositor/src/shaders.hlsl @@ -49,6 +49,12 @@ VSOut vs_main(uint vid : SV_VertexID) return o; } +// Plage de la source NV12 liée en t0/t1 : 1 = pleine, 0 = limitée (`Compositor::draw_video`). +cbuffer Source : register(b1) +{ + float full_range; +}; + Texture2D texY : register(t0); Texture2D texUV : register(t1); Texture2D texImg : register(t2); // wallpaper image RGBA (fond, mode 6) ; mode 18 : l'écran cadré isolé @@ -85,6 +91,14 @@ float3 sample_yuv(float2 uv) { float y = texY.Sample(samp, uv); float2 cbcr = texUV.Sample(samp, uv); + // Plage pleine (vidéo importée, #1531) : ramenée en limitée avant la matrice. Branche + // uniforme : une source limitée garde exactement son calcul d'avant. + [branch] if (full_range > 0.5) + { + y = y * (219.0 / 255.0) + 16.0 / 255.0; + // Autour du neutre 128 : un gris plein reste gris (sans ça, 128 → 128,44). + cbcr = (cbcr - 128.0 / 255.0) * (224.0 / 255.0) + 128.0 / 255.0; + } return yuv709_limited(y, cbcr); } diff --git a/crates/compositor/src/shaders.metal b/crates/compositor/src/shaders.metal index 2a469a504c..06ee2d87ba 100644 --- a/crates/compositor/src/shaders.metal +++ b/crates/compositor/src/shaders.metal @@ -141,6 +141,42 @@ inline float3 yuv709_limited(float y, float2 cbcr) return clamp(rgb, 0.0, 1.0); } +// BT.709 pleine plage -> RGB : Y sur 0..255, CbCr centrés sur 128 et étalés sur 255. +inline float3 yuv709_full(float y, float2 cbcr) +{ + float Cb = cbcr.x - 128.0 / 255.0; + float Cr = cbcr.y - 128.0 / 255.0; + float3 rgb; + rgb.r = y + 1.5748 * Cr; + rgb.g = y - 0.1873 * Cb - 0.4681 * Cr; + rgb.b = y + 1.8556 * Cb; + return clamp(rgb, 0.0, 1.0); +} + +// BT.601 / BT.2020 -> RGB. `m` : coefficients Cr->R, Cb->G, Cr->G, Cb->B. BT.709 garde ses +// deux fonctions à constantes littérales, pour rendre ses frames à l'octet près comme avant. +// `ten_bit` : un échantillon P010 lu en `R16Unorm` vaut code10 * 64 / 65535, pas code8 / 255 ; +// on revient au code 10 bits pour appliquer ses bornes (64..940, chroma 512, pleine 0..1023). +inline float3 yuv_matrix(float y, float2 cbcr, float4 m, bool full, bool ten_bit) +{ + float Yf, Cb, Cr; + if (ten_bit) { + float code = 65535.0 / 64.0; + Yf = full ? y * code / 1023.0 : (y * code - 64.0) / 876.0; + Cb = full ? (cbcr.x * code - 512.0) / 1023.0 : (cbcr.x * code - 512.0) / 896.0; + Cr = full ? (cbcr.y * code - 512.0) / 1023.0 : (cbcr.y * code - 512.0) / 896.0; + } else { + Yf = full ? y : (y * 255.0 - 16.0) / 219.0; + Cb = full ? cbcr.x - 128.0 / 255.0 : (cbcr.x * 255.0 - 128.0) / 224.0; + Cr = full ? cbcr.y - 128.0 / 255.0 : (cbcr.y * 255.0 - 128.0) / 224.0; + } + float3 rgb; + rgb.r = Yf + m.x * Cr; + rgb.g = Yf - m.y * Cb - m.z * Cr; + rgb.b = Yf + m.w * Cb; + return clamp(rgb, 0.0, 1.0); +} + // `texture2d::sample` rend TOUJOURS un `float4` en MSL, là où le HLSL // `Texture2D` rend un scalaire : d'où les `.r` / `.rg` que le port d'origine // n'avait pas (et qui ne compilaient pas). @@ -148,9 +184,18 @@ inline float3 sample_yuv(float2 uv, texture2d texY, texture2d texUV) { - float y = texY.sample(samp, uv).r; - float2 cbcr = texUV.sample(samp, uv).rg; - return yuv709_limited(y, cbcr); + float4 y = texY.sample(samp, uv); + float4 c = texUV.sample(samp, uv); + // Bleu à 1 sur la vue CbCr : pleine plage (420f, xf20). + bool full = c.b > 0.5; + // Vert ou bleu à 1 sur la vue Y : matrice BT.601 ou BT.2020 posée par `nv12_srvs` ; + // alpha à 0 sur cette vue : échantillons 10 bits. + bool ten_bit = y.a < 0.5; + if (y.g > 0.5) + return yuv_matrix(y.r, c.rg, float4(1.402, 0.344136, 0.714136, 1.772), full, ten_bit); + if (y.b > 0.5) + return yuv_matrix(y.r, c.rg, float4(1.4746, 0.164553, 0.571353, 1.8814), full, ten_bit); + return full ? yuv709_full(y.r, c.rg) : yuv709_limited(y.r, c.rg); } // SDF segment à bouts ronds — la primitive des flèches d'annotation. diff --git a/crates/compositor/src/timeline_walk.rs b/crates/compositor/src/timeline_walk.rs index e7829180cc..ac05f17a14 100644 --- a/crates/compositor/src/timeline_walk.rs +++ b/crates/compositor/src/timeline_walk.rs @@ -19,6 +19,7 @@ use crate::cursor::CursorTrack; use crate::d3d::Gpu; use crate::ffi::AVFrame; use crate::frame_geometry::webcam_is_real; +use crate::live::cursor_key; use crate::pipeline::{ClipSource, Decoder}; use crate::regions::{speed_segments_for_window, SpeedSegment}; use crate::scene::Scene; @@ -75,6 +76,25 @@ pub(crate) fn frame_step(next: NextFrameTime, offset_sec: f64, target_sec: f64) } } +/// Durée disponible d'une source vidéo : le plus long de la durée annoncée par le flux et de +/// celle du conteneur (`container_us`, en µs, négatif si inconnue). +/// +/// Une capture d'écran statique (ScreenCaptureKit ne livre rien tant que l'écran ne change +/// pas) ne contient parfois qu'UNE frame : `duration=0.067` pour un fichier de 87 s dont la +/// piste audio, elle, couvre tout. Borner le clip à la durée du flux le raccourcissait à une +/// frame, voire l'écartait sans bruit s'il commençait après. La sémantique de hold de +/// `advance_decoder_to` tient déjà la dernière frame jusqu'au bout de la fenêtre ; la borne +/// doit donc être celle du conteneur, qui couvre aussi l'audio. +pub(crate) fn cover_container_duration(stream_sec: Option, container_us: i64) -> Option { + let container_sec = (container_us > 0) + .then(|| container_us as f64 / 1_000_000.0) + .filter(|s| s.is_finite() && *s > 0.0); + match (stream_sec, container_sec) { + (Some(stream), Some(container)) => Some(stream.max(container)), + (stream, container) => stream.or(container), + } +} + /// Avance un décodeur vers `target_source_time`, sémantique de "hold" : à l'instant t on /// affiche la DERNIÈRE frame dont le pts est ≤ t, jamais une frame dont le pts est encore à /// venir. `timeline_offset_sec` remet les pts webcam dans le référentiel écran @@ -258,25 +278,46 @@ unsafe fn catch_up(dec: &mut Decoder, target: f64, clip_start: f64) -> Result<() } /// Normal recordings can have audio up to roughly half a second longer than -/// video. Hold the last frame only in that margin: an unbounded fallback would -/// hide a genuinely truncated camera recording. +/// video. Hold the last frame of a camera only in that margin: an unbounded +/// fallback would hide a genuinely truncated camera recording. A screen whose +/// container covers the clip start is held without bound (see `walk_timeline`). const CLIP_START_TAIL_TOLERANCE_SEC: f64 = 0.5; -/// Seek a clip start within the normal tail margin. A future frame across an -/// interior gap is invalid, but a request before the first frame of an offset -/// stream retains the existing first-frame hold. -unsafe fn seek_clip_start(decoder: &mut Decoder, target: f64) -> Result { - let frame = decoder.seek_to_or_last(target)?; - if frame.is_null() { - return Ok(false); +/// Whether the frame a clip-start seek landed on (`current`) can stand in at +/// `target`, or `None` when only the first frame can tell. `hold_tail_sec` is how +/// far past the last frame the hold may reach. +fn accepts_clip_start(current: f64, target: f64, hold_tail_sec: f64) -> Option { + if current <= target && target - current <= hold_tail_sec + 1e-6 { + return Some(true); } - let current = decoder.cur_time_sec(); if (target - current).abs() <= CLIP_START_TAIL_TOLERANCE_SEC + 1e-6 { - return Ok(true); + return Some(true); } if current <= target { + return Some(false); + } + None +} + +/// Seek a clip start within the tail margin. A sparse recording can have a +/// nearby preceding frame even when the next update is far ahead. Recover +/// that frame within the same margin, while still rejecting larger gaps and +/// retaining the first-frame hold for an offset stream. +unsafe fn seek_clip_start(decoder: &mut Decoder, target: f64, hold_tail_sec: f64) -> Result { + let frame = decoder.seek_to_or_last(target)?; + if frame.is_null() { return Ok(false); } + if let Some(accepted) = accepts_clip_start(decoder.cur_time_sec(), target, hold_tail_sec) { + return Ok(accepted); + } + // `seek_to` rounds forward. Try the bounded preceding window before deciding + // that the clip starts in a hole; do not show the future update early. + let from = (target - CLIP_START_TAIL_TOLERANCE_SEC).max(0.0); + let previous = decoder.seek_to(from)?; + if !previous.is_null() && decoder.cur_time_sec() <= target { + return advance_decoder_to(decoder, target, 0.0); + } // Only a distant future result needs this extra seek. Distinguish a late // stream origin from a missing interior interval using the actual first // frame, without changing decoder timestamps or the normal seek path. @@ -393,13 +434,14 @@ unsafe fn walk_timeline( clip_index + 1, ); } + // Only the screen bounds the clip. A camera that ends first holds its last frame + // (`advance_decoder_to`), as the preview does: the recorder stamps the camera with the + // HUD's wall-clock length, a few hundred ms short of the native screen file, and + // clamping to it cut that tail off every recorded take and shifted every later clip. let mut source_end_sec = clip.source_end_sec; if let Some(duration) = screen_available_duration { source_end_sec = source_end_sec.min(duration); } - if let Some(duration) = webcam_available_screen_end { - source_end_sec = source_end_sec.min(duration); - } if source_end_sec + 1e-6 < clip.source_end_sec { eprintln!( "[pipeline] warning: clip #{} raccourci de {:.3}s (fin demandée {:.3}s, fin disponible {:.3}s; screen=\"{}\", webcam=\"{}\")", @@ -428,13 +470,22 @@ unsafe fn walk_timeline( programme_sec, out_fps as f64, ); + // A turned clip is planned in what the viewer sees: its cursor track is the mirrored one. + let turned = clip_scene.as_ref().is_some_and(|s| s.screen_turned()); if clip_scene.is_some() { comp.set_scene(clip_scene); } // un seul seek keyframe, puis chaque décodeur avance selon son propre pts jusqu'aux // temps source demandés par les spans de vitesse. - if !seek_clip_start(sdec, clip.source_start_sec) + // La fin du clip est déjà bornée par la durée disponible : si le conteneur couvre le + // début du clip, la dernière frame du flux écran peut être tenue jusque-là. + let screen_hold_tail = if screen_available_duration.is_some_and(|d| d > clip.source_start_sec) { + f64::INFINITY + } else { + CLIP_START_TAIL_TOLERANCE_SEC + }; + if !seek_clip_start(sdec, clip.source_start_sec, screen_hold_tail) .with_context(|| format!("Cannot export clip {}: could not read the screen video.", clip_index + 1))? { bail!("Cannot export clip {}: the screen video has no readable frame at the clip's start.", clip_index + 1); @@ -449,23 +500,27 @@ unsafe fn walk_timeline( } else { format!("Cannot export clip {}: the screen video has no readable frame at the clip's start.", clip_index + 1) }; - if !seek_clip_start(wdec, webcam_start).with_context(|| webcam_error.clone())? { + let webcam_hold_tail = if has_camera { CLIP_START_TAIL_TOLERANCE_SEC } else { screen_hold_tail }; + if !seek_clip_start(wdec, webcam_start, webcam_hold_tail).with_context(|| webcam_error.clone())? { bail!(webcam_error); } if cursor_enabled { - if !cursor_tracks.contains_key(&clip.screen) { + // Keyed by file AND turn: the same file turned is another track. + let key = cursor_key(&clip.screen, turned); + if !cursor_tracks.contains_key(&key) { let path = format!("{}.cursor.json", clip.screen); if let Ok(raw) = CursorTrack::load(&path, 0.0, 24.0 * 3600.0) { - cursor_tracks.insert(clip.screen.clone(), raw.smoothed(cursor_smoothing)); + let raw = if turned { raw.turned() } else { raw }; + cursor_tracks.insert(key.clone(), raw.smoothed(cursor_smoothing)); } // absente/illisible → pas d'entrée : ce clip s'exporte sans curseur (visible, // pas masqué en un curseur fantôme d'un autre clip). } - if cursor_active_path.as_deref() != Some(clip.screen.as_str()) { - if let Some(track) = cursor_tracks.get(&clip.screen) { + if cursor_active_path.as_deref() != Some(key.as_str()) { + if let Some(track) = cursor_tracks.get(&key) { comp.set_cursor(track.clone()); - cursor_active_path = Some(clip.screen.clone()); + cursor_active_path = Some(key); } else { comp.clear_cursor(); comp.set_cursor_time(None); @@ -554,7 +609,51 @@ unsafe fn walk_timeline( #[cfg(test)] mod tests { - use super::{frame_step, FrameStep, NextFrameTime, RenderedFrame}; + use super::{ + accepts_clip_start, cover_container_duration, frame_step, FrameStep, NextFrameTime, + RenderedFrame, CLIP_START_TAIL_TOLERANCE_SEC, + }; + + /// Le fichier signalé, clip 5–15 s : la seule frame est à 0 s, le conteneur couvre le clip. + /// Elle doit être tenue, pas refusée comme une queue de flux tronquée. + #[test] + fn a_clip_after_the_only_frame_holds_it_when_the_container_covers_it() { + assert_eq!(accepts_clip_start(0.0, 5.0, f64::INFINITY), Some(true)); + } + + /// Sans conteneur pour la couvrir (webcam, durée inconnue), la queue reste bornée. + #[test] + fn a_bounded_hold_still_refuses_a_distant_last_frame() { + assert_eq!(accepts_clip_start(0.0, 5.0, CLIP_START_TAIL_TOLERANCE_SEC), Some(false)); + assert_eq!(accepts_clip_start(9.8, 10.2, CLIP_START_TAIL_TOLERANCE_SEC), Some(true)); + } + + /// Une frame FUTURE lointaine (trou intérieur ou origine tardive) ne profite pas du hold + /// illimité : c'est la première frame qui tranche. + #[test] + fn an_unbounded_hold_does_not_accept_a_distant_future_frame() { + assert_eq!(accepts_clip_start(20.0, 15.0, f64::INFINITY), None); + } + + /// Le fichier signalé : une frame (0,067 s) de vidéo, 87,38 s de conteneur. Le clip ne doit + /// être ni raccourci à une frame ni écarté. + #[test] + fn a_one_frame_video_stream_is_covered_by_the_container() { + assert_eq!(cover_container_duration(Some(40.0 / 600.0), 87_380_000), Some(87.38)); + } + + #[test] + fn a_normal_file_keeps_its_stream_duration() { + assert_eq!(cover_container_duration(Some(10.0), 10_000_000), Some(10.0)); + assert_eq!(cover_container_duration(Some(10.0), 9_990_000), Some(10.0)); + } + + #[test] + fn an_unknown_side_falls_back_to_the_other() { + assert_eq!(cover_container_duration(None, 5_000_000), Some(5.0)); + assert_eq!(cover_container_duration(Some(5.0), i64::MIN), Some(5.0)); + assert_eq!(cover_container_duration(None, i64::MIN), None); + } /// La cadence de lecture, en une phrase : à 24 fps, une seconde réelle doit adopter 24 /// frames et pas une de plus. Le bug d'origine (un pas fixe de 1/60 s, une frame par diff --git a/crates/compositor/tests/common/nv12.rs b/crates/compositor/tests/common/nv12.rs index 66353350c8..79b39f09d5 100644 --- a/crates/compositor/tests/common/nv12.rs +++ b/crates/compositor/tests/common/nv12.rs @@ -72,6 +72,12 @@ impl Nv12Frame { } } + /// Étiquette la frame en plage pleine (`AVCOL_RANGE_JPEG`), comme un décodeur `yuvj*`. + pub fn full_range(mut self) -> Nv12Frame { + self.frame.color_range = openscreen_compositor::ffi::AVColorRange::AVCOL_RANGE_JPEG; + self + } + pub fn as_ptr(&self) -> *const AVFrame { &*self.frame as *const AVFrame } diff --git a/crates/compositor/tests/export_timing.rs b/crates/compositor/tests/export_timing.rs index 0ed7f77b0d..15e796557d 100644 --- a/crates/compositor/tests/export_timing.rs +++ b/crates/compositor/tests/export_timing.rs @@ -25,6 +25,8 @@ //! -fps_mode vfr -c:v libopenh264 -g 60 -pix_fmt yuv420p video_gap.mp4 //! ffmpeg -i screen_colors.mp4 -c copy -output_ts_offset 1 offset_start.mp4 //! ffmpeg -i video_gap.mp4 -c copy -output_ts_offset 1 offset_gap.mp4 +//! ffmpeg -i screen_colors.mp4 -vf "select='not(mod(n,60))'" \ +//! -fps_mode vfr -c:v libopenh264 -g 60 -pix_fmt yuv420p sparse_colors.mp4 //! The gap fixture must retain timestamps: video frames stop at 0.983 s and //! resume at 3.000 s. An equivalent H.264 encoder is sufficient if libopenh264 //! is unavailable. @@ -141,6 +143,33 @@ fn gif_frame_palettes(bytes: &[u8]) -> Vec> { palettes } +/// Each image's delay in centiseconds, from the Graphics Control Extension +/// before it, walking the blocks as `gif_frame_palettes` does. +fn gif_frame_delays(bytes: &[u8]) -> Vec { + let skip_sub_blocks = |mut p: usize| { + while p < bytes.len() && bytes[p] != 0 { + p += 1 + bytes[p] as usize; + } + p + 1 + }; + let table_len = |packed: u8| if packed & 0x80 == 0 { 0 } else { 3 * (1usize << ((packed & 0x07) + 1)) }; + let mut delays = Vec::new(); + let mut p = 6 + 7 + table_len(bytes[10]); + while p < bytes.len() { + match bytes[p] { + 0x21 => { + if bytes[p + 1] == 0xF9 { + delays.push(u16::from_le_bytes([bytes[p + 4], bytes[p + 5]])); + } + p = skip_sub_blocks(p + 2); + } + 0x2C => p = skip_sub_blocks(p + 10 + table_len(bytes[p + 9]) + 1), + _ => break, + } + } + delays +} + /// Mean `R - B` across a palette. The fixture's first second is pure red /// (large positive) and its last is white (≈ 0), so this single number /// separates "covered the timeline" from "stuck on frame 0". @@ -212,6 +241,52 @@ fn mp4_export_frame_count_follows_output_fps() { assert_eq!(probed, 120, "muxed file disagrees with the reported count"); } +/// The last frame must survive in the file, not only in the walk. VideoToolbox returned it +/// without a duration, the MP4 edit list ended on its pts, and a player dropped it whenever that +/// pts fell on a whole millisecond: 30 fps exports of these two lengths played 276 of 277 and +/// 1176 of 1177 frames (v2.3.0-rc.2 macOS pass, D4). +#[test] +fn mp4_export_keeps_the_last_frame_in_the_file() { + let Some(dir) = media_dir() else { + eprintln!("skipped: set OPENSCREEN_TEST_MEDIA"); + return; + }; + let gpu = Gpu::create(false).expect("gpu"); + let params = ExportParams { + width: 640, + height: 360, + fps: Some(30), + codec: ExportCodec::H264, + bit_rate: None, + }; + for programme_sec in [9.229979, 39.229979] { + // Whole fixtures back to back, the remainder in a last clip. + let whole = (programme_sec / SOURCE_SEC).floor() as usize; + let mut clips: Vec = (0..whole).map(|_| whole_clip(&dir)).collect(); + let mut last = whole_clip(&dir); + last.source_end_sec = programme_sec - whole as f64 * SOURCE_SEC; + clips.push(last); + + let outputs = TestOutputDir::new("last-frame"); + let out = outputs.0.join("out.mp4"); + let comp = Compositor::new_sized(&gpu, params.width, params.height).expect("compositor"); + let stats = pipeline::run_composited_multi( + &clips, + &out.to_string_lossy(), + &gpu, + &comp, + &Cfg::c8(), + ¶ms, + &mut |_| {}, + ) + .expect("mp4 export"); + let expected = ((programme_sec - 0.001) * 30.0).ceil() as u64; + assert_eq!(stats.frames, expected, "walk at P = {programme_sec}"); + let probed = pipeline::probe_frame_count(&out.to_string_lossy()).expect("probe"); + assert_eq!(probed, expected, "frames a player decodes at P = {programme_sec}"); + } +} + /// A camera that ended before its requested offset must fail the whole export, /// even after a good clip. Otherwise an empty or incomplete MP4 replaces the destination. #[test] @@ -478,6 +553,98 @@ fn mp4_export_bounds_tail_recovery_and_reports_the_correct_source() { std::fs::remove_dir_all(out_dir).expect("remove test outputs"); } +/// A static screen may have a nearby preceding frame but no new frame for a second. +/// Starting a cut between those updates must hold the preceding image, then adopt +/// the next image when it is due, both in frame inspection and a real MP4 export. +#[test] +fn mp4_export_recovers_a_nearby_frame_before_a_sparse_clip_start() { + let Some(dir) = media_dir() else { + eprintln!("skipped: set OPENSCREEN_TEST_MEDIA"); + return; + }; + let sparse = dir.join("sparse_colors.mp4"); + assert!( + sparse.exists(), + "generate sparse_colors.mp4 using the fixture header recipe" + ); + let outputs = TestOutputDir::new("sparse-clip-start"); + let gpu = Gpu::create(false).expect("gpu"); + let comp = Compositor::new_sized(&gpu, 640, 360).expect("compositor"); + let params = ExportParams { + width: 640, + height: 360, + fps: Some(30), + codec: ExportCodec::H264, + bit_rate: None, + }; + for has_camera in [false, true] { + for after_valid_clip in [false, true] { + let mut clip = whole_clip(&dir); + clip.screen = sparse.to_string_lossy().into_owned(); + if !has_camera { + clip.webcam.clear(); + } + // Frames at 1.0 s (green) and 2.0 s (blue), with the cut at 1.25 s. + clip.source_start_sec = 1.25; + clip.source_end_sec = 2.5; + let mut clips = Vec::new(); + if after_valid_clip { + let first = ClipSource { + screen: clip.screen.clone(), + webcam: clip.webcam.clone(), + source_start_sec: 0.0, + source_end_sec: 0.5, + ..whole_clip(&dir) + }; + clips.push(first); + } + clips.push(clip); + let first_frame = if after_valid_clip { 15 } else { 0 }; + let frames = unsafe { + openscreen_compositor::timeline_walk::render_frames( + &clips, + &gpu, + &comp, + &Cfg::c8(), + 30, + &[first_frame, first_frame + 22, first_frame + 23], + ) + } + .expect("inspect a clip starting between sparse screen updates"); + assert_eq!(frames.len(), 3); + for (i, frame) in frames.iter().enumerate() { + let center = ((frame.height / 2 * frame.width + frame.width / 2) * 4) as usize; + let rgb = &frame.rgba[center..center + 3]; + let expected_channel = if i < 2 { 1 } else { 2 }; + assert!( + rgb[expected_channel] > rgb[0].saturating_add(40), + "frame {i}: {rgb:?}" + ); + assert!( + rgb[expected_channel] > rgb[3 - expected_channel].saturating_add(40), + "frame {i}: {rgb:?}" + ); + } + let out = outputs.0.join("out.mp4"); + let stats = pipeline::run_composited_multi( + &clips, + &out.to_string_lossy(), + &gpu, + &comp, + &Cfg::c8(), + ¶ms, + &mut |_| {}, + ) + .expect("export a clip starting between sparse screen updates"); + assert_eq!(stats.frames, first_frame + 38); + assert_eq!( + pipeline::probe_frame_count(&out.to_string_lossy()).expect("probe"), + stats.frames + ); + } + } +} + /// A seek inside a multi-second hole must not show a frame from the future. /// Exercise both source roles and preservation after an earlier valid clip. #[test] @@ -612,6 +779,65 @@ fn mp4_export_accepts_a_nonzero_first_video_timestamp() { } } +/// A recorded camera ends a few hundred ms before the screen (its length is the HUD's +/// wall clock, the screen file runs to the native stop). The export must hold the last +/// camera frame to the end of the clip, as the preview does, not cut the clip there: +/// otherwise each take loses its tail and every later clip starts early. +#[test] +fn mp4_export_holds_a_camera_that_ends_before_the_screen() { + let Some(dir) = media_dir() else { + eprintln!("skipped: set OPENSCREEN_TEST_MEDIA"); + return; + }; + let outputs = TestOutputDir::new("short-camera"); + let gpu = Gpu::create(false).expect("gpu"); + let comp = Compositor::new_sized(&gpu, 640, 360).expect("compositor"); + let params = ExportParams { + width: 640, + height: 360, + fps: Some(30), + codec: ExportCodec::H264, + bit_rate: None, + }; + // Camera started 0.5 s before the screen: it ends at 3.5 s of screen time. + let mut take = whole_clip(&dir); + take.webcam_offset_sec = -0.5; + let clips = [take, whole_clip(&dir)]; + let out = outputs.0.join("out.mp4"); + let stats = pipeline::run_composited_multi( + &clips, + &out.to_string_lossy(), + &gpu, + &comp, + &Cfg::c8(), + ¶ms, + &mut |_| {}, + ) + .expect("a short camera must not fail the export"); + assert_eq!(stats.frames, 240, "the take must keep its last 0.5 s"); + + // The second clip starts on the grid where the first one ends in the preview. + let frames = unsafe { + openscreen_compositor::timeline_walk::render_frames( + &clips, + &gpu, + &comp, + &Cfg::c8(), + 30, + &[119, 120], + ) + } + .expect("inspect the cut"); + let centre = |f: &openscreen_compositor::timeline_walk::RenderedFrame| { + let i = ((f.height / 2 * f.width + f.width / 2) * 4) as usize; + [f.rgba[i], f.rgba[i + 1], f.rgba[i + 2]] + }; + let [r, g, b] = centre(&frames[0]); + assert!(r > 200 && g > 200 && b > 200, "frame 119 is the take's white tail, got {r},{g},{b}"); + let [r, g, b] = centre(&frames[1]); + assert!(r > 200 && g < 60 && b < 60, "frame 120 opens the next clip in red, got {r},{g},{b}"); +} + /// The macOS frame probe must count decoded video rather than require MP4 /// frame-count metadata; Matroska recordings commonly omit that metadata. #[cfg(target_os = "macos")] @@ -728,18 +954,19 @@ fn mp4_export_validation_accepts_h264_and_hevc() { std::fs::remove_dir(out_dir).expect("remove output dir"); } -/// VideoToolbox can mux one tail sample with zero duration and mark it discard. -/// A positive composed-frame count must not let that file replace a good export. +/// A one-frame export is the last-frame case at its limit: VideoToolbox returned that frame +/// without a duration, the muxer marked it discard, and validation refused a file with nothing +/// to play. With every packet given its frame's duration, the frame is muxed, decodes, and +/// replaces the destination. (Refusing an undecodable file stays covered in `export_control`.) #[cfg(target_os = "macos")] #[test] -fn mp4_export_rejects_an_undecodable_single_tail_frame_without_publishing() { +fn mp4_export_publishes_a_single_tail_frame() { let Some(dir) = media_dir() else { eprintln!("skipped: set OPENSCREEN_TEST_MEDIA"); return; }; - let out_dir = dir.join(format!("undecodable_single_tail-{}", std::process::id())); - std::fs::create_dir_all(&out_dir).expect("output dir"); - let out = out_dir.join("out.mp4"); + let outputs = TestOutputDir::new("single-tail"); + let out = outputs.0.join("out.mp4"); let gpu = Gpu::create(false).expect("gpu"); let comp = Compositor::new_sized(&gpu, 640, 360).expect("compositor"); let params = ExportParams { @@ -752,46 +979,19 @@ fn mp4_export_rejects_an_undecodable_single_tail_frame_without_publishing() { let mut tail = whole_clip(&dir); tail.webcam.clear(); tail.source_start_sec = SOURCE_SEC - 0.01; - for existing_destination in [false, true] { - if existing_destination { - std::fs::write(&out, b"previous export").expect("seed destination"); - } - let result = pipeline::run_composited_multi( - &[ClipSource { - screen: tail.screen.clone(), - webcam: String::new(), - source_start_sec: tail.source_start_sec, - source_end_sec: tail.source_end_sec, - webcam_offset_sec: 0.0, - has_audio: false, - }], - &out.to_string_lossy(), - &gpu, - &comp, - &Cfg::c8(), - ¶ms, - &mut |_| {}, - ); - let error = result - .err() - .expect("undecodable output must not be published"); - assert!( - format!("{error:#}").contains("validating exported video"), - "{error:#}" - ); - if existing_destination { - assert_eq!(std::fs::read(&out).unwrap(), b"previous export"); - std::fs::remove_file(&out).expect("remove destination"); - } else { - assert!(!out.exists()); - } - assert_eq!( - std::fs::read_dir(&out_dir).unwrap().count(), - 0, - "partial output left behind" - ); - } - std::fs::remove_dir(out_dir).expect("remove output dir"); + std::fs::write(&out, b"previous export").expect("seed destination"); + let stats = pipeline::run_composited_multi( + &[tail], + &out.to_string_lossy(), + &gpu, + &comp, + &Cfg::c8(), + ¶ms, + &mut |_| {}, + ) + .expect("a single tail frame must export"); + assert_eq!(stats.frames, 1); + assert_eq!(pipeline::probe_frame_count(&out.to_string_lossy()).expect("probe"), 1); } /// Cancelling an MP4 mid-render stops the walk at the next frame, reports `ExportCancelled`, @@ -909,6 +1109,41 @@ fn gif_export_cancelled_mid_render_publishes_nothing() { /// request — it just takes them all from the first 0.8 s, so every frame is red /// and the GIF plays 5x slow. Comparing the first and last frame palettes /// catches exactly that. +/// v2.3.0-rc.4 D5: a real MP4 and GIF export over a 600 or 755 file keeps that mode. +#[cfg(unix)] +#[test] +fn mp4_and_gif_exports_keep_the_mode_of_the_replaced_file() { + use std::os::unix::fs::PermissionsExt; + let Some(dir) = media_dir() else { + eprintln!("skipped: set OPENSCREEN_TEST_MEDIA"); + return; + }; + let out_dir = TestOutputDir::new("export-mode"); + let gpu = Gpu::create(false).expect("gpu"); + let comp = Compositor::new_sized(&gpu, 320, 180).expect("compositor"); + let mut clip = whole_clip(&dir); + clip.source_end_sec = 0.5; + let clips = [clip]; + for mode in [0o600, 0o755] { + let mp4 = out_dir.0.join(format!("out-{mode:o}.mp4")); + let gif = out_dir.0.join(format!("out-{mode:o}.gif")); + for out in [&mp4, &gif] { + std::fs::write(out, b"previous export").unwrap(); + std::fs::set_permissions(out, std::fs::Permissions::from_mode(mode)).unwrap(); + } + let params = ExportParams { width: 320, height: 180, fps: Some(30), codec: ExportCodec::H264, bit_rate: None }; + pipeline::run_composited_multi(&clips, &mp4.to_string_lossy(), &gpu, &comp, &Cfg::c8(), ¶ms, &mut |_| {}) + .expect("mp4 export"); + let params = GifExportParams { width: Some(320), height: Some(180), fps: Some(12), loop_count: None, dither: false }; + gif_export::export_gif(&clips, &gif, &gpu, &comp, &Cfg::c8(), ¶ms, &mut |_| {}).expect("gif export"); + for out in [&mp4, &gif] { + assert_ne!(std::fs::read(out).unwrap(), b"previous export"); + let kept = std::fs::metadata(out).unwrap().permissions().mode() & 0o777; + assert_eq!(kept, mode, "{} replaced a {mode:o} file, got {kept:o}", out.display()); + } + } +} + #[test] fn gif_export_spans_the_whole_timeline() { let Some(dir) = media_dir() else { @@ -945,7 +1180,40 @@ fn gif_export_spans_the_whole_timeline() { let bytes = std::fs::read(&out).expect("read gif"); let palettes = gif_frame_palettes(&bytes); - assert_eq!(palettes.len(), 48, "GIF carries {} frames", palettes.len()); + // Each second is one still colour: a frame that changed nothing lengthens + // the image before it instead of being written, so there are fewer images + // than frames, and their delays still add up to the 4 s. + assert!( + (4..48).contains(&palettes.len()), + "a still second must be one image, not twelve: {} images", + palettes.len() + ); + let delays = gif_frame_delays(&bytes); + assert_eq!(delays.len(), palettes.len(), "one delay per image"); + let total: u32 = delays.iter().map(|&d| d as u32).sum(); + assert!((399..=401).contains(&total), "the delays add up to {total} cs, not 400"); + + // The cut to green, 1 s in, gets a palette of its own at once: mapped to the + // red second's palette it came out brown, for the 1.5 s until the next one. + let starts: Vec = delays + .iter() + .scan(0u32, |t, &d| { + let start = *t; + *t += d as u32; + Some(start) + }) + .collect(); + let green = starts + .iter() + .position(|&t| (99..=101).contains(&t)) + .expect("an image starting at the cut to green, 1 s in"); + let greenness = palettes[green] + .iter() + .map(|c| c[1] as f64 - c[0] as f64) + .sum::() + / palettes[green].len() as f64; + // lavfi's green is (0, 128, 0), against a red palette's strongly negative mean. + assert!(greenness > 10.0, "the green second's palette is not green ({greenness:.1})"); let first = redness(&palettes[0]); let last = redness(&palettes[palettes.len() - 1]); diff --git a/crates/compositor/tests/full_range_render.rs b/crates/compositor/tests/full_range_render.rs new file mode 100644 index 0000000000..174aa2af5a --- /dev/null +++ b/crates/compositor/tests/full_range_render.rs @@ -0,0 +1,58 @@ +//! #1531 : une vidéo en plage pleine passe par le vrai shader D3D11 (`sample_yuv`) sans dériver. +//! +//! Frame NV12 synthétique et unie, étiquetée pleine : (128, 128, 128) doit sortir gris neutre +//! à 128. Une chroma ramenée en limitée hors du centre (128 → 128,44) teintait ce gris. +//! Sans adaptateur matériel, le test se saute. + +// Windows seulement : le chemin D3D11 de ce harnais n'existe que là. +#![cfg(windows)] + +use openscreen_compositor::compositor::Compositor; +use openscreen_compositor::config::Cfg; +use openscreen_compositor::frame_geometry::live_params_from_scene; +use openscreen_compositor::scene::Scene; + +mod common; +use common::{gpu, Nv12Frame}; + +const OUT: (u32, u32) = (320, 180); + +#[test] +fn full_range_grey_stays_neutral_grey() { + let Some(gpu) = gpu() else { return }; + let comp = Compositor::new_sized(&gpu, OUT.0, OUT.1).expect("compositor"); + let screen = Nv12Frame::new(&gpu, (64, 64), |_, _| 128, |_, _| [128, 128]).full_range(); + let scene = Scene::from_json( + r##"{"clips":[{"screenPath":"/s.mp4","webcamPath":"","sourceStartSec":0,"sourceEndSec":10,"webcamOffsetSec":0,"hasAudio":false}], + "layout":{"preset":"no-webcam","webcamSize":1,"webcamShape":"rounded","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false}, + "effects":{"padding":0,"blur":false,"shadow":0,"roundnessFrac":0,"motionBlur":0}, + "background":{"kind":"color","color":"#000000"}, + "zoomRegions":[],"annotations":[], + "cursor":{"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0,"clipToBounds":false,"theme":"none"}, + "cropByClip":[null], + "output":{"width":320,"height":180,"fps":30}}"##, + ) + .expect("scène valide"); + let mut live = live_params_from_scene(&scene); + live.has_webcam = false; + comp.set_live_params(live); + comp.set_has_webcam(false); + comp.set_scene(Some(scene)); + comp.clear_cursor(); + let mut cfg = Cfg::c8(); + cfg.bg_blur = 0.0; + cfg.zoom = false; + cfg.layout_anim = false; + cfg.cursor = false; + cfg.mblur_n = 1; + cfg.shadow = false; + let (w, _, rgba) = unsafe { + comp.compose_frame(screen.as_ptr(), screen.as_ptr(), 0.0, &cfg).expect("compose_frame"); + comp.readback_direct().expect("readback") + }; + let i = (((OUT.1 / 2) * w + OUT.0 / 2) * 4) as usize; + let [r, g, b] = [rgba[i], rgba[i + 1], rgba[i + 2]]; + println!("centre : ({r}, {g}, {b})"); + assert!(r == g && g == b, "gris teinté : ({r}, {g}, {b})"); + assert!(r.abs_diff(128) <= 1, "gris {r} au lieu de 128"); +} diff --git a/crates/compositor/tests/hevc10_decode.rs b/crates/compositor/tests/hevc10_decode.rs new file mode 100644 index 0000000000..56ca01f935 --- /dev/null +++ b/crates/compositor/tests/hevc10_decode.rs @@ -0,0 +1,128 @@ +//! Une vidéo HEVC Main 10 (iPhone, #1390) se rend avec ses couleurs, pas en bandes. +//! +//! VideoToolbox sort un `CVPixelBuffer` P010 (16 bits par échantillon) pour un flux 10 bits. +//! Lu comme du NV12 8 bits, chaque échantillon devient deux texels : l'image se couvre de +//! bandes jaunes et vertes. La mire est coupée en deux, rouge à gauche, bleu à droite ; +//! chaque moitié doit garder sa couleur. +//! +//! La CI ne le joue pas : il faut le GPU et un encodeur HEVC 10 bits. +//! +//! ```sh +//! ffmpeg -f lavfi -i "color=c=red:size=640x360:rate=30:duration=1,drawbox=x=320:y=0:w=320:h=360:color=blue:t=fill" \ +//! -vf format=p010le -c:v hevc_videotoolbox -profile:v main10 -tag:v hvc1 split10.mov +//! OPENSCREEN_HEVC10_SOURCE=$PWD/split10.mov cargo test -p openscreen-compositor --test hevc10_decode +//! ``` +#![cfg(target_os = "macos")] + +use openscreen_compositor::compositor::{live_params_from_scene, Compositor}; +use openscreen_compositor::config; +use openscreen_compositor::d3d::Gpu; +use openscreen_compositor::pipeline::ClipSource; +use openscreen_compositor::scene::Scene; +use openscreen_compositor::timeline_walk::{render_frames, RenderedFrame}; + +fn scene_json(source: &str) -> String { + let source = serde_json::to_string(source).expect("chemin JSON"); + format!( + r##"{{ + "clips": [{{"screenPath":{source},"webcamPath":"","sourceStartSec":0,"sourceEndSec":1,"webcamOffsetSec":0,"hasAudio":false}}], + "layout": {{"preset":"no-webcam","webcamSize":1.0,"webcamShape":"rounded","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false}}, + "effects": {{"padding":0,"blur":false,"shadow":0.0,"roundnessFrac":0,"motionBlur":0.0}}, + "background": {{"kind":"color","color":"#303030"}}, + "zoomRegions": [], + "cursor": {{"show":false,"size":1,"smoothing":0,"motionBlur":0,"clickBounce":0,"clipToBounds":false,"theme":"default"}}, + "cropByClip": [null], + "output": {{"width":640,"height":360,"fps":null}} + }}"## + ) +} + +/// La frame 10 de `source`, rendue seule sur un fond uni, sans effet. +fn render_frame_10(source: &str) -> RenderedFrame { + let source = source.to_owned(); + let gpu = Gpu::create_auto(false).expect("device"); + let clips = [ClipSource { + screen: source.clone(), + webcam: String::new(), + source_start_sec: 0.0, + source_end_sec: 1.0, + webcam_offset_sec: 0.0, + has_audio: false, + }]; + let mut cfg = config::all().pop().expect("C8"); + cfg.zoom = false; + cfg.layout_anim = false; + cfg.mblur_n = 1; + cfg.bg_blur = 0.0; + cfg.cursor = false; + let scene = Scene::from_json(&scene_json(&source)).expect("scène"); + let comp = Compositor::new_sized(&gpu, 640, 360).expect("compositeur"); + comp.set_live_params(live_params_from_scene(&scene)); + comp.set_scene(Some(scene)); + let mut frames = + unsafe { render_frames(&clips, &gpu, &comp, &cfg, 30, &[10]) }.expect("render_frames"); + frames.swap_remove(0) +} + +/// Chaque moitié de la mire garde `left` / `right`, à `tol` près par canal. +fn assert_halves(frame: &RenderedFrame, left: [u8; 3], right: [u8; 3], tol: u8) { + let px = |x: u32, y: u32| { + let i = ((y * frame.width + x) * 4) as usize; + [frame.rgba[i], frame.rgba[i + 1], frame.rgba[i + 2]] + }; + let (w, h) = (frame.width, frame.height); + for (x, want) in [(w / 4, left), (3 * w / 4, right)] { + for y in [h / 4, h / 2, 3 * h / 4] { + let got = px(x, y); + let close = got.iter().zip(want).all(|(&g, w)| g.abs_diff(w) <= tol); + assert!(close, "({x},{y}) : {got:?}, attendu ≈ {want:?}"); + } + } +} + +#[test] +fn a_10_bit_hevc_frame_keeps_its_colours() { + let Some(source) = std::env::var_os("OPENSCREEN_HEVC10_SOURCE") else { + eprintln!("OPENSCREEN_HEVC10_SOURCE absent — test sauté"); + return; + }; + let frame = render_frame_10(&source.to_string_lossy()); + assert_halves(&frame, [255, 0, 0], [0, 0, 255], 40); +} + +/// Une frame pleine plage (`420f`, `xf20`) n'est pas relue comme limitée : sans le +/// correctif, le gris sombre tombe à ~19 et le gris clair monte à ~242. +/// +/// ```sh +/// ffmpeg -f lavfi -i "color=c=0x202020:size=640x360:rate=30:duration=1,drawbox=x=320:y=0:w=320:h=360:color=0xe0e0e0:t=fill" \ +/// -vf scale=out_range=full,format=p010le -color_range pc -c:v hevc_videotoolbox -profile:v main10 -tag:v hvc1 grey10full.mov +/// OPENSCREEN_FULL_RANGE_SOURCE=$PWD/grey10full.mov cargo test -p openscreen-compositor --test hevc10_decode +/// ``` +#[test] +fn a_full_range_frame_keeps_its_levels() { + let Some(source) = std::env::var_os("OPENSCREEN_FULL_RANGE_SOURCE") else { + eprintln!("OPENSCREEN_FULL_RANGE_SOURCE absent — test sauté"); + return; + }; + let frame = render_frame_10(&source.to_string_lossy()); + assert_halves(&frame, [0x20; 3], [0xe0; 3], 6); +} + +/// Une frame BT.601 n'est pas relue en BT.709 : la matrice vient de `AVFrame::colorspace` +/// (le VUI du flux). Sans le correctif, le vert pur tombe à ~216. +/// +/// ```sh +/// ffmpeg -f lavfi -i "color=c=0x00ff00:size=640x360:rate=30:duration=1,drawbox=x=320:y=0:w=320:h=360:color=0x0000ff:t=fill" \ +/// -vf scale=out_color_matrix=bt601,format=nv12 -colorspace smpte170m -color_primaries smpte170m -color_trc smpte170m \ +/// -c:v h264_videotoolbox split601.mov +/// OPENSCREEN_BT601_SOURCE=$PWD/split601.mov cargo test -p openscreen-compositor --test hevc10_decode +/// ``` +#[test] +fn a_bt601_frame_keeps_its_hues() { + let Some(source) = std::env::var_os("OPENSCREEN_BT601_SOURCE") else { + eprintln!("OPENSCREEN_BT601_SOURCE absent — test sauté"); + return; + }; + let frame = render_frame_10(&source.to_string_lossy()); + assert_halves(&frame, [0, 255, 0], [0, 0, 255], 12); +} diff --git a/electron-builder.json5 b/electron-builder.json5 index 8b11d8cc77..12c369cc70 100644 --- a/electron-builder.json5 +++ b/electron-builder.json5 @@ -47,6 +47,8 @@ // (unknown scene fields are #[serde(default)]) rather than erroring. See the script header // and technical-documentation/engineering/build-and-packaging.md. "beforePack": "scripts/before-pack.cjs", + // Writes app-update.yml into the macOS bundle, which `--mac --dir` never gets (#1431). + "afterPack": "scripts/after-pack.cjs", // Standard AppImage updates need .upd_info and a .zsync alongside electron-updater's blockmap. "artifactBuildCompleted": "scripts/appimage-updates.cjs", "afterAllArtifactBuild": "scripts/appimage-updates.cjs", @@ -74,6 +76,8 @@ "it", "ja", "ko", + "nl", + "pl", "pt-BR", "pt_BR", "ru", @@ -476,6 +480,8 @@ "it", "ja-JP", "ko-KR", + "nl", + "pl", "pt-BR", "ru", "tr", diff --git a/electron/ai-edition/chat-execution.test.ts b/electron/ai-edition/chat-execution.test.ts new file mode 100644 index 0000000000..07844d700e --- /dev/null +++ b/electron/ai-edition/chat-execution.test.ts @@ -0,0 +1,165 @@ +import { once } from "node:events"; +import type { StructuredToolInterface } from "@langchain/core/tools"; +import { describe, expect, it, vi } from "vitest"; +import { createEmptyDocument } from "../../src/lib/ai-edition/schema"; +import { + createExecutionTracker, + executionSummary, + executionText, + toolExecutionStep, +} from "./chat-execution"; +import { buildTools } from "./deep-agent/service"; + +describe("execution metadata", () => { + it("keeps reasoning and repeated/concurrent invocation identities in observed order", () => { + const tracker = createExecutionTracker(vi.fn()); + tracker.thinking("**Read** the transcript"); + const first = toolExecutionStep("getTranscript", {}); + const second = toolExecutionStep("getTranscript", {}); + tracker.tool(first); + tracker.tool(second); + tracker.tool({ ...second, status: "failed", summary: "No transcript" }); + tracker.tool({ ...first, status: "succeeded" }); + tracker.thinking("Try another source"); + const steps = tracker.finish(false); + expect(steps.map((step) => step.kind)).toEqual(["reasoning", "tool", "tool", "reasoning"]); + expect(steps[1]).toMatchObject({ id: first.id, status: "succeeded" }); + expect(steps[2]).toMatchObject({ id: second.id, status: "failed" }); + expect(first.id).not.toBe(second.id); + }); + it("keeps sensitive arbitrary strings out of argument and result metadata", () => { + const step = toolExecutionStep("addAnnotation", { + apiKey: "private", + password: 12345, + text: "private", + assetId: "asset_1", + startSec: 2, + words: ["private"], + }); + expect(step.arguments).toBe("assetId: asset_1, startSec: 2, words: 1"); + expect(executionText("Bearer private apiKey=private sk-abcdefghijklmnop")).not.toContain( + "private", + ); + expect( + executionSummary({ + ok: true, + resultJson: JSON.stringify({ raw: "private", words: ["private"], count: 1 }), + }), + ).toBe("words: 1, count: 1"); + expect(executionSummary({ ok: false, resultJson: '{"error":"No transcript"}' })).toBe( + "No transcript", + ); + expect(executionText("x".repeat(2000))).toHaveLength(1000); + }); + it("uses the real executor verdict and pairs each repeated call by id", async () => { + const events = vi.fn(); + const holder = { current: createEmptyDocument({ projectId: "steps", title: "Test" }) }; + const tools = buildTools( + holder, + { + executionStep: events, + toolStart: vi.fn(), + toolEnd: vi.fn(), + text: vi.fn(), + thinking: vi.fn(), + error: vi.fn(), + }, + true, + ); + const tool = tools.find((entry) => entry.name === "getTranscript")! as StructuredToolInterface; + await tool.invoke({ assetId: "missing" }); + await tool.invoke({ assetId: "missing" }); + const steps = events.mock.calls.map(([step]) => step); + expect(steps).toHaveLength(4); + expect(steps[1]).toMatchObject({ + id: steps[0].id, + status: "failed", + summary: expect.stringMatching(/asset|transcript/i), + durationMs: expect.any(Number), + }); + expect(steps[3]).toMatchObject({ id: steps[2].id, status: "failed" }); + expect(steps[0].id).not.toBe(steps[2].id); + }); +}); + +it("reports repeated real SDK calls and schema/unknown-tool failures once per invocation", async () => { + const { createServer } = await import("node:http"); + + const { invokeOpenScreenAgent } = await import("./deep-agent/service"); + const steps = new Map< + string, + import("../../src/lib/ai-edition/chatExecution").ToolExecutionStep + >(); + let requests = 0; + const server = createServer(async (req, res) => { + for await (const _ of req) { + /* consume local fixture request */ + } + res.writeHead(200, { "Content-Type": "text/event-stream" }); + const first = ++requests === 1; + const delta = first + ? { + role: "assistant", + tool_calls: [ + "getCurrentDocument", + "getCurrentDocument", + "getTranscript", + "nonexistentTool", + ].map((name, index) => ({ + index, + id: `call_${index}`, + type: "function", + function: { + name, + arguments: JSON.stringify(name === "getTranscript" ? { assetId: 123 } : {}), + }, + })), + } + : { role: "assistant", content: "Done" }; + for (const [value, finish] of [ + [delta, null], + [{}, first ? "tool_calls" : "stop"], + ]) + res.write( + `data: ${JSON.stringify({ id: "chat_fixture", object: "chat.completion.chunk", created: 1, model: "fixture", choices: [{ index: 0, delta: value, finish_reason: finish }] })}\n\n`, + ); + res.end("data: [DONE]\n\n"); + }); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("Missing local port"); + try { + const result = await invokeOpenScreenAgent({ + document: createEmptyDocument({ projectId: "sdk_steps", title: "Test" }), + model: { + provider: "openai-compatible", + model: "fixture", + apiKey: "local-only", + baseUrl: `http://127.0.0.1:${address.port}/v1`, + maxRetries: 0, + }, + history: [], + userMessage: "Inspect", + sink: { + executionStep: (step) => steps.set(step.id, step), + text: vi.fn(), + thinking: vi.fn(), + toolStart: vi.fn(), + toolEnd: vi.fn(), + error: vi.fn(), + }, + }); + expect(result.text).toBe("Done"); + expect([...steps.values()].map((step) => [step.name, step.status])).toEqual([ + ["getCurrentDocument", "succeeded"], + ["getCurrentDocument", "succeeded"], + ["getTranscript", "failed"], + ["nonexistentTool", "failed"], + ]); + expect([...steps.values()][2].summary).toMatch(/schema|input|assetId/i); + } finally { + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + } +}); diff --git a/electron/ai-edition/chat-execution.ts b/electron/ai-edition/chat-execution.ts new file mode 100644 index 0000000000..0d6fe73934 --- /dev/null +++ b/electron/ai-edition/chat-execution.ts @@ -0,0 +1,150 @@ +import { randomUUID } from "node:crypto"; +import { + type ChatExecutionStep, + type ToolExecutionStep, + updateExecutionStep, +} from "../../src/lib/ai-edition/chatExecution"; + +/** Keep display metadata bounded; raw provider payloads and tool results never enter the sidecar. */ +export function executionText(value: string, limit = 1000): string { + return value + .replace(/\bBearer\s+\S+/gi, "Bearer [redacted]") + .replace(/\b(?:sk-|gh[pousr]_)[A-Za-z0-9_-]{8,}/g, "[redacted]") + .replace( + /\b(?:api[_-]?key|authorization|password|secret|token)\s*[=:]\s*[^\s,;]+/gi, + "[redacted]", + ) + .slice(0, limit); +} + +export function toolExecutionStep(name: string, args: unknown): ToolExecutionStep { + const summaries: string[] = []; + if (args && typeof args === "object") { + // Describe location and scale, not arbitrary text, headers, paths or credentials. + for (const [key, value] of Object.entries(args)) { + if (/api.?key|authorization|password|secret|token/i.test(key)) continue; + if (Array.isArray(value)) summaries.push(`${key}: ${value.length}`); + else if (typeof value === "number" || typeof value === "boolean") + summaries.push(`${key}: ${value}`); + else if (["assetId", "clipId", "id"].includes(key) && typeof value === "string") + summaries.push(`${key}: ${value}`); + } + } + return { + id: randomUUID(), + kind: "tool", + name: executionText(name, 100), + arguments: executionText(summaries.join(", ")), + status: "running", + startedAt: Date.now(), + }; +} + +export function createExecutionTracker(emit: (step: ChatExecutionStep) => void) { + let steps: ChatExecutionStep[] = []; + let reasoningId: string | undefined; + const update = (step: ChatExecutionStep) => { + steps = updateExecutionStep(steps, step); + emit(step); + }; + return { + tool(step: ToolExecutionStep) { + reasoningId = undefined; + update(step); + }, + thinking(delta: string) { + const previous = steps.find((step) => step.id === reasoningId); + const text = previous?.kind === "reasoning" ? previous.text : ""; + const step: ChatExecutionStep = { + id: reasoningId ?? randomUUID(), + kind: "reasoning", + startedAt: previous?.startedAt ?? Date.now(), + text: executionText(text + delta, 32768), + }; + reasoningId = step.id; + update(step); + }, + finish(stopped: boolean) { + for (const step of steps) { + if (step.kind === "tool" && step.status === "running") + update({ + ...step, + status: stopped ? "stopped" : "failed", + durationMs: Math.max(0, Date.now() - step.startedAt), + }); + } + return steps; + }, + snapshot: () => steps, + }; +} + +export function executionSummary(result: { + ok: boolean; + summary?: string; + resultJson: string; +}): string { + if (result.summary) return executionText(result.summary); + try { + const data: unknown = JSON.parse(result.resultJson); + if (data && typeof data === "object") { + if (!result.ok && "error" in data && typeof data.error === "string") + return executionText(data.error); + const counts = Object.entries(data) + .filter( + ([key, value]) => + /^[a-zA-Z][a-zA-Z0-9]*$/.test(key) && + !/api.?key|authorization|password|secret|token/i.test(key) && + (Array.isArray(value) || typeof value === "number" || typeof value === "boolean"), + ) + .slice(0, 8) + .map(([key, value]) => `${key}: ${Array.isArray(value) ? value.length : value}`); + return executionText(counts.join(", ")); + } + } catch { + /* Raw tool results are intentionally never displayed or persisted. */ + } + return ""; +} + +/** Reconcile announced calls with executor verdicts, including calls rejected before execution. */ +export function createToolExecutionObserver(emit: (step: ToolExecutionStep) => void) { + const calls = new Map(); + const begin = (callId: string, name: string, args: unknown) => { + const existing = calls.get(callId); + if (existing) return existing; + const step = toolExecutionStep(name, args); + calls.set(callId, step); + emit(step); + return step; + }; + return { + reset() { + calls.clear(); + }, + begin, + publish(step: ToolExecutionStep) { + const previous = calls.get(step.id) ?? begin(step.id, step.name, {}); + const next = { + ...step, + id: previous.id, + startedAt: previous.startedAt, + arguments: step.arguments || previous.arguments, + }; + calls.set(step.id, next); + emit(next); + }, + result(callId: string, content: string) { + const previous = calls.get(callId); + if (!previous || previous.status !== "running") return; + const step: ToolExecutionStep = { + ...previous, + status: "failed", + summary: executionText(content), + durationMs: Math.max(0, Date.now() - previous.startedAt), + }; + calls.set(callId, step); + emit(step); + }, + }; +} diff --git a/electron/ai-edition/chat-persistence.test.ts b/electron/ai-edition/chat-persistence.test.ts index 782ed579d2..5053884ae9 100644 --- a/electron/ai-edition/chat-persistence.test.ts +++ b/electron/ai-edition/chat-persistence.test.ts @@ -3,6 +3,8 @@ import { tmpdir } from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { createEmptyDocument } from "../../src/lib/ai-edition/schema"; +import { ChatPersistence } from "./chat-persistence"; +import { createTurnUsageAccumulator } from "./deep-agent/usage"; vi.mock("./deep-agent/service", () => ({ invokeOpenScreenAgent: vi.fn() })); vi.mock("./deep-agent/chat-model", () => ({ @@ -58,6 +60,7 @@ beforeEach(() => { text: "reply", document: args.document, mutated: false, + usage: createTurnUsageAccumulator().snapshot(), })); }); @@ -67,6 +70,244 @@ afterEach(async () => { }); describe("local Agent chat history", () => { + const reportedUsage = { + inputTokens: 31, + outputTokens: 7, + totalTokens: 38, + modelCallCount: 2, + countedCallCount: 2, + status: "complete" as const, + cachedInputTokens: 9, + reasoningOutputTokens: 3, + }; + + it.each([true, false])("restores usage on the owning message (success=%s)", async (success) => { + invokeMock.mockImplementation(async (args) => ({ + text: success ? "reply" : "", + reason: success ? undefined : "provider failed", + document: args.document, + mutated: false, + usage: { ...reportedUsage, rawProvider: { apiKey: key } }, + })); + const session = createSession("proj_usage"); + const result = await runChat("proj_usage", session.id, "question", config()); + if (!success) expect(result.userMessage).toMatchObject({ role: "user", usage: reportedUsage }); + await restart(); + const messages = selectSession("proj_usage", session.id)?.messages ?? []; + expect(messages).toHaveLength(success ? 2 : 1); + expect(messages.at(-1)?.usage).toEqual(reportedUsage); + expect(messages.at(-1)?.durationMs).toEqual(expect.any(Number)); + if (success) expect(messages[0]?.usage).toBeUndefined(); + expect(readFileSync(path.join(root, "proj_usage.chat.json"), "utf8")).not.toContain(key); + }); + + it.each([ + true, + false, + ])("binds delayed usage to the original session (success=%s)", async (success) => { + let reply!: (value: Awaited>) => void; + invokeMock.mockImplementation(() => new Promise((resolve) => (reply = resolve))); + const first = createSession("proj_binding", "A"); + const second = createSession("proj_binding", "B"); + const turn = runChat("proj_binding", first.id, "question", config()); + await vi.waitFor(() => expect(invokeMock).toHaveBeenCalled()); + selectSession("proj_binding", second.id); + reply({ + text: success ? "reply" : "", + document: createEmptyDocument({ title: "x", projectId: "proj_binding" }), + mutated: false, + usage: reportedUsage, + }); + await turn; + await restart(); + expect(selectSession("proj_binding", first.id)?.messages.at(-1)?.usage).toEqual(reportedUsage); + expect(selectSession("proj_binding", second.id)?.messages).toEqual([]); + }); + + it.each([ + ["session", true], + ["session", false], + ["project", true], + ["project", false], + ["rewind", true], + ["rewind", false], + ])("discards delayed results after %s removal (success=%s)", async (boundary, success) => { + const projectId = "proj_boundary"; + const session = createSession(projectId); + const document = createEmptyDocument({ title: "x", projectId }); + await runChat(projectId, session.id, "earlier", config(), document); + const earlier = selectSession(projectId, session.id)!.messages[0]; + let reply!: (value: Awaited>) => void; + invokeMock.mockImplementation(() => new Promise((resolve) => (reply = resolve))); + const turn = runChat(projectId, session.id, "removed question", config(), document); + await vi.waitFor(() => expect(invokeMock).toHaveBeenCalledTimes(2)); + if (boundary === "session") deleteSession(projectId, session.id); + if (boundary === "project") { + await deleteProjectChat(projectId); + createSession(projectId, "replacement"); + } + if (boundary === "rewind") + expect(rewindToMessage(projectId, session.id, earlier.id).success).toBe(true); + reply({ text: success ? "late reply" : "", document, mutated: false, usage: reportedUsage }); + expect(await turn).toMatchObject({ success: false }); + await restart(); + const allMessages = listSessions(projectId).flatMap( + (entry) => selectSession(projectId, entry.id)?.messages ?? [], + ); + expect( + allMessages.some( + (entry) => entry.content === "late reply" || entry.content === "removed question", + ), + ).toBe(false); + }); + + it.each([ + true, + false, + ])("discards a pending turn rewound to its own user message (success=%s)", async (success) => { + const projectId = "proj_own_rewind"; + const session = createSession(projectId); + const document = createEmptyDocument({ title: "before", projectId }); + let reply!: (value: Awaited>) => void; + invokeMock.mockImplementation(() => new Promise((resolve) => (reply = resolve))); + const turn = runChat(projectId, session.id, "pending question", config(), document); + await vi.waitFor(() => expect(invokeMock).toHaveBeenCalled()); + const user = selectSession(projectId, session.id)!.messages[0]; + expect(rewindToMessage(projectId, session.id, user.id).success).toBe(true); + reply({ + text: success ? "stale reply" : "", + document: createEmptyDocument({ title: "stale edit", projectId }), + mutated: true, + usage: reportedUsage, + }); + const result = await turn; + expect(result.success).toBe(false); + expect(result.document).toBeUndefined(); + expect(result.assistantMessage).toBeUndefined(); + expect(result.userMessage).toBeUndefined(); + expect(selectSession(projectId, session.id)?.messages).toEqual([]); + await restart(); + const restored = selectSession(projectId, session.id)?.messages; + expect(restored).toEqual([]); + }); + + it("uses the actual session created for a stale id", async () => { + await runChat("proj_stale", "stale-id", "question", config()); + await restart(); + const session = listSessions("proj_stale")[0]; + expect(selectSession("proj_stale", session.id)?.messages[1]?.usage?.status).toBe("unavailable"); + }); + + it("retains legacy messages and drops only invalid usage", async () => { + const session = createSession("proj_corrupt"); + await runChat("proj_corrupt", session.id, "question", config()); + await flushChatPersistence(); + const file = path.join(root, "proj_corrupt.chat.json"); + const saved = JSON.parse(readFileSync(file, "utf8")); + delete saved.sessions[0].messages[0].usage; + saved.sessions[0].messages[1].usage = { ...reportedUsage, countedCallCount: 3 }; + writeFileSync(file, JSON.stringify(saved)); + await restart(); + const messages = selectSession("proj_corrupt", session.id)?.messages; + expect(messages?.map((entry) => entry.content)).toEqual(["question", "reply"]); + expect(messages?.every((entry) => entry.usage === undefined)).toBe(true); + }); + + it.each([ + { inputTokens: -1 }, + { outputTokens: 1.5 }, + { totalTokens: Number.MAX_SAFE_INTEGER + 1 }, + { countedCallCount: 3 }, + { modelCallCount: 0, countedCallCount: 0 }, + { totalTokens: 39 }, + { cachedInputTokens: 32 }, + { cacheCreationInputTokens: 32 }, + { reasoningOutputTokens: 8 }, + { status: "unavailable" as const, countedCallCount: 0 }, + { status: "partial" as const, countedCallCount: 0 }, + ])("drops malformed usage while retaining its message: %j", async (invalid) => { + const store = new ChatPersistence(root); + await store.write("proj_validation", [ + { + id: "session", + projectId: "proj_validation", + title: "Title", + createdAt: "now", + messages: [ + { + id: "message", + role: "assistant", + content: "retained", + createdAt: "now", + usage: { ...reportedUsage, ...invalid }, + }, + ], + }, + ]); + const messages = new ChatPersistence(root).read("proj_validation")[0]?.messages; + expect(messages).toHaveLength(1); + expect(messages?.[0]).toMatchObject({ id: "message", content: "retained" }); + expect(messages?.[0].usage).toBeUndefined(); + }); + + it.each([ + { ...reportedUsage, status: "partial" as const }, + { + ...reportedUsage, + status: "partial" as const, + inputTokens: Number.MAX_SAFE_INTEGER, + totalTokens: 1, + }, + { + inputTokens: 0, + outputTokens: 0, + totalTokens: 0, + modelCallCount: 1, + countedCallCount: 0, + status: "unavailable" as const, + }, + ])("retains valid partial lower bounds and unavailable usage: %j", async (usage) => { + const store = new ChatPersistence(root); + await store.write("proj_valid", [ + { + id: "session", + projectId: "proj_valid", + title: "Title", + createdAt: "now", + messages: [{ id: "message", role: "assistant", content: "reply", createdAt: "now", usage }], + }, + ]); + expect(new ChatPersistence(root).read("proj_valid")[0]?.messages[0].usage).toEqual(usage); + }); + + it("does not create usage before model validation succeeds", async () => { + const session = createSession("proj_no_config"); + const result = await runChat("proj_no_config", session.id, "question", { + getConfig: () => null, + } as unknown as LlmConfigStore); + expect(result).toMatchObject({ success: false }); + expect(result.userMessage).toBeUndefined(); + expect(invokeMock).not.toHaveBeenCalled(); + expect(selectSession("proj_no_config", session.id)?.messages).toEqual([]); + }); + + it("keeps failure diagnostics and usage out of subsequent model history", async () => { + invokeMock.mockImplementation(async (args) => ({ + text: "", + reason: "diagnostic-secret", + document: args.document, + mutated: false, + usage: reportedUsage, + })); + const session = createSession("proj_failure_history"); + await runChat("proj_failure_history", session.id, "first question", config()); + await runChat("proj_failure_history", session.id, "second question", config()); + expect(invokeMock.mock.lastCall?.[0].history).toEqual([ + { role: "user", content: "first question" }, + { role: "user", content: "second question" }, + ]); + }); + it("restores multiple sessions and persists rename and delete per project", async () => { const first = createSession("proj_a", "First"); const deleted = createSession("proj_a", "Deleted"); @@ -237,7 +478,12 @@ describe("local Agent chat history", () => { await vi.waitFor(() => expect(invokeMock).toHaveBeenCalled()); await deleteProjectChat("proj_mid_turn"); const document = createEmptyDocument({ title: "x", projectId: "proj_mid_turn" }); - reply({ text: "late reply", document, mutated: false }); + reply({ + text: "late reply", + document, + mutated: false, + usage: createTurnUsageAccumulator().snapshot(), + }); expect(await turn).toMatchObject({ success: false }); await restart(); diff --git a/electron/ai-edition/chat-persistence.ts b/electron/ai-edition/chat-persistence.ts index 5a86a69879..c309558eb6 100644 --- a/electron/ai-edition/chat-persistence.ts +++ b/electron/ai-edition/chat-persistence.ts @@ -7,16 +7,67 @@ import { existsSync, readFileSync } from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; import { z } from "zod"; +import { + executionStepSchema, + restoredExecutionSteps, +} from "../../src/lib/ai-edition/chatExecution"; import { renameWithRetry } from "../atomic-rename"; const toolCallSchema = z.object({ name: z.string(), summary: z.string() }); +const tokenCountSchema = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER); +const usageSchema = z + .object({ + inputTokens: tokenCountSchema, + outputTokens: tokenCountSchema, + totalTokens: tokenCountSchema, + modelCallCount: tokenCountSchema, + countedCallCount: tokenCountSchema, + status: z.enum(["complete", "partial", "unavailable"]), + cachedInputTokens: tokenCountSchema.optional(), + cacheCreationInputTokens: tokenCountSchema.optional(), + reasoningOutputTokens: tokenCountSchema.optional(), + }) + .refine((usage) => { + if (usage.countedCallCount > usage.modelCallCount) return false; + if ((usage.cachedInputTokens ?? 0) > usage.inputTokens) return false; + if ((usage.cacheCreationInputTokens ?? 0) > usage.inputTokens) return false; + if ((usage.reasoningOutputTokens ?? 0) > usage.outputTokens) return false; + if (usage.status === "unavailable") { + return ( + usage.countedCallCount === 0 && + usage.inputTokens === 0 && + usage.outputTokens === 0 && + usage.totalTokens === 0 + ); + } + if (usage.countedCallCount === 0) return false; + if (usage.status === "complete") { + const sum = usage.inputTokens + usage.outputTokens; + return ( + usage.modelCallCount > 0 && + usage.countedCallCount === usage.modelCallCount && + Number.isSafeInteger(sum) && + sum === usage.totalTokens + ); + } + // Partial aggregates retain safe lower bounds independently on overflow. + return true; + }); +// A damaged usage extension must not invalidate an otherwise readable v1 message. +const optionalUsageSchema = z.unknown().transform((value) => { + const parsed = usageSchema.safeParse(value); + return parsed.success ? parsed.data : undefined; +}); const messageSchema = z.object({ + executionSteps: z.array(executionStepSchema).optional().catch(undefined), id: z.string().min(1), role: z.enum(["user", "assistant"]), content: z.string(), stopped: z.boolean().optional(), createdAt: z.string().min(1), toolCalls: z.array(toolCallSchema).optional(), + usage: optionalUsageSchema.optional(), + durationMs: tokenCountSchema.optional().catch(undefined), }); const sessionSchema = z.object({ id: z.string().min(1), @@ -66,6 +117,8 @@ export class ChatPersistence { if (session.compaction && session.compaction.coveredCount > session.messages.length) continue; seen.add(session.id); + for (const message of session.messages) + message.executionSteps = restoredExecutionSteps(message.executionSteps); sessions.push(session); } return sessions; diff --git a/electron/ai-edition/chat-service.compaction.test.ts b/electron/ai-edition/chat-service.compaction.test.ts index 37f715b354..2facd331c8 100644 --- a/electron/ai-edition/chat-service.compaction.test.ts +++ b/electron/ai-edition/chat-service.compaction.test.ts @@ -4,6 +4,7 @@ // behind the summarizer, so no test here needs a provider or a key. import { beforeEach, describe, expect, it, vi } from "vitest"; +import { createTurnUsageAccumulator } from "./deep-agent/usage"; vi.mock("./deep-agent/service", () => ({ invokeOpenScreenAgent: vi.fn(), @@ -15,6 +16,7 @@ vi.mock("./deep-agent/chat-model", () => ({ })); import { + compactSession, compactSessionNow, createSession, getSessionContextUsage, @@ -60,11 +62,39 @@ beforeEach(() => { chatModelMock.mockReset(); invokeMock.mockImplementation(async (args) => { histories.push([...args.history]); - return { text: "ok", document: args.document, mutated: false }; + return { + text: "ok", + document: args.document, + mutated: false, + usage: createTurnUsageAccumulator().snapshot(), + }; }); }); describe("compaction", () => { + it("reports a manual provider failure without changing the conversation", async () => { + const summarize = stubSummarizer("short summary"); + const session = createSession("proj_compact_network_failure"); + for (let i = 0; i < 6; i += 1) { + await runChat( + "proj_compact_network_failure", + session.id, + `Detailed message ${i}`, + stubConfig(), + ); + } + const before = structuredClone(selectSession("proj_compact_network_failure", session.id)); + summarize.mockRejectedValue(new Error("connect ECONNREFUSED")); + await expect( + compactSessionNow("proj_compact_network_failure", session.id, stubConfig()), + ).rejects.toThrow("connect ECONNREFUSED"); + expect(selectSession("proj_compact_network_failure", session.id)).toEqual(before); + // The legacy best-effort caller keeps its existing non-throwing behavior. + await expect( + compactSession("proj_compact_network_failure", session.id, stubConfig()), + ).resolves.toBeNull(); + expect(selectSession("proj_compact_network_failure", session.id)).toEqual(before); + }); it("NEVER runs on its own, however big the history gets", async () => { // The headline rule. A turn used to measure the history against a // guessed 80k-token budget and, past 70% of it, block on a whole extra diff --git a/electron/ai-edition/chat-service.execution.test.ts b/electron/ai-edition/chat-service.execution.test.ts new file mode 100644 index 0000000000..4767b2ae71 --- /dev/null +++ b/electron/ai-edition/chat-service.execution.test.ts @@ -0,0 +1,184 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; +import { createEmptyDocument } from "../../src/lib/ai-edition/schema"; +import { createExecutionTracker, toolExecutionStep } from "./chat-execution"; +import { ChatPersistence } from "./chat-persistence"; + +vi.mock("./deep-agent/service", () => ({ invokeOpenScreenAgent: vi.fn() })); + +import { + configureChatPersistence, + createSession, + deleteSession, + flushChatPersistence, + runChat, + selectSession, +} from "./chat-service"; +import { invokeOpenScreenAgent } from "./deep-agent/service"; +import type { LlmConfigStore } from "./llm-config-store"; + +let root: string; +const usage = { + inputTokens: 0, + outputTokens: 0, + totalTokens: 0, + modelCallCount: 0, + countedCallCount: 0, + status: "unavailable" as const, +}; +const config = { + getConfig: () => ({ provider: "openai", model: "fixture" }), + getCredential: () => ({ value: "test" }), +} as unknown as LlmConfigStore; +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), "openscreen-execution-")); + configureChatPersistence(root); +}); +afterEach(async () => { + await flushChatPersistence(); + await rm(root, { recursive: true, force: true }); + vi.clearAllMocks(); +}); + +it.each([ + "success", + "failed", + "stopped", +])("persists ordered %s steps on their originating turn across session changes and restart", async (mode) => { + const session = createSession("p"); + const other = createSession("p"); + vi.mocked(invokeOpenScreenAgent).mockImplementationOnce(async (args) => { + args.sink.thinking("**Inspect** the project"); + const first = toolExecutionStep("getTranscript", { assetId: "a" }); + const second = toolExecutionStep("getTranscript", { assetId: "b" }); + args.sink.executionStep?.(first); + args.sink.executionStep?.(second); + args.sink.executionStep?.({ + ...second, + status: "failed", + summary: "No transcript", + durationMs: 25, + }); + args.sink.executionStep?.({ ...first, status: "succeeded", durationMs: 50 }); + args.sink.thinking("Choose the available source"); + if (mode === "stopped") args.sink.executionStep?.(toolExecutionStep("getCurrentDocument", {})); + selectSession("p", other.id); + return { + text: mode === "failed" ? "" : "## Result\n\n- Done", + reason: mode === "failed" ? "Provider failed" : undefined, + document: args.document, + mutated: false, + cancelled: mode === "stopped", + usage, + }; + }); + const source = " indented\n\n```ts\n const x = 1;\n```\n"; + const result = await runChat( + "p", + session.id, + source, + config, + createEmptyDocument({ projectId: "p", title: "test" }), + ); + expect(vi.mocked(invokeOpenScreenAgent).mock.lastCall?.[0].userMessage).toBe(source); + const message = mode === "failed" ? result.userMessage : result.assistantMessage; + expect(message?.executionSteps?.map((step) => step.kind)).toEqual( + mode === "stopped" + ? ["reasoning", "tool", "tool", "reasoning", "tool"] + : ["reasoning", "tool", "tool", "reasoning"], + ); + expect(message?.executionSteps?.[2]).toMatchObject({ + status: "failed", + summary: "No transcript", + }); + if (mode === "stopped") + expect(message?.executionSteps?.at(-1)).toMatchObject({ status: "stopped" }); + await flushChatPersistence(); + configureChatPersistence(root); + const history = selectSession("p", session.id)!; + expect(history.messages[0].content).toBe(source); + expect(history.messages.at(-1)?.executionSteps).toEqual(message?.executionSteps); + expect(selectSession("p", other.id)?.messages).toEqual([]); + const persisted = await readFile(join(root, "p.chat.json"), "utf8"); + expect(persisted).not.toContain("resultJson"); +}); + +it("writes a tool start to disk before the turn ends", async () => { + const session = createSession("p"); + let onDisk = ""; + vi.mocked(invokeOpenScreenAgent).mockImplementationOnce(async (args) => { + args.sink.executionStep?.(toolExecutionStep("getTranscript", {})); + await flushChatPersistence(); + onDisk = await readFile(join(root, "p.chat.json"), "utf8"); + return { text: "Done", document: args.document, mutated: false, usage }; + }); + await runChat("p", session.id, "hello", config); + expect(onDisk).toContain('"getTranscript"'); + expect(onDisk).toContain('"running"'); +}); + +it("does not recreate execution history after the originating session is deleted", async () => { + const session = createSession("p"); + vi.mocked(invokeOpenScreenAgent).mockImplementationOnce(async (args) => { + args.sink.thinking("Inspect"); + deleteSession("p", session.id); + args.sink.executionStep?.(toolExecutionStep("getTranscript", {})); + return { text: "Done", document: args.document, mutated: false, usage }; + }); + const result = await runChat("p", session.id, "hello", config); + expect(result.success).toBe(false); + await flushChatPersistence(); + configureChatPersistence(root); + expect(selectSession("p", session.id)).toBeNull(); +}); + +it("restores an interrupted running step and tolerates legacy or malformed metadata", async () => { + const store = new ChatPersistence(root); + const tracker = createExecutionTracker(vi.fn()); + tracker.tool(toolExecutionStep("getTranscript", {})); + const base = { id: "a", projectId: "p", title: "A", createdAt: "now" }; + await store.write("p", [ + { + ...base, + messages: [ + { id: "m1", role: "user", content: "legacy", createdAt: "now" }, + { + id: "m2", + role: "assistant", + content: "partial", + createdAt: "now", + executionSteps: tracker.snapshot(), + }, + ], + }, + ]); + const history = store.read("p")[0]; + expect(history.messages[0].executionSteps).toBeUndefined(); + expect(history.messages[1].executionSteps?.[0]).toMatchObject({ status: "stopped" }); + await writeFile( + store.fileFor("p"), + JSON.stringify({ + version: 1, + projectId: "p", + sessions: [ + { + ...base, + messages: [ + { + id: "old", + role: "assistant", + content: "Retained message", + createdAt: "now", + executionSteps: [{ kind: "invalid" }], + }, + ], + }, + ], + }), + ); + const malformed = store.read("p")[0].messages[0]; + expect(malformed.content).toBe("Retained message"); + expect(malformed.executionSteps).toBeUndefined(); +}); diff --git a/electron/ai-edition/chat-service.toolloop.test.ts b/electron/ai-edition/chat-service.toolloop.test.ts index 1d7e42d1ce..b00287ae58 100644 --- a/electron/ai-edition/chat-service.toolloop.test.ts +++ b/electron/ai-edition/chat-service.toolloop.test.ts @@ -10,6 +10,7 @@ import { createEmptyDocument, documentSchema, } from "../../src/lib/ai-edition/schema"; +import { createTurnUsageAccumulator } from "./deep-agent/usage"; vi.mock("./deep-agent/service", () => ({ invokeOpenScreenAgent: vi.fn(), @@ -99,7 +100,12 @@ beforeEach(() => { // ponytail: default mock — tests override per-scenario. invokeMock.mockImplementation(async (args) => { fixture.captured.push({ args }); - return { text: "ok", document: args.document, mutated: false }; + return { + text: "ok", + document: args.document, + mutated: false, + usage: createTurnUsageAccumulator().snapshot(), + }; }); }); @@ -109,7 +115,7 @@ beforeEach(() => { async function streamAgent( args: Parameters[0], events: ReadonlyArray<{ kind: string; payload: unknown }>, -): Promise<{ text: string; document: AxcutDocument; mutated: boolean }> { +): Promise { const sink = args.sink; for (const event of events) { switch (event.kind) { @@ -138,6 +144,7 @@ async function streamAgent( text: "Done.", document: args.document, mutated: false, + usage: createTurnUsageAccumulator().snapshot(), }; } @@ -146,7 +153,12 @@ describe("runChat tool loop", () => { invokeMock.mockImplementationOnce(async (args) => { args.sink.toolStart("addTrim", { startSec: 5, endSec: 8, reason: "silence" }); args.sink.toolEnd("addTrim", true, "added trim 0:05.0 – 0:08.0"); - return { text: "Done.", document: args.document, mutated: true }; + return { + text: "Done.", + document: args.document, + mutated: true, + usage: createTurnUsageAccumulator().snapshot(), + }; }); const session = createSession("proj_loop"); @@ -168,7 +180,12 @@ describe("runChat tool loop", () => { invokeMock.mockImplementationOnce(async (args) => { args.sink.toolStart("addTrim", { startSec: 1, endSec: 2 }); args.sink.toolEnd("addTrim", true, "added trim 0:01.0 – 0:02.0"); - return { text: "Done.", document: args.document, mutated: true }; + return { + text: "Done.", + document: args.document, + mutated: true, + usage: createTurnUsageAccumulator().snapshot(), + }; }); const session = createSession("proj_rewind"); @@ -224,6 +241,7 @@ describe("runChat tool loop", () => { invokeMock.mockImplementationOnce(async (args) => ({ text: "", document: args.document, + usage: createTurnUsageAccumulator().snapshot(), mutated: false, })); const session = createSession("proj_empty_reply"); @@ -273,6 +291,7 @@ describe("runChat tool loop", () => { text: "Hi!", document: args.document, mutated: false, + usage: createTurnUsageAccumulator().snapshot(), })); const s = createSession("proj_text_only"); const result = await runChat("proj_text_only", s.id, "hello", stubConfig()); @@ -291,7 +310,12 @@ describe("runChat tool loop", () => { args.sink.thinking("concluding."); args.sink.toolStart("addTrim", { startSec: 1, endSec: 2 }); args.sink.toolEnd("addTrim", true, "added trim 0:01.0 – 0:02.0"); - return { text: "Done.", document: args.document, mutated: true }; + return { + text: "Done.", + document: args.document, + mutated: true, + usage: createTurnUsageAccumulator().snapshot(), + }; }); const sink = { @@ -335,7 +359,12 @@ describe("runChat tool loop", () => { invokeMock.mockReset(); invokeMock.mockImplementationOnce(async (args) => { args.sink.error("Upstream 404 404 Page not found"); - return { text: "", document: args.document, mutated: false }; + return { + text: "", + document: args.document, + mutated: false, + usage: createTurnUsageAccumulator().snapshot(), + }; }); const sinkErr = { text: (delta: string) => fixture.events.push({ kind: "text", payload: delta }), @@ -402,6 +431,7 @@ describe("allowAgentEdits reaches the agent", () => { text: "I went ahead and cut them.", document: args.document, mutated: true, + usage: createTurnUsageAccumulator().snapshot(), })); const session = createSession("proj_consent_belt"); const result = await runChat( @@ -421,6 +451,7 @@ describe("allowAgentEdits reaches the agent", () => { text: "Cut them.", document: args.document, mutated: true, + usage: createTurnUsageAccumulator().snapshot(), })); const session = createSession("proj_consent_allowed"); const result = await runChat( @@ -454,7 +485,20 @@ describe("stopped chat history", () => { controller.abort(); args.sink.error("AbortError"); args.sink.text("Late"); - return { text: "Partial reply", document: args.document, mutated: true, cancelled: true }; + return { + text: "Partial reply", + document: args.document, + mutated: true, + cancelled: true, + usage: { + inputTokens: 20, + outputTokens: 5, + totalTokens: 25, + modelCallCount: 2, + countedCallCount: 1, + status: "partial", + }, + }; }); const session = createSession("stop_history"); const result = await runChat( @@ -471,6 +515,8 @@ describe("stopped chat history", () => { cancelled: true, assistantMessage: { content: "Partial reply", + usage: { totalTokens: 25, status: "partial" }, + durationMs: expect.any(Number), stopped: true, toolCalls: [{ name: "addTrim", summary: "Kept trim" }], }, @@ -481,7 +527,12 @@ describe("stopped chat history", () => { await flushChatPersistence(); configureChatPersistence(root); const stored = selectSession("stop_history", session.id)!; - expect(stored.messages[1]).toMatchObject({ stopped: true, content: "Partial reply" }); + expect(stored.messages[1]).toMatchObject({ + stopped: true, + content: "Partial reply", + usage: { totalTokens: 25, status: "partial" }, + durationMs: expect.any(Number), + }); expect(stored.messages[0].checkpointId).toBeNull(); await runChat("stop_history", session.id, "continue", stubConfig(), fixtureDocument()); expect(selectSession("stop_history", session.id)?.messages).toHaveLength(4); @@ -496,6 +547,7 @@ describe("stopped chat history", () => { text: "", document: args.document, mutated: false, + usage: createTurnUsageAccumulator().snapshot(), cancelled: true, })); const session = createSession("stop_before_text"); diff --git a/electron/ai-edition/chat-service.ts b/electron/ai-edition/chat-service.ts index 081e254638..e1ea71ee7a 100644 --- a/electron/ai-edition/chat-service.ts +++ b/electron/ai-edition/chat-service.ts @@ -7,6 +7,7 @@ import { randomUUID } from "node:crypto"; import { HumanMessage, SystemMessage } from "@langchain/core/messages"; +import type { ChatExecutionStep, ToolExecutionStep } from "../../src/lib/ai-edition/chatExecution"; import { type AxcutTimelineOperation, applyTimelineOperation, @@ -18,6 +19,7 @@ import { } from "../../src/lib/ai-edition/schema"; import type { ShaderEffectChecker } from "../../src/lib/ai-edition/shaderEffects"; import type { + AgentTurnUsage, AiEditionChatMessage, AiEditionChatResult, AiEditionToolCallSummary, @@ -32,6 +34,7 @@ import { compactionSplitIndex, DEFAULT_BUDGET_TOKENS, } from "./chat-compaction"; +import { createExecutionTracker } from "./chat-execution"; import { ChatPersistence } from "./chat-persistence"; import type { CursorTelemetryReader } from "./deep-agent/service"; import type { DocumentService } from "./document-service"; @@ -297,6 +300,8 @@ export function deleteSession(projectId: string, sessionId: string): boolean { } export interface ChatEventSink { + usage?: (usage: AgentTurnUsage) => void; + executionStep?: (step: ChatExecutionStep) => void; /** Streamed text delta from the model. */ text?: (delta: string) => void; /** Streamed delta from the model's reasoning block (Anthropic/MiniMax @@ -331,6 +336,8 @@ const noop = () => undefined; /** ponytail: zero-config sink that swallows every event. */ const NOOP_SINK: Required = { + usage: noop, + executionStep: noop, text: noop, thinking: noop, toolStart: noop, @@ -436,9 +443,30 @@ export async function runChat( const appliedToolCalls: AiEditionToolCallSummary[] = []; + const originatingSession = session; + const isOriginValid = () => + sessionsByProject.get(projectId) === sessions && + sessions.get(originatingSession.id) === originatingSession && + originatingSession.messages.includes(userMessage); + let pendingSave: ReturnType | undefined; + const execution = createExecutionTracker((step) => { + if (!isOriginValid()) return; + userMessage.executionSteps = execution.snapshot(); + emit.executionStep(step); + // A start is written at once: restart recovery can only stop a step it can read. + if (step.kind === "tool" && step.status === "running") persistProject(projectId); + else if (!pendingSave) + pendingSave = setTimeout(() => { + pendingSave = undefined; + if (isOriginValid()) persistProject(projectId); + }, 200); + }); let acceptingEvents = true; const canEmit = () => acceptingEvents && !env.signal?.aborted; const agentSink = { + executionStep: (step: ToolExecutionStep) => { + if (acceptingEvents && isOriginValid()) execution.tool(step); + }, document: (document: AxcutDocument) => { if (editsAllowed && canEmit()) emit.document(document); }, @@ -446,7 +474,13 @@ export async function runChat( if (canEmit()) emit.text(delta); }, thinking: (delta: string) => { - if (canEmit()) emit.thinking(delta); + if (canEmit()) { + emit.thinking(delta); + execution.thinking(delta); + } + }, + usage: (usage: AgentTurnUsage) => { + if (canEmit()) emit.usage(usage); }, toolStart: (name: string, args: unknown) => { if (canEmit()) emit.toolStart(name, args); @@ -465,6 +499,7 @@ export async function runChat( const { invokeOpenScreenAgent } = await import("./deep-agent/service"); + const startedAt = performance.now(); const result = await invokeOpenScreenAgent({ signal: env.signal, document: workingDocument ?? emptyDocumentForTextOnly(projectId), @@ -485,7 +520,10 @@ export async function runChat( checkShaderEffect: env.checkShaderEffect, }); + const executionSteps = execution.finish(Boolean(result.cancelled)); + if (pendingSave) clearTimeout(pendingSave); acceptingEvents = false; + const durationMs = Math.max(0, Math.round(performance.now() - startedAt)); // The project was deleted while the agent ran: its sessions are gone, so a // reply recorded now would be reported as sent and then never be seen again. @@ -493,26 +531,46 @@ export async function runChat( return { success: false, error: "The project was deleted while the agent was replying." }; } + // Bind the turn to the actual session and user message captured above, even + // when the caller supplied a stale id or selected another session meanwhile. + if (sessions.get(session.id) !== session) { + return { success: false, error: "The chat session was deleted while the agent was replying." }; + } + const originatingMessage = session.messages.find((entry) => entry.id === userMessage.id); + // Rewind preserves message ids but replaces survivors with copies. Identity + // also rejects a turn whose own user message was the rewind target. + if (originatingMessage !== userMessage) { + return { success: false, error: "The chat was rewound while the agent was replying." }; + } + if (!result.text && !result.cancelled) { // ponytail: surface the deep-agent's diagnostic so the user can see // *why* the model produced no text (e.g. MiniMax streaming only // thinking blocks, or a non-text content shape that our extractor // missed). Falls back to the generic message when the agent didn't // provide a reason. + originatingMessage.usage = result.usage; + originatingMessage.durationMs = durationMs; + persistProject(projectId); return { success: false, error: result.reason ?? "Empty response from model.", // The message is stored all the same, and can be rewound to retry it. userMessageId: userMessage.id, userMessageCheckpointId: userMessage.checkpointId ?? undefined, + userMessage: originatingMessage, }; } + delete userMessage.executionSteps; const assistantMessage: AiEditionChatMessage = { + executionSteps, id: randomUUID(), role: "assistant", content: result.text, stopped: result.cancelled || undefined, + usage: result.usage, + durationMs, createdAt: new Date().toISOString(), toolCalls: appliedToolCalls.length ? appliedToolCalls : undefined, }; @@ -705,6 +763,7 @@ export async function compactSessionNow( model: config.model, baseUrl: config.baseUrl, reasoningEffort: config.reasoningEffort, + throwOnError: true, }); if (!ok) return null; persistProject(projectId); @@ -850,6 +909,7 @@ async function tryCompactSession(opts: { model: string; baseUrl?: string; reasoningEffort?: string; + throwOnError?: boolean; }): Promise<{ summaryMessageId: string | null; summary: string } | null> { const { session, plan, apiKey, provider, model, baseUrl, reasoningEffort } = opts; const oldMessages = plan.payload.slice(0, plan.splitIndex); @@ -876,7 +936,10 @@ async function tryCompactSession(opts: { if (!summary) { return null; } - } catch { + } catch (error) { + // A button press must reach the renderer's failure toast; `null` means + // there was not enough history, not that the provider could not be reached. + if (opts.throwOnError) throw error; // ponytail: a failed summarize must not break the chat turn — leave // the session as-is and let the next turn try again. return null; diff --git a/electron/ai-edition/deep-agent/cancellation.test.ts b/electron/ai-edition/deep-agent/cancellation.test.ts index bd47382805..bddf25de06 100644 --- a/electron/ai-edition/deep-agent/cancellation.test.ts +++ b/electron/ai-edition/deep-agent/cancellation.test.ts @@ -188,6 +188,9 @@ describe("agent cancellation", () => { ], }); chunk(res, {}, "tool_calls"); + res.write( + `data: ${JSON.stringify({ id: "chat-test", object: "chat.completion.chunk", choices: [], usage: { prompt_tokens: 20, completion_tokens: 5, total_tokens: 25 } })}\n\n`, + ); end(res); } else { chunk(res, { role: "assistant", content: "Applied. " }); @@ -208,6 +211,12 @@ describe("agent cancellation", () => { controller.abort(); const result = await run; expect(result.cancelled).toBe(true); + expect(result.usage).toMatchObject({ + totalTokens: 25, + modelCallCount: 2, + countedCallCount: 1, + status: "partial", + }); expect(result.document.project.vocabulary).toBe("Kept"); expect(result.mutated).toBe(true); expect(events.document).toHaveBeenCalledOnce(); diff --git a/electron/ai-edition/deep-agent/service.test.ts b/electron/ai-edition/deep-agent/service.test.ts index a82b14681d..839d33a0a7 100644 --- a/electron/ai-edition/deep-agent/service.test.ts +++ b/electron/ai-edition/deep-agent/service.test.ts @@ -39,6 +39,7 @@ import { buildSystemPrompt, buildTools, type CursorTelemetryReader, + modelCallText, type OpenScreenAgentSink, runDocumentTool, SYSTEM_PROMPT, @@ -53,6 +54,12 @@ import { const OPENSCREEN_TOOLS: readonly string[] = OPENSCREEN_TOOL_NAMES; const PHANTOM_TOOLS: readonly string[] = PHANTOM_TOOL_NAMES; +it("describes the visible transcript marks left by a non-destructive word cut", () => { + expect(TOOL_DESCRIPTIONS.removeWords).toMatch(/red.*strikethrough/i); + expect(TOOL_DESCRIPTIONS.removeWords).toContain("Restore"); + expect(TOOL_DESCRIPTIONS.removeWords).not.toContain("The transcript is untouched."); +}); + /** Valid arguments for every tool, chosen so the executor's verdict is split * across the table: some succeed, some are refused for an unknown id, and * `replaceTimeline` is refused for the user-placed clips. A table where every @@ -916,3 +923,17 @@ describe("an effect's code is checked off the main thread, before the executor r expect(asked).toEqual([]); }); }); + +describe("a turn's model calls", () => { + // v2.3.0-rc.1 e2e: "(1.60 s)All four cuts landed", a bullet run into the sentence before it. + it("start each later call's text on a new paragraph", () => { + expect(modelCallText("", "Reading the transcript first.", true)).toBe( + "Reading the transcript first.", + ); + expect(modelCallText("Reading the transcript first.", "Transcript read", true)).toBe( + "\n\nTranscript read", + ); + expect(modelCallText("Transcript read", ": four cuts", false)).toBe(": four cuts"); + expect(modelCallText("Done.\n\n", "- one", true)).toBe("- one"); + }); +}); diff --git a/electron/ai-edition/deep-agent/service.ts b/electron/ai-edition/deep-agent/service.ts index aea2aaf828..b7dafcb93e 100644 --- a/electron/ai-edition/deep-agent/service.ts +++ b/electron/ai-edition/deep-agent/service.ts @@ -1,3 +1,4 @@ +import type { ToolRunnableConfig } from "@langchain/core/tools"; // ponytail: port of axcut's AxcutDeepAgentService (apps/server/src/services/ // axcut-deep-agent.ts). Wraps LangChain's `createAgent` (LangGraph ReAct graph) // with our existing document tools and emits agent.text / agent.toolStart / @@ -32,6 +33,7 @@ import { CAPTION_WORDS_PER_LINE_MAX, CAPTION_WORDS_PER_LINE_MIN, } from "../../../src/lib/ai-edition/captions/wordsPerLine"; +import type { ToolExecutionStep } from "../../../src/lib/ai-edition/chatExecution"; import { AUDIO_FADE_MAX_MS } from "../../../src/lib/ai-edition/document/audioTracks"; import { VOCABULARY_MAX_CHARS } from "../../../src/lib/ai-edition/document/transcript"; import type { AxcutDocument } from "../../../src/lib/ai-edition/schema"; @@ -47,7 +49,7 @@ import { ZOOM_DEPTH_LEGEND, } from "../../../src/lib/ai-edition/timeline/zoom-scale"; import { ZOOM_TRANSITION_LEGEND } from "../../../src/lib/ai-edition/timeline/zoom-transitions"; -import type { SegmentationSupport } from "../../../src/native/contracts"; +import type { AgentTurnUsage, SegmentationSupport } from "../../../src/native/contracts"; import { AGENT_EDITS_SETTING, type AgentToolExecution, @@ -101,6 +103,12 @@ import { setCaptionSettingsArgs, setVocabularyArgs, } from "../caption-tools"; +import { + createToolExecutionObserver, + executionSummary, + executionText, + toolExecutionStep, +} from "../chat-execution"; import { setEditorSettingsArgs } from "../editor-settings-tool"; import { abortable } from "./cancellation"; import { @@ -109,8 +117,12 @@ import { messageContentToThinking, type OpenScreenChatModelConfig, } from "./chat-model"; +import { createTurnUsageAccumulator } from "./usage"; export interface OpenScreenAgentSink { + /** Latest cumulative usage from completed model calls; never sum stream chunks. */ + usage?: (usage: AgentTurnUsage) => void; + executionStep?: (step: ToolExecutionStep) => void; text: (delta: string) => void; /** Streaming delta from the model's reasoning block (Anthropic/MiniMax * thinking). Provider-agnostic — for providers without thinking this is @@ -234,7 +246,7 @@ export const TOOL_DESCRIPTIONS: Record = { setWordText: "Correct ONE word's text, by the id getTranscriptWords returns. This changes the TRANSCRIPT and nothing else: the captions follow it, the film is untouched and no audio is cut. Use it when the transcriber misheard something — a name, a technical term — and the user asks for it to read correctly. Passing an empty string BLANKS the word: it keeps its place in the media but leaves the captions, which is how a junk token like \"(inaudible)\" is removed without cutting the speech around it. Writing the transcriber's own text back clears the correction. This is NOT how you make a spoken word go away — that removes only the label and leaves the film saying it; use removeWords, which cuts the audio with it.", removeWords: - "Cut the spoken words selected by `wordIds` from getTranscriptWords out of the film: fillers, a repeated or flubbed phrase, a false start, a duplicate take. Pass the same `assetId` as that read: word IDs are numbered independently in each asset. Omit assetId only when the IDs are unique across all transcripts. Read words in context and choose the IDs yourself; the executor does not classify speech or remove every matching text. It resolves each ID to one recorded asset, valid source timestamps, and exactly one fully covering clip, then adds non-destructive trims; words that follow one another in the transcript become ONE trim, so a phrase goes out whole, gaps included. If any ID is missing, invalid, duplicated, already cut or ambiguous, nothing changes. The transcript is untouched. The result lists each word and the ACTUAL trim span (which addTrim may shape), asset, clip and trim ID; report only those results. " + + "Cut the spoken words selected by `wordIds` from getTranscriptWords out of the film: fillers, a repeated or flubbed phrase, a false start, a duplicate take. Pass the same `assetId` as that read: word IDs are numbered independently in each asset. Omit assetId only when the IDs are unique across all transcripts. Read words in context and choose the IDs yourself; the executor does not classify speech or remove every matching text. It resolves each ID to one recorded asset, valid source timestamps, and exactly one fully covering clip, then adds non-destructive trims; words that follow one another in the transcript become ONE trim, so a phrase goes out whole, gaps included. If any ID is missing, invalid, duplicated, already cut or ambiguous, nothing changes. The transcript words and timings are preserved. Cut words remain visible with a red strikethrough; hovering a cut word reveals Restore. Do not claim that the transcript is unmarked or that strikethrough is unavailable. The result lists each word and the ACTUAL trim span (which addTrim may shape), asset, clip and trim ID; report only those results. " + CUT_TRANSITIONS_RESULT, addTrim: `Add ONE trim range: a cut of a span inside a clip (this source-time span will not be played or exported) that does NOT split the clip. Times are in seconds of the asset's source time. This is the preferred (and for 'remove silences' requests, the only) way to handle silences; it preserves the user's placed clips and only adds a cut. When you have several cuts to make, use addTrims and send them together — this one is for a single cut or a later correction. A cut belongs to ONE clip: \`clipId\` is inferred when a single clip covers the range, but when several clips draw on the same asset over it the call FAILS and lists them — pass the \`clipId\` you mean (ids come from getCurrentDocument). ${ZOOM_TRANSITIONS_NOTE} ${CUT_TRANSITIONS_RESULT}`, addTrims: `Add MANY trim ranges in one call: \`ranges\` is a list, each entry taking exactly the fields addTrim takes. Use this whenever you have more than one cut to make — 'remove the silences' on a half-hour recording is hundreds of cuts, and sending them one at a time costs one round trip each. Each range stands or falls ALONE: one that cannot be placed is refused by itself and listed in \`refused\` with its index and the reason, while every other range is still applied. Nothing is rolled back, so a single bad bound never costs you the rest. The result leads with requested / appliedCount / refusedCount so you can see a partial outcome without re-reading the document — report what was refused rather than claiming the whole list landed. ${ZOOM_TRANSITIONS_NOTE} Each applied entry carries the same \`cutTransitions\` warning addTrim reports.`, @@ -265,7 +277,7 @@ export const TOOL_DESCRIPTIONS: Record = { setAspectRatio: "Set the shape of the exported video: 16:9 for X, YouTube or a slide, 9:16 for a vertical story or short, 1:1 or 4:5 for a square feed, auto to follow the recording, or one of the clips' own shapes (getCurrentDocument's settingsContext.originalAspectRatios). Annotations on the frame keep their place and shape. The same as setEditorSettings with aspectRatio alone.", setEditorSettings: - 'Change the project\'s look: pass only the settings to change, named as getCurrentDocument returns them under `settings`. Background: wallpaper (a bundled path from settingsContext.bundledWallpapers, a CSS colour or gradient, or a base64 PNG/JPEG data URL; never a file path or web URL; an embedded image reads back as a marker, leave the key out to keep it), wallpaperMotion (moves a gradient or image only), backgroundBlur 0–1. Frame: aspectRatio ("auto", a preset, or an originalAspectRatios token; a fixed format picked for the first time fills the frame with a window that follows the cursor, which formatFollowCursor turns off when settingsContext.formatFill is "available"), padding 0–100 %, borderRadius 0–64 px, shadowIntensity 0–1, frame (none, window, laptop, phone, monitor) and frameTheme (light, dark), motionBlurAmount 0–1, depthOfField (blurs 3D-tilted zooms only). Camera: webcamLayoutPreset, webcamMirrored, webcamReactiveZoom (shrinks on zoom), webcamMaskShape (square or rectangle, the camera\'s own proportions; an untouched roundness follows it), webcamRoundness 0–1, webcamSizePreset 15–60, webcamAnchor (a corner or an edge middle), webcamCropRegion (the framing: a square share of the picture, width = height ≥ 1/3) and webcamCropPan (where that window sits, 0–1 per axis, 0.5 centred), webcamBackgroundMode (none, transparent, blur, custom: needs settingsContext.cameraSegmentation ready), webcamBlurIntensity 0–1, webcamWallpaper (as wallpaper). Cursor, under `cursor`: show, autoHide, theme (settingsContext.cursorThemes), asArrow (the recorded cursor kinds drawn as the arrow instead), size 1.5–6, smoothing 0–1, motionBlur 0–1, clickBounce 0–2, clickImpact, clickSound (only heard when the take recorded clicks) and clickSoundGainDb −24–0, model3d. Also autoFocusAll (every zoom follows the cursor) and audioGainDb −12–12 dB, the master volume. A value out of range or a key not listed is refused with the reason, and nothing changes. The result lists what was applied, including what the editor changes with it, and `inactive` for any of those the current state leaves without effect.', + 'Change the project\'s look: pass only the settings to change, named as getCurrentDocument returns them under `settings`. Background: wallpaper (a bundled path from settingsContext.bundledWallpapers, a CSS colour or gradient, or a base64 PNG/JPEG data URL; never a file path or web URL; an embedded image reads back as a marker, leave the key out to keep it), wallpaperMotion (moves a gradient or image only), backgroundBlur 0–1. Frame: aspectRatio ("auto", a preset, or an originalAspectRatios token; a fixed format picked for the first time fills the frame with a window that follows the cursor, which formatFollowCursor turns off when settingsContext.formatFill is "available"), padding 0–100 %, borderRadius 0–64 px, shadowIntensity 0–1, frame (none, window, laptop, phone, monitor) and frameTheme (light, dark), motionBlurAmount 0–1, depthOfField (blurs 3D-tilted zooms only). Camera: webcamLayoutPreset, webcamMirrored, webcamReactiveZoom (shrinks on zoom), webcamMaskShape (square or rectangle, the camera\'s own proportions; an untouched roundness follows it), webcamRoundness 0–1, webcamSizePreset 15–60, webcamAnchor (a corner or an edge middle), webcamCropRegion (the framing: a square share of the picture, width = height ≥ 1/3) and webcamCropPan (where that window sits, 0–1 per axis, 0.5 centred), webcamBackgroundMode (none, transparent, blur, custom: needs settingsContext.cameraSegmentation ready), webcamBlurIntensity 0–1, webcamWallpaper (as wallpaper). Cursor, under `cursor`: show, autoHide, theme (settingsContext.cursorThemes), asArrow (the recorded cursor kinds drawn as the arrow instead), size 1.5–6, smoothing 0–1, motionBlur 0–1, clickBounce 0–2, clickImpact, clickSound (only heard when the take recorded clicks) and clickSoundGainDb −24–0, model3d. Also autoFocusAll (every zoom follows the cursor), audioGainDb −12–12 dB, the master volume, and recordingMuted (silences the recording\'s own sound in the preview and the export; added audio tracks still play). A value out of range or a key not listed is refused with the reason, and nothing changes. The result lists what was applied, including what the editor changes with it, and `inactive` for any of those the current state leaves without effect.', addEffect: "Write a visual effect as a shader and pose it over a span of the edited timeline (virtual seconds), or pose one of this project's effects again (effectId). Use it for a look the settings do not offer. `effect.code` is WGSL that defines `fn effect(uv: vec2) -> vec4`, called for every pixel; uv is 0..1 with (0,0) top-left. Available inside it: `frame(uv)` samples the composed picture; `footage(p)` samples the recording itself at its own resolution at point p of its surface (the space of `fx_to_footage`), transparent outside 0..1, with no cursor or webcam on it: read it for anything that magnifies, frame() for the rest; `params.` reads each declared parameter; `ctx.time` is programme seconds; `ctx.region.x` goes 0 to 1 across the region (y its length in s, z seconds since it began); `ctx.resolution` and `ctx.aspect`; `ctx.cursor` (xy in uv, z = 1 when on screen); `ctx.clicks[0..3]`, the last four clicks, newest first (xy in uv where it happened, z seconds since, w = 1 when the slot holds one); `ctx.footage` and `ctx.camera`, the recording's and the webcam's rects (x, y, w, h in uv; camera zero-sized when absent) — all already placed through zoom and layout; `fx_to_footage(uv)` and `fx_from_footage(p)` map between output uv and the recording's own surface (0..1 across the recording) through the warp the compositor draws it with, zoom, layout and 3D camera included, and `ctx.footage_aspect` is its width over height: the effect works in whichever of the two spaces fits it; `fx_px(n)`, n pixels of a 1080p frame as a fraction of the height; `fx_hash(vec2)`, a random number in [0,1) identical on every GPU. Rules: the effect is a pure function of those inputs (no state between frames), so the preview and the export show the same pixels; reason in uv or fx_px, never raw pixels; every loop is a `for` with literal bounds (`for (var i = 0; i < 16; i++)`), at most 4096 iterations per pixel — the one hard rule. `params` declares up to 8 (recommended) parameters, each `{type: number|toggle|color|point, name, label, default}` (number also `min`, `max`); declare one only when moving it visibly changes the effect over its whole range. `target`: `image` (default: the picture under the annotations and captions), `text` (the annotations and captions alone, premultiplied alpha), `both` (everything); use text or both only when asked. One effect runs at a time: to combine two looks, write them in one shader. The result gives the cost tier, in frame() and footage() reads per pixel, with what a 1080p frame pays on a fast discrete GPU (RTX 4070 Ti) and on an integrated one (Apple M1): light ≤ 8, up to 0.06 ms and 1.3 ms; medium ≤ 32, 0.14 ms and 4.3 ms; heavy ≤ 128, 0.54 ms and 16 ms; very heavy beyond, 2 ms and 49 ms at 512 reads. It also gives a recommendation when the tier runs long: light anywhere, medium up to a minute, heavy up to 10 s, very heavy a few seconds. It is a recommendation, never a refusal. Code that does not compile is refused with the compiler's error at the line of your code: fix it and call again. Check the result with getFrames.", setEffect: @@ -381,18 +393,45 @@ function documentTool( inTurn: TurnQueue, ) { return tool( - (args: z.infer) => + (args: z.infer, config?: ToolRunnableConfig) => inTurn(async () => { runtime.signal?.throwIfAborted(); + const step = toolExecutionStep(name, args); + step.id = config?.toolCall?.id ?? step.id; + sink.executionStep?.(step); sink.toolStart(name, args); - const execution = await runDocumentTool(holder.current, name, args, editsAllowed, runtime); - runtime.signal?.throwIfAborted(); - if (execution.document) { - holder.current = execution.document; - sink.document?.(execution.document); + try { + const execution = await runDocumentTool( + holder.current, + name, + args, + editsAllowed, + runtime, + ); + runtime.signal?.throwIfAborted(); + if (execution.document) { + holder.current = execution.document; + sink.document?.(execution.document); + } + sink.toolEnd(name, execution.ok, execution.summary); + sink.executionStep?.({ + ...step, + status: execution.ok ? "succeeded" : "failed", + durationMs: Math.max(0, Date.now() - step.startedAt), + summary: executionSummary(execution), + }); + return execution.resultJson; + } catch (error) { + sink.executionStep?.({ + ...step, + status: runtime.signal?.aborted ? "stopped" : "failed", + durationMs: Math.max(0, Date.now() - step.startedAt), + summary: runtime.signal?.aborted + ? undefined + : executionText(error instanceof Error ? error.message : String(error)), + }); + throw error; } - sink.toolEnd(name, execution.ok, execution.summary); - return execution.resultJson; }), { name, description: TOOL_DESCRIPTIONS[name], schema }, ); @@ -681,8 +720,19 @@ export async function probeCursorTelemetry( */ const SILENT_TOOL_EVENTS: ReadonlySet = new Set(["on_tool_start", "on_tool_end"]); +/** + * The text one stream chunk adds to the turn. A turn is several model calls, one per round of + * tool results, and their texts used to be glued: "…transcript first.Transcript read: …", or a + * list item run into the sentence before it. The first text of each later call starts a new + * paragraph. + */ +export function modelCallText(turnText: string, delta: string, startsCall: boolean): string { + return startsCall && turnText.trim() && !turnText.endsWith("\n\n") ? `\n\n${delta}` : delta; +} + export interface InvokeResult { cancelled?: boolean; + usage: AgentTurnUsage; text: string; document: AxcutDocument; mutated: boolean; @@ -696,11 +746,14 @@ export interface InvokeResult { export async function invokeOpenScreenAgent(args: InvokeArgs): Promise { const { document, model, history, userMessage, sink } = args; const editsAllowed = args.editsAllowed !== false; + const toolExecutions = createToolExecutionObserver((step) => sink.executionStep?.(step)); const lifetime = new AbortController(); const signal = args.signal ? AbortSignal.any([args.signal, lifetime.signal]) : lifetime.signal; let finalText = ""; + let callHasText = false; let chatModelChunks: unknown[] = []; + const usage = createTurnUsageAccumulator(); const holder: DocumentHolder = { current: document }; const initialDocumentJSON = JSON.stringify(document); @@ -712,13 +765,18 @@ export async function invokeOpenScreenAgent(args: InvokeArgs): Promise | undefined; const name = typeof event.name === "string" ? (event.name as string) : ""; + if (eventType === "on_chat_model_start") { + toolExecutions.reset(); + callHasText = false; + } + if (eventType === "on_chat_model_end") { + const output = data?.output as + | { tool_calls?: Array<{ id?: string; name: string; args: unknown }> } + | undefined; + for (const call of output?.tool_calls ?? []) + if (call.id) toolExecutions.begin(call.id, call.name, call.args); + } + if (eventType === "on_chain_end") { + const output = data?.output as + | { messages?: Array<{ tool_call_id?: string; status?: string; content?: unknown }> } + | undefined; + // The SDK omits status on schema errors. Successful document tools already emitted a terminal verdict. + for (const message of output?.messages ?? []) + if (message.tool_call_id) + toolExecutions.result(message.tool_call_id, messageContentToText(message.content)); + } if (eventType === "on_chat_model_stream") { const chunk = data?.chunk as Record | undefined; if (chunk) chatModelChunks.push(chunk); @@ -767,8 +849,10 @@ export async function invokeOpenScreenAgent(args: InvokeArgs): Promise e.event) .join(",")}). Last chunk: ${sample}`; sink.error(reason); - return { text: "", document: holder.current, mutated, reason }; + return { text: "", document: holder.current, mutated, reason, usage: usage.snapshot() }; } - return { text: finalText.trim(), document: holder.current, mutated }; + return { text: finalText, document: holder.current, mutated, usage: usage.snapshot() }; } catch (err) { if (args.signal?.aborted) { return { - text: finalText.trim(), + text: finalText, document: holder.current, mutated: JSON.stringify(holder.current) !== initialDocumentJSON, cancelled: true, + usage: usage.snapshot(), }; } // ponytail: surface the LangChain/HTTP error (with name + truncated @@ -839,6 +924,7 @@ export async function invokeOpenScreenAgent(args: InvokeArgs): Promise { + let requests = 0; + const server = createServer((req, res) => { + req.resume(); + req.on("end", () => { + requests++; + if (mode === "failure" && requests === 2) { + res.writeHead(400); + res.end( + JSON.stringify({ error: { message: "fixture failure", type: "invalid_request_error" } }), + ); + return; + } + if (mode === "anthropic") { + sse( + res, + [ + { + type: "message_start", + message: { + id: "msg_1", + type: "message", + role: "assistant", + model: "claude-fixture", + content: [], + stop_reason: null, + stop_sequence: null, + usage: { + input_tokens: 10, + output_tokens: 0, + cache_read_input_tokens: 4, + cache_creation_input_tokens: 3, + }, + }, + }, + { type: "content_block_start", index: 0, content_block: { type: "text", text: "" } }, + { type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "Done" } }, + { type: "content_block_stop", index: 0 }, + { + type: "message_delta", + delta: { stop_reason: "end_turn", stop_sequence: null }, + usage: { output_tokens: 5 }, + }, + { type: "message_stop" }, + ], + true, + ); + return; + } + if (mode === "responses") { + const response = { + id: "resp_1", + object: "response", + created_at: 1, + status: "completed", + model: "gpt-fixture", + output: [ + { + id: "m1", + type: "message", + role: "assistant", + status: "completed", + content: [{ type: "output_text", text: "Done", annotations: [] }], + }, + ], + usage: { + input_tokens: 10, + output_tokens: 5, + total_tokens: 15, + input_tokens_details: { cached_tokens: 4 }, + output_tokens_details: { reasoning_tokens: 2 }, + }, + }; + sse( + res, + [ + { + type: "response.created", + response: { ...response, status: "in_progress", output: [] }, + }, + { + type: "response.output_text.delta", + item_id: "m1", + output_index: 0, + content_index: 0, + delta: "Done", + }, + { type: "response.completed", response }, + ], + true, + ); + return; + } + const useTool = (mode === "chat" || mode === "failure") && requests === 1; + const message = useTool + ? { + role: "assistant", + content: "", + tool_calls: [ + { + id: "call_1", + type: "function", + function: { name: "getCurrentDocument", arguments: "{}" }, + }, + ], + } + : { role: "assistant", content: mode === "empty" ? "" : "Done" }; + const usage = { + prompt_tokens: 10, + completion_tokens: 5, + total_tokens: 15, + prompt_tokens_details: { cached_tokens: 4 }, + completion_tokens_details: { reasoning_tokens: 2 }, + }; + if (mode === "nonstream") { + res.setHeader("content-type", "application/json"); + res.end( + JSON.stringify({ + id: "chat1", + object: "chat.completion", + created: 1, + model: "fixture", + choices: [{ index: 0, message, finish_reason: "stop" }], + usage, + }), + ); + return; + } + sse(res, [ + { + id: "chat1", + object: "chat.completion.chunk", + created: 1, + model: "fixture", + choices: [ + { + index: 0, + delta: useTool + ? { role: "assistant", tool_calls: [{ index: 0, ...message.tool_calls?.[0] }] } + : message, + finish_reason: null, + }, + ], + }, + { + id: "chat1", + object: "chat.completion.chunk", + created: 1, + model: "fixture", + choices: [], + usage, + }, + ]); + }); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("missing fixture address"); + const baseURL = `http://127.0.0.1:${address.port}`; + try { + const model = + mode === "anthropic" + ? new ChatAnthropic({ + apiKey: "fixture", + model: "claude-fixture", + anthropicApiUrl: baseURL, + maxRetries: 0, + }) + : new ChatOpenAI({ + apiKey: "fixture", + model: "fixture", + configuration: { baseURL }, + maxRetries: 0, + useResponsesApi: mode === "responses", + disableStreaming: mode === "nonstream", + }); + const agent = createAgent({ + model, + tools: [ + tool(() => "read", { + name: "getCurrentDocument", + description: "read", + schema: z.object({}), + }), + ], + }); + const accumulator = createTurnUsageAccumulator(); + const ends: Record[] = []; + try { + for await (const event of agent.streamEvents({ + messages: [{ role: "user", content: "Read" }], + })) { + accumulator.observe(event); + if (event.event === "on_chat_model_end") + ends.push(event.data.output as Record); + } + } catch (error) { + if (mode !== "failure") throw error; + } + expect(ends).toHaveLength(mode === "chat" ? 2 : 1); + expect(ends[0].usage_metadata).toMatchObject({ + input_tokens: mode === "anthropic" ? 17 : 10, + output_tokens: 5, + total_tokens: mode === "anthropic" ? 22 : 15, + }); + expect(accumulator.snapshot()).toMatchObject({ + inputTokens: mode === "anthropic" ? 17 : mode === "chat" ? 20 : 10, + outputTokens: mode === "chat" ? 10 : 5, + modelCallCount: mode === "chat" || mode === "failure" ? 2 : 1, + countedCallCount: mode === "chat" ? 2 : 1, + status: mode === "failure" ? "partial" : "complete", + cachedInputTokens: mode === "chat" ? 8 : 4, + }); + if (mode === "anthropic") expect(accumulator.snapshot().cacheCreationInputTokens).toBe(3); + requests = 0; + const usageUpdates = vi.fn(); + const result = await invokeOpenScreenAgent({ + document: createEmptyDocument({ projectId: "usage", title: "usage" }), + model: { + provider: + mode === "anthropic" + ? "anthropic" + : mode === "nonstream" + ? "google" + : "openai-compatible", + model: + mode === "nonstream" + ? "gemini-3-fixture" + : mode === "responses" + ? "gpt-5-fixture" + : "fixture", + apiKey: "fixture", + baseUrl: baseURL, + maxRetries: 0, + }, + history: [], + userMessage: "Read", + sink: { + usage: usageUpdates, + text: vi.fn(), + thinking: vi.fn(), + toolStart: vi.fn(), + toolEnd: vi.fn(), + error: vi.fn(), + }, + }); + expect(usageUpdates).toHaveBeenCalled(); + expect(usageUpdates.mock.lastCall?.[0].totalTokens).toBe(result.usage.totalTokens); + if (mode === "chat") { + expect(usageUpdates.mock.calls.map(([value]) => value.totalTokens)).toEqual([15, 30]); + } + if (mode === "empty") expect(result.reason).toContain("Empty response"); + expect(result.usage).toMatchObject({ + inputTokens: mode === "anthropic" ? 17 : mode === "chat" ? 20 : 10, + outputTokens: mode === "chat" ? 10 : 5, + status: mode === "failure" ? "partial" : "complete", + }); + } finally { + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ); + } +}, 15000); + +it("returns unavailable usage for model-construction errors", async () => { + const factory = vi + .spyOn(chatModelFactory, "createOpenScreenChatModel") + .mockRejectedValueOnce(new Error("construction failed")); + const result = await invokeOpenScreenAgent({ + document: createEmptyDocument({ projectId: "usage-error", title: "usage" }), + model: { provider: "openai-compatible", model: "fixture", baseUrl: "not a URL" }, + history: [], + userMessage: "Read", + sink: { + text: vi.fn(), + thinking: vi.fn(), + toolStart: vi.fn(), + toolEnd: vi.fn(), + error: vi.fn(), + }, + }); + expect(result.usage).toMatchObject({ modelCallCount: 0, status: "unavailable" }); + factory.mockRestore(); +}); + +it.each([ + "nonstream-zero", + "stream-zero", + "missing-usage", +])("accounts for %s HTTP metadata", async (mode) => { + const server = createServer((req, res) => { + req.resume(); + req.on("end", () => { + const usage = + mode !== "missing-usage" + ? { prompt_tokens: 10, completion_tokens: 0, total_tokens: 10 } + : undefined; + const message = { role: "assistant", content: "Done" }; + if (mode === "nonstream-zero") { + res.setHeader("content-type", "application/json"); + res.end( + JSON.stringify({ + id: "zero", + object: "chat.completion", + created: 1, + model: "fixture", + choices: [{ index: 0, message, finish_reason: "stop" }], + usage, + }), + ); + } else + sse(res, [ + { + id: "missing", + object: "chat.completion.chunk", + created: 1, + model: "fixture", + choices: [{ index: 0, delta: message, finish_reason: "stop" }], + usage, + }, + ]); + }); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("missing address"); + try { + const model = new ChatOpenAI({ + apiKey: "fixture", + model: "fixture", + configuration: { baseURL: `http://127.0.0.1:${address.port}` }, + disableStreaming: mode === "nonstream-zero", + maxRetries: 0, + }); + const agent = createAgent({ model, tools: [] }); + const usage = createTurnUsageAccumulator(); + for await (const event of agent.streamEvents({ + messages: [{ role: "user", content: "Read" }], + })) { + usage.observe(event); + if (event.event === "on_chat_model_end" && mode === "nonstream-zero") { + // Locked ChatOpenAI drops explicit zero completion_tokens from both + // metadata records. Without the raw count, completeness is unknowable. + expect(event.data.output.usage_metadata).toMatchObject({ + input_tokens: 10, + total_tokens: 10, + }); + expect(event.data.output.usage_metadata).not.toHaveProperty("output_tokens"); + expect(event.data.output.response_metadata.tokenUsage).toEqual({ + promptTokens: 10, + totalTokens: 10, + }); + } + } + expect(usage.snapshot()).toMatchObject({ + inputTokens: mode !== "missing-usage" ? 10 : 0, + modelCallCount: 1, + outputTokens: 0, + totalTokens: mode !== "missing-usage" ? 10 : 0, + status: + mode === "nonstream-zero" ? "partial" : mode === "stream-zero" ? "complete" : "unavailable", + }); + } finally { + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ); + } +}); diff --git a/electron/ai-edition/deep-agent/usage.test.ts b/electron/ai-edition/deep-agent/usage.test.ts new file mode 100644 index 0000000000..a52e59ec11 --- /dev/null +++ b/electron/ai-edition/deep-agent/usage.test.ts @@ -0,0 +1,235 @@ +import { describe, expect, it } from "vitest"; +import { createTurnUsageAccumulator } from "./usage"; + +const start = (run_id = "a") => ({ event: "on_chat_model_start", run_id }); +const end = (usage_metadata?: unknown, run_id = "a") => ({ + event: "on_chat_model_end", + run_id, + data: { output: { usage_metadata } }, +}); +const full = { input_tokens: 10, output_tokens: 5, total_tokens: 15 }; +describe("turn usage", () => { + it.each([ + ["cachedInputTokens", "input", "cache_read"], + ["cacheCreationInputTokens", "input", "cache_creation"], + ["reasoningOutputTokens", "output", "reasoning"], + ] as const)("omits %s when core overflow retains a smaller parent lower bound", (field, direction, detail) => { + const a = createTurnUsageAccumulator(); + const parent = direction === "input" ? "input_tokens" : "output_tokens"; + const details = direction === "input" ? "input_token_details" : "output_token_details"; + a.observe(end({ input_tokens: 0, output_tokens: 0, [parent]: 1, [details]: { [detail]: 1 } })); + a.observe( + end( + { + input_tokens: 0, + output_tokens: 0, + [parent]: Number.MAX_SAFE_INTEGER, + [details]: { [detail]: Number.MAX_SAFE_INTEGER - 1 }, + }, + "b", + ), + ); + const result = a.snapshot(); + expect(result).toMatchObject({ status: "partial", modelCallCount: 2, countedCallCount: 2 }); + expect(direction === "input" ? result.inputTokens : result.outputTokens).toBe(1); + expect(result.totalTokens).toBe(1); + expect(result).not.toHaveProperty(field); + }); + + it("bounds enriched details against retained conflicting core", () => { + const a = createTurnUsageAccumulator(); + a.observe(end(full)); + a.observe( + end({ + input_tokens: 20, + output_tokens: 10, + total_tokens: 30, + input_token_details: { cache_read: 15, cache_creation: 15 }, + output_token_details: { reasoning: 8 }, + }), + ); + expect(a.snapshot()).toMatchObject({ inputTokens: 10, outputTokens: 5, status: "partial" }); + expect(a.snapshot()).not.toHaveProperty("cachedInputTokens"); + expect(a.snapshot()).not.toHaveProperty("cacheCreationInputTokens"); + expect(a.snapshot()).not.toHaveProperty("reasoningOutputTokens"); + }); + + it("enriches missing details on duplicate finals", () => { + const a = createTurnUsageAccumulator(); + a.observe(end(full)); + a.observe(end({ ...full, input_token_details: { cache_read: 4 } })); + expect(a.snapshot()).toMatchObject({ cachedInputTokens: 4, status: "complete" }); + }); + it("preserves first details and marks conflicting details partial", () => { + const a = createTurnUsageAccumulator(); + a.observe(end({ ...full, input_token_details: { cache_read: 4 } })); + a.observe(end({ ...full, input_token_details: { cache_read: 8 } })); + expect(a.snapshot()).toMatchObject({ cachedInputTokens: 4, status: "partial" }); + }); + it("merges complementary partial finals", () => { + const a = createTurnUsageAccumulator(); + a.observe(end({ input_tokens: 10 })); + a.observe(end({ output_tokens: 5, total_tokens: 15 })); + expect(a.snapshot()).toMatchObject({ + inputTokens: 10, + outputTokens: 5, + totalTokens: 15, + status: "complete", + }); + }); + + it("upgrades total-only metadata when later input/output agree", () => { + const a = createTurnUsageAccumulator(); + a.observe(end({ total_tokens: 15 })); + a.observe(end({ input_tokens: 10, output_tokens: 5 })); + expect(a.snapshot()).toMatchObject({ totalTokens: 15, status: "complete" }); + }); + + it("sums three completed calls to 660", () => { + const a = createTurnUsageAccumulator(); + for (const [id, input, output] of [ + ["a", 100, 10], + ["b", 200, 20], + ["c", 300, 30], + ] as const) + a.observe(end({ input_tokens: input, output_tokens: output }, id)); + expect(a.snapshot()).toMatchObject({ + inputTokens: 600, + outputTokens: 60, + totalTokens: 660, + modelCallCount: 3, + countedCallCount: 3, + status: "complete", + }); + }); + it("retains optional details when a duplicate end omits them", () => { + const a = createTurnUsageAccumulator(); + a.observe(end({ ...full, input_token_details: { cache_read: 4 } })); + a.observe(end(full)); + expect(a.snapshot().cachedInputTokens).toBe(4); + }); + + it("is unavailable without observations or credible usage", () => { + const a = createTurnUsageAccumulator(); + expect(a.snapshot().status).toBe("unavailable"); + a.observe(start()); + a.observe(end()); + expect(a.snapshot()).toMatchObject({ + modelCallCount: 1, + countedCallCount: 0, + status: "unavailable", + }); + }); + it("counts final messages once, ignores chunks and upgrades missing usage", () => { + const a = createTurnUsageAccumulator(); + a.observe(start()); + a.observe({ + event: "on_chat_model_stream", + run_id: "a", + data: { chunk: { usage_metadata: full } }, + }); + a.observe(end()); + a.observe(end(full)); + a.observe(end(full)); + expect(a.snapshot()).toMatchObject({ + inputTokens: 10, + outputTokens: 5, + totalTokens: 15, + modelCallCount: 1, + countedCallCount: 1, + status: "complete", + }); + }); + it("retains the first credible record on conflict", () => { + const a = createTurnUsageAccumulator(); + a.observe(end(full)); + a.observe(end({ ...full, input_tokens: 11, total_tokens: 16 })); + expect(a.snapshot()).toMatchObject({ inputTokens: 10, totalTokens: 15, status: "partial" }); + }); + it("upgrades a partial final record when later metadata supplies missing core fields", () => { + const a = createTurnUsageAccumulator(); + a.observe(end({ input_tokens: 10 })); + a.observe(end(full)); + expect(a.snapshot()).toMatchObject({ inputTokens: 10, totalTokens: 15, status: "complete" }); + }); + it("accepts explicit zero and omitted total", () => { + const a = createTurnUsageAccumulator(); + a.observe(end({ input_tokens: 0, output_tokens: 0 })); + expect(a.snapshot()).toMatchObject({ countedCallCount: 1, totalTokens: 0, status: "complete" }); + }); + it.each([ + { input_tokens: 8 }, + { output_tokens: 3 }, + { total_tokens: 12 }, + { ...full, total_tokens: 20 }, + { input_tokens: 10, output_tokens: "5" }, + { input_tokens: 10, output_tokens: -1 }, + { input_tokens: 10, output_tokens: Infinity }, + { input_tokens: 10, output_tokens: 0.5 }, + ])("retains lower bounds for %j", (usage) => { + const a = createTurnUsageAccumulator(); + a.observe(end(usage)); + expect(a.snapshot()).toMatchObject({ countedCallCount: 1, status: "partial" }); + }); + it("rejects unsafe sums", () => { + const a = createTurnUsageAccumulator(); + a.observe(end({ input_tokens: Number.MAX_SAFE_INTEGER, output_tokens: 1 })); + expect(a.snapshot().status).toBe("partial"); + expect(Number.isSafeInteger(a.snapshot().totalTokens)).toBe(true); + }); + it("does not sum unidentified calls and keeps prior totals on failures", () => { + const a = createTurnUsageAccumulator(); + a.observe(end(full)); + a.observe(end(full, "")); + a.observe(start("b")); + a.observe({ event: "on_chat_model_error", run_id: "b" }); + expect(a.snapshot()).toMatchObject({ + totalTokens: 15, + modelCallCount: 2, + countedCallCount: 1, + status: "partial", + }); + }); + it("omits invalid details, includes subsets without adding them", () => { + const a = createTurnUsageAccumulator(); + a.observe( + end({ + ...full, + input_token_details: { cache_read: 4, cache_creation: -1 }, + output_token_details: { reasoning: 2 }, + }), + ); + expect(a.snapshot()).toMatchObject({ + totalTokens: 15, + cachedInputTokens: 4, + reasoningOutputTokens: 2, + status: "complete", + }); + expect(a.snapshot()).not.toHaveProperty("cacheCreationInputTokens"); + }); + it("marks unfinished and missing metadata calls partial", () => { + const a = createTurnUsageAccumulator(); + a.observe(end(full)); + a.observe(start("b")); + expect(a.snapshot().status).toBe("partial"); + a.observe(end(undefined, "b")); + expect(a.snapshot().status).toBe("partial"); + }); + it("ignores raw response metadata and unrelated events", () => { + const a = createTurnUsageAccumulator(); + a.observe({ event: "on_chain_end", run_id: "x" }); + a.observe({ + event: "on_chat_model_end", + run_id: "a", + data: { output: { response_metadata: { usage: full } } }, + }); + expect(a.snapshot()).toMatchObject({ modelCallCount: 1, status: "unavailable" }); + }); + it("prevents overflow across calls", () => { + const a = createTurnUsageAccumulator(); + a.observe(end({ input_tokens: Number.MAX_SAFE_INTEGER, output_tokens: 0 })); + a.observe(end(full, "b")); + expect(a.snapshot().status).toBe("partial"); + expect(Number.isSafeInteger(a.snapshot().inputTokens)).toBe(true); + }); +}); diff --git a/electron/ai-edition/deep-agent/usage.ts b/electron/ai-edition/deep-agent/usage.ts new file mode 100644 index 0000000000..b5af250da8 --- /dev/null +++ b/electron/ai-edition/deep-agent/usage.ts @@ -0,0 +1,182 @@ +import type { AgentTurnUsage } from "../../../src/native/contracts"; + +type RecordValue = Record; +function record(value: unknown): RecordValue | undefined { + return value !== null && typeof value === "object" ? (value as RecordValue) : undefined; +} +function count(value: unknown): number | undefined { + return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : undefined; +} +interface CallUsage { + core: { input?: number; output?: number; total?: number }; + inputTokens: number; + outputTokens: number; + totalTokens: number; + complete: boolean; + cachedInputTokens?: number; + cacheCreationInputTokens?: number; + reasoningOutputTokens?: number; +} +function parseUsage(value: unknown): CallUsage | undefined { + const usage = record(value); + if (!usage) return undefined; + const input = count(usage.input_tokens); + const output = count(usage.output_tokens); + const total = count(usage.total_tokens); + if (input === undefined && output === undefined && total === undefined) return undefined; + const sum = (input ?? 0) + (output ?? 0); + const safe = Number.isSafeInteger(sum); + const full = input !== undefined && output !== undefined; + let totalTokens: number; + // Incomplete core fields retain known lower bounds. Full input/output are + // authoritative even when a provider reports an inconsistent total. + if (safe && full) totalTokens = sum; + else if (safe) totalTokens = Math.max(sum, total ?? 0); + else totalTokens = total ?? Math.max(input ?? 0, output ?? 0); + const result: CallUsage = { + core: { input, output, total: total ?? (full && safe ? sum : undefined) }, + inputTokens: input ?? 0, + outputTokens: output ?? 0, + totalTokens, + complete: full && safe && (usage.total_tokens === undefined || total === sum), + }; + const inputDetails = record(usage.input_token_details); + const outputDetails = record(usage.output_token_details); + const cached = count(inputDetails?.cache_read); + const creation = count(inputDetails?.cache_creation); + const reasoning = count(outputDetails?.reasoning); + if (cached !== undefined && input !== undefined && cached <= input) + result.cachedInputTokens = cached; + if (creation !== undefined && input !== undefined && creation <= input) + result.cacheCreationInputTokens = creation; + if (reasoning !== undefined && output !== undefined && reasoning <= output) + result.reasoningOutputTokens = reasoning; + return result; +} + +/** Only completed model messages carry authoritative usage. Stream chunks can + * contain cumulative counters and must never be summed. SDK retries hidden + * inside a model invocation are outside this observable accounting boundary. */ +export function createTurnUsageAccumulator() { + const calls = new Map< + string, + { usage?: CallUsage; ended: boolean; conflict: boolean; failed: boolean } + >(); + let gap = false; + return { + observe(value: unknown) { + const event = record(value); + if ( + !event || + !["on_chat_model_start", "on_chat_model_end", "on_chat_model_error"].includes( + String(event.event), + ) + ) + return; + const id = event.run_id; + if (typeof id !== "string" || !id.trim()) { + gap = true; + return; + } + const call = calls.get(id) ?? { ended: false, conflict: false, failed: false }; + calls.set(id, call); + if (event.event === "on_chat_model_error") call.failed = true; + if (event.event !== "on_chat_model_end") return; + call.ended = true; + const usage = parseUsage(record(record(event.data)?.output)?.usage_metadata); + if (!usage) return; + if (!call.usage) call.usage = usage; + else { + const previous = call.usage; + const core = { ...previous.core }; + let enriched = false; + for (const key of ["input", "output", "total"] as const) { + const value = usage.core[key]; + if (value === undefined) continue; + if (core[key] === undefined) { + core[key] = value; + enriched = true; + } else if (core[key] !== value) call.conflict = true; + } + // Recompute from the union of compatible known fields. A duplicate may + // supply just the missing output, rather than repeating its input count. + const merged = enriched + ? parseUsage({ + input_tokens: core.input, + output_tokens: core.output, + total_tokens: core.total, + })! + : { ...previous }; + for (const key of [ + "cachedInputTokens", + "cacheCreationInputTokens", + "reasoningOutputTokens", + ] as const) { + const first = previous[key]; + const next = usage[key]; + if (first !== undefined && next !== undefined && first !== next) call.conflict = true; + const value = first ?? next; + const bound = key === "reasoningOutputTokens" ? merged.core.output : merged.core.input; + if (value !== undefined && bound !== undefined && value <= bound) merged[key] = value; + else delete merged[key]; + } + call.usage = merged; + } + }, + snapshot(): AgentTurnUsage { + const result: AgentTurnUsage = { + inputTokens: 0, + outputTokens: 0, + totalTokens: 0, + modelCallCount: calls.size, + countedCallCount: 0, + status: "unavailable", + }; + let complete = !gap && calls.size > 0; + const overflowedDetails = new Set(); + const fields = [ + "inputTokens", + "outputTokens", + "totalTokens", + "cachedInputTokens", + "cacheCreationInputTokens", + "reasoningOutputTokens", + ] as const; + for (const call of calls.values()) { + if (!call.ended || call.failed || call.conflict || !call.usage?.complete) complete = false; + if (!call.usage) continue; + result.countedCallCount++; + for (const field of fields) { + const value = call.usage[field]; + if (value === undefined || overflowedDetails.has(field)) continue; + const sum = (result[field] ?? 0) + value; + if (Number.isSafeInteger(sum)) result[field] = sum; + else { + // Keep the previous safe aggregate as a lower bound on overflow. + complete = false; + if (field !== "inputTokens" && field !== "outputTokens" && field !== "totalTokens") { + delete result[field]; + overflowedDetails.add(field); + } + } + } + } + // Core overflow can retain a smaller lower bound than independently + // summed details. Keep every published subset bounded by that retained core. + for (const field of [ + "cachedInputTokens", + "cacheCreationInputTokens", + "reasoningOutputTokens", + ] as const) { + const bound = field === "reasoningOutputTokens" ? result.outputTokens : result.inputTokens; + if (result[field] !== undefined && result[field] > bound) { + delete result[field]; + complete = false; + } + } + result.status = + result.countedCallCount === 0 ? "unavailable" : complete ? "complete" : "partial"; + return result; + }, + }; +} diff --git a/electron/ai-edition/document-service.ts b/electron/ai-edition/document-service.ts index 52e1bb97e6..acbc81bd6e 100644 --- a/electron/ai-edition/document-service.ts +++ b/electron/ai-edition/document-service.ts @@ -385,12 +385,21 @@ export class DocumentService { } /** Every project that parses, unrelinked and unannounced: a listing, not an open. */ - private async readAllProjects(): Promise { + async readAllProjects(): Promise { + return (await this.readAllProjectsReport()).documents; + } + + /** + * `readAllProjects`, plus the project files it skipped. A caller deciding whether no project + * uses a file needs those: a project that does not parse may still be using it. + */ + async readAllProjectsReport(): Promise<{ documents: AxcutDocument[]; unreadable: string[] }> { await this.ensureProjectsDir(); const entries = await fs.readdir(this.projectsRoot); // ensureProjectsDir (above) already migrated any legacy `.axcut` files. const projectFiles = entries.filter((name) => name.endsWith(PROJECT_FILE_EXTENSION)); const documents: AxcutDocument[] = []; + const unreadable: string[] = []; for (const name of projectFiles) { const filePath = path.join(this.projectsRoot, name); try { @@ -399,9 +408,10 @@ export class DocumentService { // ponytail: skip unreadable files rather than failing the whole list. // A future migration pass can recover them. console.warn(`[ai-edition] failed to read ${filePath}:`, error); + unreadable.push(filePath); } } - return documents; + return { documents, unreadable }; } /** @@ -577,6 +587,28 @@ export class DocumentService { } } + /** + * Deletes the project once `removeMedia` has run. The file is moved aside first, so a project + * that cannot be deleted fails before anything irreversible, and it comes back if + * `removeMedia` throws. + */ + async deleteProjectAfter(projectId: string, removeMedia: () => Promise): Promise { + const filePath = this.fileFor(projectId); + const parked = `${filePath}.deleting`; + await renameWithRetry(filePath, parked); + try { + await removeMedia(); + } catch (error) { + await renameWithRetry(parked, filePath); + throw error; + } + // The project is gone from every list already: a parked file left behind is only clutter. + await fs + .rm(parked, { force: true }) + .catch((error) => console.warn(`[ai-edition] could not remove ${parked}:`, error)); + await this.deleteProject(projectId); + } + async addAsset(projectId: string, input: AddAssetInput): Promise { const doc = await this.getProject(projectId); const unsupported = unsupportedAssetPath(input); diff --git a/electron/ai-edition/editor-settings-tool.test.ts b/electron/ai-edition/editor-settings-tool.test.ts index d838e95ce6..78907636bb 100644 --- a/electron/ai-edition/editor-settings-tool.test.ts +++ b/electron/ai-edition/editor-settings-tool.test.ts @@ -68,6 +68,7 @@ describe("setEditorSettings", () => { borderRadius: 12, frame: "laptop", audioGainDb: -3, + recordingMuted: true, autoFocusAll: true, cursor: { size: 2, theme: "glass-lens", show: true, asArrow: ["text"] }, }); @@ -80,6 +81,7 @@ describe("setEditorSettings", () => { borderRadius: 12, frame: "laptop", audioGainDb: -3, + recordingMuted: true, autoFocusAll: true, cursorTheme: "glass-lens", }); @@ -87,7 +89,11 @@ describe("setEditorSettings", () => { expect(settings.cursor.asArrow).toEqual(["text"]); const snapshot = read(document); - expect(snapshot.settings).toMatchObject({ padding: 20, aspectRatio: "auto" }); + expect(snapshot.settings).toMatchObject({ + padding: 20, + aspectRatio: "auto", + recordingMuted: true, + }); expect(snapshot.settings.cursor).toMatchObject({ size: 2, theme: "glass-lens" }); // What is read is what the tool takes: the whole block goes back in unchanged. expect(set(document, snapshot.settings).ok).toBe(true); diff --git a/electron/ai-edition/editor-settings-tool.ts b/electron/ai-edition/editor-settings-tool.ts index 4ed6bdf708..9b708b957e 100644 --- a/electron/ai-edition/editor-settings-tool.ts +++ b/electron/ai-edition/editor-settings-tool.ts @@ -130,6 +130,7 @@ export const setEditorSettingsArgs = z webcamWallpaper: wallpaperArg("webcamWallpaper"), webcamBlurIntensity: bounded("webcamBlurIntensity"), audioGainDb: z.number().min(-AUDIO_GAIN_DB_LIMIT).max(AUDIO_GAIN_DB_LIMIT), + recordingMuted: z.boolean(), autoFocusAll: z.boolean(), cursor: z .object({ @@ -191,6 +192,7 @@ function settingsForModel(document: AxcutDocument) { webcamWallpaper: elideDataUrl(s.webcamWallpaper), webcamBlurIntensity: s.webcamBlurIntensity, audioGainDb: s.audioGainDb, + recordingMuted: s.recordingMuted, autoFocusAll: s.autoFocusAll, cursor: { show: s.cursorShow, diff --git a/electron/ai-edition/project-recordings.test.ts b/electron/ai-edition/project-recordings.test.ts new file mode 100644 index 0000000000..06c55e2f0b --- /dev/null +++ b/electron/ai-edition/project-recordings.test.ts @@ -0,0 +1,140 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { DocumentService } from "./document-service"; +import { findProjectRecordings, removeRecordingSets } from "./project-recordings"; + +async function touch(filePath: string, content = "x"): Promise { + await fs.writeFile(filePath, content); + return filePath; +} + +describe("project recordings", () => { + let root: string; + let recordingsDir: string; + let elsewhere: string; + let documents: DocumentService; + + beforeEach(async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), "openscreen-project-recordings-")); + recordingsDir = path.join(root, "recordings"); + elsewhere = path.join(root, "Desktop"); + await fs.mkdir(recordingsDir); + await fs.mkdir(elsewhere); + documents = new DocumentService(path.join(root, "projects"), root); + }); + + afterEach(async () => { + await fs.rm(root, { recursive: true, force: true }); + }); + + async function projectWith(title: string, ...videoPaths: string[]): Promise { + const doc = await documents.createProject(title); + for (const videoPath of videoPaths) { + await documents.addAsset(doc.project.id, { + path: videoPath, + label: path.basename(videoPath), + }); + } + return doc.project.id; + } + + it("owns every file recorded beside the screen video", async () => { + const video = await touch(path.join(recordingsDir, "recording-12.mp4"), "12345"); + const companions = [ + await touch(`${video}.cursor.json`), + await touch(path.join(recordingsDir, "recording-12.session.json")), + await touch(path.join(recordingsDir, "recording-12-webcam.webm")), + ]; + await touch(path.join(recordingsDir, "recording-123.mp4")); + const id = await projectWith("Mine", video); + + const { owned, keptFor } = await findProjectRecordings( + id, + await documents.readAllProjectsReport(), + [recordingsDir], + ); + + expect(owned).toHaveLength(1); + expect(owned[0].files.sort()).toEqual([video, ...companions].sort()); + expect(owned[0].sizeBytes).toBe(5 + 3); + expect(keptFor).toEqual([]); + }); + + it("keeps a recording another project uses, and names that project", async () => { + const shared = await touch(path.join(recordingsDir, "recording-1.mp4")); + const own = await touch(path.join(recordingsDir, "recording-2.mp4")); + const id = await projectWith("Mine", shared, own); + await projectWith("Demo v2", shared); + + const { owned, keptFor } = await findProjectRecordings( + id, + await documents.readAllProjectsReport(), + [recordingsDir], + ); + + expect(owned.map((set) => set.stem)).toEqual(["recording-2"]); + expect(keptFor).toEqual(["Demo v2"]); + }); + + it("keeps every recording while a project file does not parse", async () => { + const video = await touch(path.join(recordingsDir, "recording-9.mp4")); + const id = await projectWith("Mine", video); + // Could be a project using recording-9: a newer schema, or a save cut short. + await touch(path.join(root, "projects", "broken.openscreen"), "{ not json"); + + const { owned, keptFor, unreadable } = await findProjectRecordings( + id, + await documents.readAllProjectsReport(), + [recordingsDir], + ); + + expect(owned).toEqual([]); + expect(keptFor).toEqual([]); + expect(unreadable).toBe(1); + }); + + it("reports no unreadable projects when it has no recordings to keep", async () => { + const imported = await touch(path.join(elsewhere, "clip.mp4")); + const id = await projectWith("Imported", imported); + await touch(path.join(root, "projects", "broken.openscreen"), "{ not json"); + + const { unreadable } = await findProjectRecordings( + id, + await documents.readAllProjectsReport(), + [recordingsDir], + ); + + expect(unreadable).toBe(0); + }); + + it("never owns a video outside the recordings folders", async () => { + const imported = await touch(path.join(elsewhere, "recording-5.mp4")); + const id = await projectWith("Imported", imported); + + const { owned } = await findProjectRecordings(id, await documents.readAllProjectsReport(), [ + recordingsDir, + ]); + + expect(owned).toEqual([]); + }); + + it("trashes each file, or removes it for good when asked", async () => { + const video = await touch(path.join(recordingsDir, "recording-7.mp4")); + const cursor = await touch(`${video}.cursor.json`); + const id = await projectWith("Mine", video); + const { owned } = await findProjectRecordings(id, await documents.readAllProjectsReport(), [ + recordingsDir, + ]); + + const trashItem = vi.fn(async (_filePath: string) => {}); + await removeRecordingSets(owned, "trash", trashItem); + expect(trashItem.mock.calls.map(([p]) => p).sort()).toEqual([video, cursor].sort()); + await expect(fs.access(video)).resolves.toBeUndefined(); + + await removeRecordingSets(owned, "permanent", trashItem); + await expect(fs.access(video)).rejects.toThrow(); + await expect(fs.access(cursor)).rejects.toThrow(); + }); +}); diff --git a/electron/ai-edition/project-recordings.ts b/electron/ai-edition/project-recordings.ts new file mode 100644 index 0000000000..122c7348af --- /dev/null +++ b/electron/ai-edition/project-recordings.ts @@ -0,0 +1,137 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import type { AxcutDocument } from "../../src/lib/ai-edition/schema"; + +const RECORDING_STEM = /^recording-\d+/; + +/** One take: the screen video and every file recorded beside it (cursor, session, camera). */ +export interface RecordingSet { + dir: string; + stem: string; + files: string[]; + sizeBytes: number; +} + +export interface ProjectRecordings { + owned: RecordingSet[]; + /** Titles of the other projects that use some of this project's recordings. */ + keptFor: string[]; + /** + * Project files that did not parse. Any of them may use these recordings, so while there + * are some the project owns none: its recordings are all kept. + */ + unreadable: number; +} + +export type RecordingsDisposal = "keep" | "trash" | "permanent"; + +function documentMediaPaths(doc: AxcutDocument): string[] { + return doc.assets.flatMap((asset) => + [asset.originalPath, asset.proxyPath, asset.waveformPath, asset.cameraTrack?.sourcePath].filter( + (p): p is string => typeof p === "string" && p.length > 0, + ), + ); +} + +// `path.relative` compares case-insensitively on Windows, which a string compare would not. +function sameDir(a: string, b: string): boolean { + return path.relative(path.resolve(a), path.resolve(b)) === ""; +} + +function recordingKey(filePath: string, recordingsDirs: readonly string[]): string | null { + const dir = path.dirname(path.resolve(filePath)); + const managedDir = recordingsDirs.find((candidate) => sameDir(candidate, dir)); + if (!managedDir) return null; + const stem = RECORDING_STEM.exec(path.basename(filePath))?.[0]; + return stem ? path.join(path.resolve(managedDir), stem) : null; +} + +function documentRecordingKeys(doc: AxcutDocument, recordingsDirs: readonly string[]): Set { + const keys = new Set(); + for (const mediaPath of documentMediaPaths(doc)) { + const key = recordingKey(mediaPath, recordingsDirs); + if (key) keys.add(key); + } + return keys; +} + +async function readRecordingSet(key: string): Promise { + const dir = path.dirname(key); + const stem = path.basename(key); + let names: string[]; + try { + names = await fs.readdir(dir); + } catch { + return null; + } + const files: string[] = []; + let sizeBytes = 0; + for (const name of names) { + // `recording-12` must not pick up `recording-123`. + if (name !== stem && !name.startsWith(`${stem}.`) && !name.startsWith(`${stem}-`)) continue; + const filePath = path.join(dir, name); + try { + const stat = await fs.stat(filePath); + if (!stat.isFile()) continue; + files.push(filePath); + sizeBytes += stat.size; + } catch { + // Gone between the listing and the stat: nothing left to delete. + } + } + return files.length > 0 ? { dir, stem, files, sizeBytes } : null; +} + +/** + * The recordings `projectId` owns: takes in one of OpenScreen's recordings folders that no + * other project uses. A video imported from anywhere else is never the project's to delete. + */ +export async function findProjectRecordings( + projectId: string, + projects: { documents: readonly AxcutDocument[]; unreadable: readonly string[] }, + recordingsDirs: readonly string[], +): Promise { + const { documents } = projects; + const target = documents.find((doc) => doc.project.id === projectId); + if (!target) return { owned: [], keptFor: [], unreadable: 0 }; + + const keys = documentRecordingKeys(target, recordingsDirs); + if (keys.size > 0 && projects.unreadable.length > 0) { + return { owned: [], keptFor: [], unreadable: projects.unreadable.length }; + } + const keptFor = new Set(); + for (const doc of documents) { + if (doc.project.id === projectId) continue; + const otherKeys = documentRecordingKeys(doc, recordingsDirs); + for (const key of keys) { + if (otherKeys.has(key)) { + keys.delete(key); + keptFor.add(doc.project.title); + } + } + } + + const owned: RecordingSet[] = []; + for (const key of keys) { + const set = await readRecordingSet(key); + if (set) owned.push(set); + } + return { owned, keptFor: [...keptFor], unreadable: 0 }; +} + +/** Throws on the first file that cannot be removed, so a failed trash can be retried permanently. */ +export async function removeRecordingSets( + sets: readonly RecordingSet[], + disposal: Exclude, + trashItem: (filePath: string) => Promise, +): Promise { + for (const set of sets) { + for (const filePath of set.files) { + if (disposal === "trash") { + await trashItem(filePath); + } else { + await fs.rm(filePath, { force: true }); + } + } + } +} diff --git a/electron/atomic-rename.test.ts b/electron/atomic-rename.test.ts index a6d4d091c4..d1085ef212 100644 --- a/electron/atomic-rename.test.ts +++ b/electron/atomic-rename.test.ts @@ -1,8 +1,17 @@ -import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { + chmodSync, + existsSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { renameSyncWithFallback, renameWithRetry } from "./atomic-rename"; +import { renameSyncWithFallback, renameWithRetry, writeFileReplacing } from "./atomic-rename"; // No test machine has two layers of one folder, so the MSIX virtualization (#1255) is // played by a rename that always answers EXDEV. @@ -68,3 +77,37 @@ describe("atomic rename across an MSIX-virtualized folder", () => { expect(readFileSync(preset, "utf8")).toBe("preset"); }); }); + +// v2.3.0-rc.4 D5. POSIX modes only: Windows has no mode to keep. 664 and 666 also catch a +// mode passed to writeFile, which the umask masks. +describe.skipIf(process.platform === "win32")("writeFileReplacing", () => { + let dir: string; + + beforeEach(() => { + dir = mkdtempSync(path.join(os.tmpdir(), "write-replacing-")); + }); + + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + it.each([ + 0o600, 0o640, 0o664, 0o666, 0o755, + ])("keeps the mode %o of the file it replaces", async (mode) => { + const target = path.join(dir, "out.wav"); + writeFileSync(target, "old"); + chmodSync(target, mode); + await writeFileReplacing(target, Buffer.from("new")); + expect(readFileSync(target, "utf8")).toBe("new"); + expect(statSync(target).mode & 0o777).toBe(mode); + expect(readdirSync(dir)).toEqual(["out.wav"]); + }); + + it("keeps the file it would replace when the write fails", async () => { + const target = path.join(dir, "out.m4a"); + writeFileSync(target, "old"); + await expect(writeFileReplacing(target, null as unknown as Uint8Array)).rejects.toThrow(); + expect(readFileSync(target, "utf8")).toBe("old"); + expect(readdirSync(dir)).toEqual(["out.m4a"]); + }); +}); diff --git a/electron/atomic-rename.ts b/electron/atomic-rename.ts index cde01b225c..822892ffbe 100644 --- a/electron/atomic-rename.ts +++ b/electron/atomic-rename.ts @@ -1,5 +1,7 @@ +import { randomUUID } from "node:crypto"; import { copyFileSync, renameSync, rmSync } from "node:fs"; import fs from "node:fs/promises"; +import path from "node:path"; /** * The Microsoft Store build is an MSIX package: Windows redirects its writes under AppData @@ -54,3 +56,38 @@ export function renameSyncWithFallback(from: string, to: string): void { rmSync(from, { force: true }); } } + +/** + * Replace `destination` with `data` through a staged file renamed over it: a failed write + * leaves the file already there intact (#1104). The staging name is short, so a + * destination name near the 255-byte limit still fits. + * + * The new file keeps the replaced file's mode, and its owner and group as far as we may + * set them, as an in-place write would. The mode is set by chmod, not by `writeFile`'s + * `mode`, which the umask masks (a 664 file came back 644). Best effort: a filesystem + * without POSIX modes must not fail the save. Windows ACLs are not copied. + */ +export async function writeFileReplacing(destination: string, data: Uint8Array): Promise { + await fs.mkdir(path.dirname(destination), { recursive: true }); + const partialPath = path.join(path.dirname(destination), `.openscreen-${randomUUID()}.partial`); + const replaced = await fs.stat(destination).then( + (stats) => (stats.isFile() ? stats : undefined), + () => undefined, + ); + try { + // Private while it is written when it replaces a file, which may be private too. + await fs.writeFile(partialPath, data, { mode: replaced ? 0o600 : 0o666 }); + if (replaced && process.platform !== "win32") { + // Only root may give a file away; the group alone is allowed to any of our groups. + await fs + .chown(partialPath, replaced.uid, replaced.gid) + .catch(() => fs.chown(partialPath, -1, replaced.gid)) + .catch(() => undefined); + await fs.chmod(partialPath, replaced.mode & 0o777).catch(() => undefined); + } + await renameWithRetry(partialPath, destination); + } catch (error) { + await fs.rm(partialPath, { force: true }); + throw error; + } +} diff --git a/electron/auto-updater.test.ts b/electron/auto-updater.test.ts index 3f60a2ec2e..5740f540be 100644 --- a/electron/auto-updater.test.ts +++ b/electron/auto-updater.test.ts @@ -1,3 +1,6 @@ +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import { UUID } from "builder-util-runtime"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { blockedFromInstalling, @@ -5,8 +8,25 @@ import { downloadSelfUpdate, type InstallReadiness, installSelfUpdate, + isPerUserCopy, + NSIS_INSTALL_KEY, } from "./auto-updater"; +const { PER_USER_DIR, PER_USER_EXE } = vi.hoisted(() => { + const dir = "C:\\Users\\Hélène\\AppData\\Local\\Programs\\Openscreen"; + return { PER_USER_DIR: dir, PER_USER_EXE: `${dir}\\Openscreen.exe` }; +}); + +/** Both hives answer; `reg export` writes the per-user location as UTF-16, like reg.exe. */ +function bothInstallsQuery(_command: string, args?: readonly string[]) { + if (args?.[0] === "export") { + const escaped = PER_USER_DIR.replace(/\\/g, "\\\\"); + const body = `Windows Registry Editor Version 5.00\r\n\r\n[HKEY_CURRENT_USER\\${NSIS_INSTALL_KEY}]\r\n"InstallLocation"="${escaped}"\r\n`; + fs.writeFileSync(args[2], `\ufeff${body}`, "utf16le"); + } + return { status: 0 } as ReturnType; +} + const mocks = vi.hoisted(() => ({ // Deliberately initialised to electron-updater's DEFAULTS, so a test asserting they are // false proves our configuration ran rather than reading a value that was never touched. @@ -18,11 +38,38 @@ const mocks = vi.hoisted(() => ({ downloadUpdate: vi.fn(), quitAndInstall: vi.fn(), }, - app: { isPackaged: true, getVersion: vi.fn(() => "1.9.2") }, + app: { + isPackaged: true, + getVersion: vi.fn(() => "1.9.2"), + getPath: vi.fn(() => PER_USER_EXE), + }, + installWindowsUpdate: vi.fn(), + assertWindowsUpdateIdle: vi.fn(), + requestMacMcpShutdown: vi.fn(), })); vi.mock("electron", () => ({ app: mocks.app })); vi.mock("electron-updater", () => ({ autoUpdater: mocks.autoUpdater })); +vi.mock("./mcp/mac-update-shutdown", () => ({ + requestMacMcpShutdown: mocks.requestMacMcpShutdown, +})); +vi.mock("./windows-update", () => ({ + installWindowsUpdate: mocks.installWindowsUpdate, + assertWindowsUpdateIdle: mocks.assertWindowsUpdateIdle, +})); +vi.mock("node:child_process", () => ({ spawnSync: vi.fn() })); + +function missingInstallQuery(_command: string, args?: readonly string[]) { + const root = args?.[1]?.startsWith("HKLM") ? "HKEY_LOCAL_MACHINE" : "HKEY_CURRENT_USER"; + return { + status: args?.includes("/v") ? 1 : 0, + stdout: `\r\n${root}\\Software\\OtherApp\r\n`, + } as ReturnType; +} + +beforeEach(() => { + vi.mocked(spawnSync).mockReset().mockImplementation(missingInstallQuery); +}); function state(overrides: Partial = {}): InstallReadiness { return { recording: false, inApplicationsFolder: true, platform: "linux", ...overrides }; @@ -43,6 +90,12 @@ describe("blockedFromInstalling", () => { } }); + it("blocks a Windows handoff with unsaved editor changes", () => { + expect(blockedFromInstalling(state({ platform: "win32", editorHasUnsavedChanges: true }))).toBe( + "unsaved-changes", + ); + }); + // App Translocation: a quarantined app runs from a read-only image where Squirrel cannot // replace the bundle. Pinned per-platform because CI is Linux-only and an unpinned branch // would be green here and wrong on the one platform it applies to. @@ -77,6 +130,8 @@ describe("self-update flow", () => { mocks.autoUpdater.checkForUpdates.mockReset(); mocks.autoUpdater.downloadUpdate.mockReset(); mocks.autoUpdater.quitAndInstall.mockReset(); + mocks.installWindowsUpdate.mockReset(); + mocks.requestMacMcpShutdown.mockReset(); }); it("never touches the updater on a channel a package manager owns", async () => { @@ -148,11 +203,250 @@ describe("self-update flow", () => { expect(result).toMatchObject({ error: { message: "connection reset" } }); }); - // isSilent=false so a per-machine Windows install can show its UAC prompt — a silent upgrade - // of a Program Files install hits elevation and, if dismissed, quits having done nothing. - // isForceRunAfter=true so the app comes back. - it("hands over to the installer non-silently and relaunches", async () => { - await installSelfUpdate(); - expect(mocks.autoUpdater.quitAndInstall).toHaveBeenCalledWith(false, true); + // Other platforms retain electron-updater's own quit/relaunch handoff. + it("asks macOS MCP sessions to close before handing the update to Squirrel", async () => { + const platform = Object.getOwnPropertyDescriptor(process, "platform"); + Object.defineProperty(process, "platform", { value: "darwin" }); + try { + await installSelfUpdate(); + } finally { + if (platform) Object.defineProperty(process, "platform", platform); + } + expect(mocks.requestMacMcpShutdown).toHaveBeenCalledWith({ + executablePath: process.execPath, + version: "1.9.2", + }); + expect(mocks.requestMacMcpShutdown.mock.invocationCallOrder[0]).toBeLessThan( + mocks.autoUpdater.quitAndInstall.mock.invocationCallOrder[0], + ); + }); + + it("does not close MCP sessions when the install readiness check fails", async () => { + const platform = Object.getOwnPropertyDescriptor(process, "platform"); + Object.defineProperty(process, "platform", { value: "darwin" }); + try { + await installSelfUpdate(() => false); + } finally { + if (platform) Object.defineProperty(process, "platform", platform); + } + expect(mocks.requestMacMcpShutdown).not.toHaveBeenCalled(); + expect(mocks.autoUpdater.quitAndInstall).not.toHaveBeenCalled(); + }); + it("still installs when the MCP shutdown marker cannot be published", async () => { + const platform = Object.getOwnPropertyDescriptor(process, "platform"); + Object.defineProperty(process, "platform", { value: "darwin" }); + const warning = vi.spyOn(console, "warn").mockImplementation(() => undefined); + mocks.requestMacMcpShutdown.mockImplementation(() => { + throw new Error("cache is read-only"); + }); + try { + await expect(installSelfUpdate()).resolves.toBeUndefined(); + expect(warning).toHaveBeenCalledWith( + "[updater] Could not notify MCP sessions of the update:", + expect.any(Error), + ); + expect(mocks.autoUpdater.quitAndInstall).toHaveBeenCalledWith(true, true); + } finally { + if (platform) Object.defineProperty(process, "platform", platform); + warning.mockRestore(); + } + }); + + it("retains quitAndInstall on macOS and Linux", async () => { + const platform = Object.getOwnPropertyDescriptor(process, "platform"); + try { + for (const value of ["darwin", "linux"]) { + Object.defineProperty(process, "platform", { value }); + await installSelfUpdate(); + } + } finally { + if (platform) Object.defineProperty(process, "platform", platform); + } + expect(mocks.autoUpdater.quitAndInstall).toHaveBeenCalledTimes(2); + expect(mocks.autoUpdater.quitAndInstall).toHaveBeenCalledWith(true, true); + expect(mocks.installWindowsUpdate).not.toHaveBeenCalled(); + }); + + // A bare /S run would move the per-user copy to Program Files behind UAC (rc.1 defect). + it.each([ + [PER_USER_EXE, "HKCU"], + ["C:\\Program Files\\Openscreen\\Openscreen.exe", "HKLM"], + ])("pins the running copy (%s) when both installs answer", async (exe, hive) => { + const platform = Object.getOwnPropertyDescriptor(process, "platform"); + Object.defineProperty(process, "platform", { value: "win32" }); + mocks.app.getPath.mockReturnValueOnce(exe); + vi.mocked(spawnSync).mockImplementation(bothInstallsQuery); + const canInstall = () => true; + try { + await installSelfUpdate(canInstall); + } finally { + if (platform) Object.defineProperty(process, "platform", platform); + vi.mocked(spawnSync).mockImplementation(missingInstallQuery); + } + expect(mocks.installWindowsUpdate).toHaveBeenCalledWith(expect.any(Function), canInstall, hive); + expect(mocks.autoUpdater.quitAndInstall).not.toHaveBeenCalled(); + }); + + it("keeps the app open when the per-user location cannot be read", async () => { + const platform = Object.getOwnPropertyDescriptor(process, "platform"); + Object.defineProperty(process, "platform", { value: "win32" }); + vi.mocked(spawnSync).mockReturnValue({ status: 0 } as ReturnType); + try { + await expect(installSelfUpdate()).rejects.toThrow("Could not confirm"); + } finally { + if (platform) Object.defineProperty(process, "platform", platform); + vi.mocked(spawnSync).mockImplementation(missingInstallQuery); + } + expect(mocks.installWindowsUpdate).not.toHaveBeenCalled(); + expect(mocks.autoUpdater.quitAndInstall).not.toHaveBeenCalled(); + }); + + it("waits for the Windows installer without quitting before the UAC decision", async () => { + const platform = Object.getOwnPropertyDescriptor(process, "platform"); + const canInstall = vi.fn(() => true); + let finish: (() => void) | undefined; + mocks.installWindowsUpdate.mockImplementation(async (download: () => Promise) => { + await download(); + await new Promise((resolve) => { + finish = resolve; + }); + }); + mocks.autoUpdater.downloadUpdate.mockResolvedValue(["verified-installer.exe"]); + vi.mocked(spawnSync).mockImplementation((command, args) => + args?.[1]?.startsWith("HKLM") + ? ({ status: 0 } as ReturnType) + : missingInstallQuery(command, args), + ); + Object.defineProperty(process, "platform", { value: "win32" }); + try { + let settled = false; + const install = installSelfUpdate(canInstall).then(() => { + settled = true; + }); + await vi.waitFor(() => expect(mocks.autoUpdater.downloadUpdate).toHaveBeenCalledOnce()); + expect(settled).toBe(false); + expect(mocks.installWindowsUpdate).toHaveBeenCalledWith( + expect.any(Function), + canInstall, + "HKLM", + ); + expect(mocks.autoUpdater.quitAndInstall).not.toHaveBeenCalled(); + finish?.(); + await install; + } finally { + finish?.(); + if (platform) Object.defineProperty(process, "platform", platform); + vi.mocked(spawnSync).mockImplementation(missingInstallQuery); + } + }); + + it("does not request elevation for a per-user Windows update", async () => { + const platform = Object.getOwnPropertyDescriptor(process, "platform"); + Object.defineProperty(process, "platform", { value: "win32" }); + try { + await installSelfUpdate(); + } finally { + if (platform) Object.defineProperty(process, "platform", platform); + } + expect(mocks.autoUpdater.quitAndInstall).toHaveBeenCalledWith(true, true); + expect(mocks.installWindowsUpdate).not.toHaveBeenCalled(); + }); + + it("does not enter a legacy Windows handoff while another elevation is pending", async () => { + const platform = Object.getOwnPropertyDescriptor(process, "platform"); + Object.defineProperty(process, "platform", { value: "win32" }); + mocks.assertWindowsUpdateIdle.mockImplementationOnce(() => { + throw new Error("A Windows update is pending"); + }); + try { + await expect(installSelfUpdate()).rejects.toThrow("Windows update is pending"); + } finally { + if (platform) Object.defineProperty(process, "platform", platform); + } + expect(mocks.autoUpdater.quitAndInstall).not.toHaveBeenCalled(); + expect(mocks.installWindowsUpdate).not.toHaveBeenCalled(); + }); + + it.each(["HKLM", "HKCU"])("keeps the app open when the %s query fails", async (hive) => { + const platform = Object.getOwnPropertyDescriptor(process, "platform"); + Object.defineProperty(process, "platform", { value: "win32" }); + vi.mocked(spawnSync).mockImplementation((command, args) => + args?.[1]?.startsWith(hive) + ? ({ status: null } as ReturnType) + : missingInstallQuery(command, args), + ); + try { + await expect(installSelfUpdate()).rejects.toThrow( + "Could not confirm the Windows installation type", + ); + } finally { + if (platform) Object.defineProperty(process, "platform", platform); + vi.mocked(spawnSync).mockImplementation(missingInstallQuery); + } + expect(mocks.autoUpdater.quitAndInstall).not.toHaveBeenCalled(); + expect(mocks.installWindowsUpdate).not.toHaveBeenCalled(); + }); + + it.each([ + "HKLM", + "HKCU", + ])("does not mistake a failed %s value query for absence", async (hive) => { + const platform = Object.getOwnPropertyDescriptor(process, "platform"); + Object.defineProperty(process, "platform", { value: "win32" }); + vi.mocked(spawnSync).mockImplementation((command, args) => { + if (!args?.[1]?.startsWith(hive)) return missingInstallQuery(command, args); + if (args.includes("/v")) return { status: 1 } as ReturnType; + const root = hive === "HKLM" ? "HKEY_LOCAL_MACHINE" : "HKEY_CURRENT_USER"; + return { + status: 0, + stdout: `${root}\\${NSIS_INSTALL_KEY}\r\n`, + } as ReturnType; + }); + try { + await expect(installSelfUpdate()).rejects.toThrow("Could not confirm"); + } finally { + if (platform) Object.defineProperty(process, "platform", platform); + } + expect(mocks.autoUpdater.quitAndInstall).not.toHaveBeenCalled(); + expect(mocks.installWindowsUpdate).not.toHaveBeenCalled(); + }); + + it("keeps the app open when parent enumeration also fails", async () => { + const platform = Object.getOwnPropertyDescriptor(process, "platform"); + Object.defineProperty(process, "platform", { value: "win32" }); + vi.mocked(spawnSync).mockReturnValue({ status: 1 } as ReturnType); + try { + await expect(installSelfUpdate()).rejects.toThrow("Could not confirm"); + } finally { + if (platform) Object.defineProperty(process, "platform", platform); + } + expect(mocks.autoUpdater.quitAndInstall).not.toHaveBeenCalled(); + expect(mocks.installWindowsUpdate).not.toHaveBeenCalled(); + }); +}); + +describe("isPerUserCopy", () => { + it("matches the running exe to the per-user install, ignoring case and a trailing slash", () => { + expect(isPerUserCopy(PER_USER_EXE, PER_USER_DIR)).toBe(true); + expect(isPerUserCopy(PER_USER_EXE.toUpperCase(), `${PER_USER_DIR}\\`)).toBe(true); + }); + + it("does not mistake the per-machine copy, or a sibling folder, for it", () => { + expect(isPerUserCopy("C:\\Program Files\\Openscreen\\Openscreen.exe", PER_USER_DIR)).toBe( + false, + ); + expect(isPerUserCopy(`${PER_USER_DIR}2\\Openscreen.exe`, PER_USER_DIR)).toBe(false); + }); + + it("refuses to guess without a per-user location", () => { + expect(() => isPerUserCopy(PER_USER_EXE, null)).toThrow("Could not confirm"); + }); + + it("reads the key electron-builder derives from the appId", () => { + const appId = fs + .readFileSync("electron-builder.json5", "utf8") + .match(/"appId": "([^"]+)"/)?.[1]; + const ns = UUID.parse("50e065bc-3134-11e6-9bab-38c9862bdaf3"); // electron-builder's NsisTarget + expect(NSIS_INSTALL_KEY).toBe(`Software\\${UUID.v5(appId ?? "", ns)}`); }); }); diff --git a/electron/auto-updater.ts b/electron/auto-updater.ts index fdee80d2bb..3dd4daaebb 100644 --- a/electron/auto-updater.ts +++ b/electron/auto-updater.ts @@ -8,9 +8,14 @@ // bundle and come along. What does NOT come along is anything under userData — recordings, // projects, and the ~500 MB STT model — which is exactly what we want. +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; import { app } from "electron"; import type { InstallChannel } from "./install-channel"; import { ownsItsUpdates } from "./install-channel"; +import { requestMacMcpShutdown } from "./mcp/mac-update-shutdown"; export type UpdateOutcome = | { kind: "unsupported" } @@ -22,6 +27,8 @@ export type UpdateOutcome = * all" because these are transient — the answer changes as the user works. */ export interface InstallReadiness { recording: boolean; + /** A Windows handoff cannot wait on the editor's close prompt with input disabled. */ + editorHasUnsavedChanges?: boolean; /** macOS only. Gatekeeper runs a quarantined app from a read-only image (App Translocation), * where Squirrel cannot replace the bundle; worse, an app can BECOME translocated after an * update and then never update again. */ @@ -32,6 +39,7 @@ export interface InstallReadiness { /** Pure so the veto can be tested without an updater, a Mac, or a recording. */ export function blockedFromInstalling(state: InstallReadiness): string | null { if (state.recording) return "recording"; + if (state.platform === "win32" && state.editorHasUnsavedChanges) return "unsaved-changes"; if (state.platform === "darwin" && !state.inApplicationsFolder) return "not-in-applications"; return null; } @@ -82,11 +90,83 @@ export async function downloadSelfUpdate(): Promise { } } -/** Quit and hand over to the installer. Callers MUST have checked `blockedFromInstalling`. */ -export async function installSelfUpdate(): Promise { +/** Where electron-builder's NSIS installer records an install: `Software\`, + * under HKLM per machine and HKCU per user. auto-updater.test.ts pins it to the appId. */ +export const NSIS_INSTALL_KEY = "Software\\5b412b41-8d8b-5faf-b557-de4fe681e5fc"; + +function nsisInstallStatus(hive: "HKLM" | "HKCU"): number | null { + const key = `${hive}\\${NSIS_INSTALL_KEY}`; + const args = ["query", key, "/v", "InstallLocation", "/reg:64"]; + const { status } = spawnSync("reg", args, { windowsHide: true }); + if (status !== 1) return status; + // reg's exit 1 means any failure, including access denied. Confirm absence by + // successfully enumerating the parent, without parsing localized error text. + const parent = spawnSync("reg", ["query", `${hive}\\Software`, "/reg:64"], { + windowsHide: true, + encoding: "utf8", + maxBuffer: 16 * 1024 * 1024, + }); + if (parent.status !== 0) return null; + const root = hive === "HKLM" ? "HKEY_LOCAL_MACHINE" : "HKEY_CURRENT_USER"; + const keys = parent.stdout.split(/\r?\n/).map((line) => line.trim().toLowerCase()); + return keys.includes(`${root}\\${NSIS_INSTALL_KEY}`.toLowerCase()) ? null : 1; +} + +/** Where an install lives, from `reg export`: it writes UTF-16, where `reg query` prints in the + * console code page and would garble a non-ASCII user folder. */ +function nsisInstallLocation(hive: "HKLM" | "HKCU"): string | null { + const file = path.join(os.tmpdir(), `openscreen-install-${hive}-${process.pid}.reg`); + try { + const args = ["export", `${hive}\\${NSIS_INSTALL_KEY}`, file, "/y", "/reg:64"]; + if (spawnSync("reg", args, { windowsHide: true }).status !== 0) return null; + const match = fs.readFileSync(file, "utf16le").match(/^"InstallLocation"="(.*)"\r?$/m); + return match ? match[1].replace(/\\(.)/g, "$1") : null; + } catch { + return null; + } finally { + fs.rmSync(file, { force: true }); + } +} + +/** Whether the running exe is the per-user install. Only asked when a per-user and a per-machine + * install coexist: the installer cannot tell which one launched it, so the app pins it. */ +export function isPerUserCopy(exePath: string, perUserLocation: string | null): boolean { + if (perUserLocation === null) { + throw new Error("Could not confirm the Windows installation type. Please retry the update."); + } + // path.win32.relative compares case-insensitively, as Windows does. + return path.win32.relative(perUserLocation, path.win32.dirname(exePath)) === ""; +} + +/** Hand over to the installer. Callers MUST have checked `blockedFromInstalling`. */ +export async function installSelfUpdate(canInstall: () => boolean = () => true): Promise { const autoUpdater = await getUpdater(); - // isSilent=false so a per-machine Windows install can show its elevation prompt: a silent - // upgrade of a Program Files install hits UAC and, if the user dismisses it, quits having - // done nothing. isForceRunAfter=true so the app comes back up. - autoUpdater.quitAndInstall(false, true); + if (process.platform === "win32") { + const { installWindowsUpdate, assertWindowsUpdateIdle } = await import("./windows-update"); + // A concurrent check must not fall back to quitting while another UAC is pending. + assertWindowsUpdateIdle(); + const installed = { HKLM: nsisInstallStatus("HKLM"), HKCU: nsisInstallStatus("HKCU") }; + if (Object.values(installed).some((status) => status !== 0 && status !== 1)) { + throw new Error("Could not confirm the Windows installation type. Please retry the update."); + } + // A bare /S run updates the per-machine copy whenever there is one, so the hive is pinned. + if (installed.HKLM === 0) { + const perUser = + installed.HKCU === 0 && isPerUserCopy(app.getPath("exe"), nsisInstallLocation("HKCU")); + const download = () => autoUpdater.downloadUpdate(); + await installWindowsUpdate(download, canInstall, perUser ? "HKCU" : "HKLM"); + return; + } + } + if (!canInstall()) return; + if (process.platform === "darwin" && app.isPackaged) { + try { + requestMacMcpShutdown({ executablePath: process.execPath, version: app.getVersion() }); + } catch (error) { + console.warn("[updater] Could not notify MCP sessions of the update:", error); + } + } + // Silent: the assisted installer would otherwise show its whole wizard (#1437). + // isForceRunAfter=true so the app comes back up. + autoUpdater.quitAndInstall(true, true); } diff --git a/electron/background-update.test.ts b/electron/background-update.test.ts index 53f92b0c5d..336b81410a 100644 --- a/electron/background-update.test.ts +++ b/electron/background-update.test.ts @@ -1,6 +1,5 @@ import { describe, expect, it, vi } from "vitest"; import { - planBackgroundUpdate, runUnblockedDownloadAndInstall, shouldQuitAndInstallAfterRestartPrompt, shouldStartBackgroundUpdateTimer, @@ -17,31 +16,6 @@ describe("background update policy", () => { expect(shouldStartBackgroundUpdateTimer({ isPackaged: true, ownsItsUpdates: true })).toBe(true); }); - it("does not plan a current-version dialog on the background path", () => { - expect(planBackgroundUpdate({ outcome: { kind: "current" }, mode: "notify" })).toEqual({ - action: "none", - }); - expect( - planBackgroundUpdate({ outcome: { kind: "unsupported" }, mode: "download-and-install" }), - ).toEqual({ action: "none" }); - }); - - it("plans notify / download / download-and-install from an available update", () => { - const outcome = { kind: "downloaded" as const, version: "1.10.0" }; - expect(planBackgroundUpdate({ outcome, mode: "notify" })).toEqual({ - action: "notify-available", - version: "1.10.0", - }); - expect(planBackgroundUpdate({ outcome, mode: "download" })).toEqual({ - action: "download", - version: "1.10.0", - }); - expect(planBackgroundUpdate({ outcome, mode: "download-and-install" })).toEqual({ - action: "download-and-install", - version: "1.10.0", - }); - }); - it("does not call quitAndInstall until Restart Now returns 0", async () => { const install = vi.fn(); const cancelled = await runUnblockedDownloadAndInstall({ diff --git a/electron/background-update.ts b/electron/background-update.ts index ec17e34316..e078f8f8fb 100644 --- a/electron/background-update.ts +++ b/electron/background-update.ts @@ -1,18 +1,22 @@ import type { UpdateOutcome } from "./auto-updater"; -/** How far a background-discovered update may go on its own. No mode ever - * flips `autoInstallOnAppQuit`: install is always an explicit - * `quitAndInstall` behind a restart prompt, because `window-all-closed` - * quits this app and the HUD is a window — install-on-quit would fire a - * ~243 MB installer when the user merely closed the HUD. */ -export type UpdateMode = "notify" | "download" | "download-and-install"; - -export const DEFAULT_UPDATE_MODE: UpdateMode = "notify"; -export const BACKGROUND_UPDATE_INTERVAL_MS = 24 * 60 * 60 * 1000; - -export function parseUpdateMode(raw: unknown): UpdateMode { - if (raw === "notify" || raw === "download" || raw === "download-and-install") return raw; - return DEFAULT_UPDATE_MODE; +// One behaviour, the one Codex and VS Code have: check in the background, download in the +// background, then show an update button. Nothing installs until the user clicks it, so no update +// ever interrupts a take, an export or an agent's `openscreen mcp` session on its own, and no +// install ever asks for an admin password on the way out. + +/** A minute after launch, then every six hours. Most sessions end long before 24 hours, so a check + * that only ran once a day of uptime almost never ran at all. */ +export const BACKGROUND_UPDATE_FIRST_CHECK_MS = 60 * 1000; +export const BACKGROUND_UPDATE_INTERVAL_MS = 6 * 60 * 60 * 1000; + +/** What a background find shows: a button in the HUD and the editor, and a tray item. */ +export interface UpdateBadge { + /** `ready`: downloaded, so the click restarts into it. `available`: the background download + * failed, so the click downloads it again; or the self-updater failed and only the release + * lookup found it, so the click opens the release page. */ + state: "available" | "ready"; + version: string; } export function shouldStartBackgroundUpdateTimer(input: { @@ -22,28 +26,6 @@ export function shouldStartBackgroundUpdateTimer(input: { return input.isPackaged && input.ownsItsUpdates; } -export type BackgroundUpdatePlan = - | { action: "none" } - | { action: "notify-available"; version: string } - | { action: "download"; version: string } - | { action: "download-and-install"; version: string }; - -/** Background path: never plan a "you are current" dialog. */ -export function planBackgroundUpdate(input: { - outcome: UpdateOutcome; - mode: UpdateMode; -}): BackgroundUpdatePlan { - if (input.outcome.kind !== "downloaded") return { action: "none" }; - switch (input.mode) { - case "notify": - return { action: "notify-available", version: input.outcome.version }; - case "download": - return { action: "download", version: input.outcome.version }; - case "download-and-install": - return { action: "download-and-install", version: input.outcome.version }; - } -} - export function shouldQuitAndInstallAfterRestartPrompt(response: number): boolean { return response === 0; } diff --git a/electron/bringToFront.test.ts b/electron/bringToFront.test.ts new file mode 100644 index 0000000000..46d96c8b01 --- /dev/null +++ b/electron/bringToFront.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it, vi } from "vitest"; +import { bringToFront } from "./bringToFront"; + +function fakeWindow(minimized: boolean) { + const calls: string[] = []; + return { + calls, + isMinimized: () => minimized, + restore: vi.fn(() => calls.push("restore")), + show: vi.fn(() => calls.push("show")), + focus: vi.fn(() => calls.push("focus")), + }; +} + +describe("bringToFront", () => { + it("restores a minimized window before showing and focusing it", () => { + const win = fakeWindow(true); + bringToFront(win); + expect(win.calls).toEqual(["restore", "show", "focus"]); + }); + + it("shows and focuses a window that is not minimized, hidden or behind others", () => { + const win = fakeWindow(false); + bringToFront(win); + expect(win.calls).toEqual(["show", "focus"]); + }); +}); diff --git a/electron/bringToFront.ts b/electron/bringToFront.ts new file mode 100644 index 0000000000..e3eb684ccf --- /dev/null +++ b/electron/bringToFront.ts @@ -0,0 +1,13 @@ +import type { BrowserWindow } from "electron"; + +/** + * Brings back a window the user may have minimized or hidden. `focus()` alone does + * neither: a minimized window stays in the taskbar and a hidden one stays hidden. + */ +export function bringToFront( + win: Pick, +) { + if (win.isMinimized()) win.restore(); + win.show(); + win.focus(); +} diff --git a/electron/cameraPreview.test.ts b/electron/cameraPreview.test.ts new file mode 100644 index 0000000000..1c30bb5b03 --- /dev/null +++ b/electron/cameraPreview.test.ts @@ -0,0 +1,493 @@ +import { EventEmitter } from "node:events"; +import type { BrowserWindow, WebContents } from "electron"; +import { describe, expect, it, vi } from "vitest"; +import { CAMERA_PREVIEW_SIZES } from "../src/lib/cameraPreview"; +import { collectMacCaptureExcludedWindowIds } from "../src/lib/nativeMacRecording"; +import { CameraPreviewController } from "./cameraPreview"; + +function setup() { + const owner = Object.assign(new EventEmitter(), { send: vi.fn() }); + const overlay = Object.assign(new EventEmitter(), { + getMediaSourceId: () => "window:123:0", + webContents: { send: vi.fn() }, + showInactive: vi.fn(), + hide: vi.fn(), + destroy: vi.fn(() => { + overlay.emit("closed"); + }), + }); + const create = vi.fn(() => overlay as unknown as BrowserWindow); + const resize = vi.fn(); + const preview = new CameraPreviewController(create, resize); + const sender = owner as unknown as WebContents; + const overlaySender = overlay.webContents as unknown as WebContents; + return { owner, overlay, create, resize, preview, sender, overlaySender }; +} + +const jpeg = "data:image/jpeg;base64,/9j/"; + +describe("floating camera preview lifecycle", () => { + it("reserves the same exclusion ID before the renderer enables the preview", () => { + const { preview, sender, overlay, create } = setup(); + const reserved = preview.ensure(sender); + const excludedIds = collectMacCaptureExcludedWindowIds([reserved!.getMediaSourceId()]); + expect(excludedIds).toContain(123); + overlay.emit("ready-to-show"); + expect(overlay.showInactive).not.toHaveBeenCalled(); + preview.configure(sender, true, true); + expect(preview.ensure(sender)).toBe(reserved); + expect(create).toHaveBeenCalledOnce(); + expect(overlay.showInactive).toHaveBeenCalledOnce(); + expect(excludedIds).toContain(123); + }); + + it("ensuring an existing preview preserves visibility and refuses a different owner", () => { + const { preview, sender, overlay, create } = setup(); + preview.configure(sender, true, true); + overlay.emit("ready-to-show"); + expect(preview.ensure(sender)).toBe(overlay); + expect(preview.ensure({} as WebContents)).toBeNull(); + preview.frame(sender, jpeg); + expect(overlay.webContents.send).toHaveBeenCalledWith("camera-preview:frame", jpeg); + expect(overlay.hide).not.toHaveBeenCalled(); + expect(create).toHaveBeenCalledOnce(); + }); + + it("waits for the window to paint, then shows without taking focus", () => { + const { preview, sender, overlay } = setup(); + preview.configure(sender, true, true); + expect(overlay.showInactive).not.toHaveBeenCalled(); + overlay.emit("ready-to-show"); + expect(overlay.showInactive).toHaveBeenCalledOnce(); + }); + + it("hides and restores the same window, retaining its capture exclusion id", () => { + const { preview, sender, overlaySender, overlay, owner, create } = setup(); + preview.configure(sender, true, true); + overlay.emit("ready-to-show"); + preview.hide(overlaySender); + expect(owner.send).toHaveBeenCalledWith("camera-preview:hidden"); + preview.frame(sender, jpeg); + expect(overlay.webContents.send).not.toHaveBeenCalled(); + preview.configure(sender, true, true); + expect(create).toHaveBeenCalledOnce(); + expect(overlay.destroy).not.toHaveBeenCalled(); + preview.frame(sender, jpeg); + expect(overlay.webContents.send).toHaveBeenCalledWith("camera-preview:frame", jpeg); + }); + + it("does not flash a preview disabled before its renderer is ready", () => { + const { preview, sender, overlay } = setup(); + preview.configure(sender, true, true); + preview.configure(sender, true, false); + overlay.emit("ready-to-show"); + expect(overlay.showInactive).not.toHaveBeenCalled(); + }); + + it("ignores other renderers and rejects oversized or non-JPEG frames", () => { + const { preview, sender, overlay, create } = setup(); + const stranger = {} as WebContents; + preview.configure(sender, true, true); + preview.configure(stranger, false, false); + preview.hide(stranger); + for (const invalid of [undefined, "https://example.com/camera", jpeg + "x".repeat(128_000)]) { + preview.frame(sender, invalid); + } + preview.frame(stranger, jpeg); + expect(overlay.webContents.send).not.toHaveBeenCalled(); + expect(overlay.destroy).not.toHaveBeenCalled(); + expect(create).toHaveBeenCalledOnce(); + }); + + it("allows only the preview to resize to a supported width without replacing its window", () => { + const { preview, sender, overlaySender, resize, create, overlay } = setup(); + preview.configure(sender, true, true); + preview.resize(sender, 320); + preview.resize(overlaySender, Number.NaN); + preview.resize(overlaySender, 10000); + expect(resize).not.toHaveBeenCalled(); + preview.resize(overlaySender, 320); + expect(resize).toHaveBeenCalledWith(overlay, 320, 16 / 9); + expect(create).toHaveBeenCalledOnce(); + expect(overlay.destroy).not.toHaveBeenCalled(); + }); + + it("sizes the same window to the delivered frame and preserves that ratio on resize", () => { + const { preview, sender, overlaySender, resize, overlay } = setup(); + preview.configure(sender, true, true); + preview.frame(sender, jpeg, 9 / 16); + expect(resize).toHaveBeenLastCalledWith(overlay, 224, 9 / 16); + preview.resize(overlaySender, 320); + expect(resize).toHaveBeenLastCalledWith(overlay, 320, 9 / 16); + const calls = resize.mock.calls.length; + for (const ratio of [0, -1, Number.NaN, Number.POSITIVE_INFINITY, 100, "16:9"]) { + preview.frame(sender, jpeg, ratio); + } + expect(resize).toHaveBeenCalledTimes(calls); + }); + + it("recreates an unexpectedly closed preview and waits for the replacement to be ready", () => { + const { preview, sender, overlay, create, owner } = setup(); + const replacement = setup().overlay; + create + .mockReturnValueOnce(overlay as unknown as BrowserWindow) + .mockReturnValueOnce(replacement as unknown as BrowserWindow); + preview.configure(sender, true, true); + overlay.emit("ready-to-show"); + overlay.emit("closed"); + expect(owner.send).toHaveBeenLastCalledWith("camera-preview:hidden"); + preview.frame(sender, jpeg); + expect(overlay.webContents.send).not.toHaveBeenCalled(); + + preview.configure(sender, true, true); + expect(create).toHaveBeenCalledTimes(2); + expect(replacement.showInactive).not.toHaveBeenCalled(); + replacement.emit("ready-to-show"); + expect(replacement.showInactive).toHaveBeenCalledOnce(); + }); + + it("treats system-menu Close as hide, preserving size and capture exclusion", () => { + const { preview, sender, overlay, overlaySender, owner, create, resize } = setup(); + preview.configure(sender, true, true); + overlay.emit("ready-to-show"); + preview.resize(overlaySender, 320); + const event = { preventDefault: vi.fn() }; + overlay.emit("close", event); + expect(event.preventDefault).toHaveBeenCalledOnce(); + expect(overlay.hide).toHaveBeenCalledOnce(); + expect(owner.send).toHaveBeenLastCalledWith("camera-preview:hidden"); + preview.configure(sender, true, true); + expect(preview.ensure(sender)).toBe(overlay); + expect(create).toHaveBeenCalledOnce(); + expect(resize).toHaveBeenLastCalledWith(overlay, 320, 16 / 9); + expect(overlay.showInactive).toHaveBeenCalledTimes(2); + expect(overlay.destroy).not.toHaveBeenCalled(); + // Shutdown uses destroy(), which bypasses the cancellable close event. + preview.dispose(); + expect(overlay.destroy).toHaveBeenCalledOnce(); + }); + + it("ignores a stale closed event from a previously disposed preview", () => { + const { preview, sender, overlay, create } = setup(); + // Delay closure to exercise a late event after the replacement already exists. + overlay.destroy.mockImplementationOnce(() => undefined); + const replacement = setup().overlay; + create + .mockReturnValueOnce(overlay as unknown as BrowserWindow) + .mockReturnValueOnce(replacement as unknown as BrowserWindow); + preview.configure(sender, true, true); + preview.dispose(); + preview.configure(sender, true, true); + replacement.emit("ready-to-show"); + + overlay.emit("closed"); + preview.frame(sender, jpeg); + expect(replacement.webContents.send).toHaveBeenCalledWith("camera-preview:frame", jpeg); + preview.configure(sender, true, true); + expect(create).toHaveBeenCalledTimes(2); + expect(replacement.showInactive).toHaveBeenCalledTimes(2); + }); + + it("destroys the auxiliary window when the HUD is destroyed", () => { + const { preview, sender, owner, overlay } = setup(); + preview.configure(sender, true, true); + owner.send.mockClear(); + owner.emit("destroyed"); + expect(overlay.destroy).toHaveBeenCalledOnce(); + expect(owner.send).not.toHaveBeenCalled(); + preview.frame(sender, jpeg); + expect(overlay.webContents.send).not.toHaveBeenCalled(); + }); + + it("creates a hidden exclusion window even when the camera starts off", () => { + const { preview, sender, owner, overlay, create } = setup(); + preview.configure(sender, false, true); + overlay.emit("ready-to-show"); + expect(create).toHaveBeenCalledOnce(); + expect(overlay.hide).toHaveBeenCalledOnce(); + expect(overlay.showInactive).not.toHaveBeenCalled(); + preview.frame(sender, jpeg); + expect(overlay.webContents.send).not.toHaveBeenCalled(); + preview.configure(sender, true, true); + expect(create).toHaveBeenCalledOnce(); + expect(overlay.showInactive).toHaveBeenCalledOnce(); + owner.emit("destroyed"); + expect(overlay.destroy).toHaveBeenCalledOnce(); + }); + + it("retains the exclusion window and one owner listener across camera toggles", () => { + const { preview, sender, owner, overlay, create } = setup(); + for (let i = 0; i < 15; i++) { + preview.configure(sender, true, true); + if (i === 0) overlay.emit("ready-to-show"); + preview.configure(sender, false, true); + } + expect(create).toHaveBeenCalledOnce(); + expect(overlay.hide).toHaveBeenCalledTimes(15); + expect(overlay.showInactive).toHaveBeenCalledTimes(15); + expect(overlay.destroy).not.toHaveBeenCalled(); + expect(owner.listenerCount("destroyed")).toBe(1); + owner.emit("destroyed"); + expect(owner.listenerCount("destroyed")).toBe(0); + expect(overlay.destroy).toHaveBeenCalledOnce(); + }); + it("blocks preview display and frames during a monitor take even across HUD toggles", () => { + const { preview, sender, overlay, owner } = setup(); + preview.configure(sender, true, true); + overlay.emit("ready-to-show"); + preview.setCaptureAllowed(false); + expect(owner.send).toHaveBeenLastCalledWith("camera-preview:availability", false); + preview.configure(sender, true, true); + preview.frame(sender, jpeg); + expect(overlay.showInactive).toHaveBeenCalledOnce(); + expect(overlay.webContents.send).not.toHaveBeenCalled(); + preview.setCaptureAllowed(true); + expect(overlay.showInactive).toHaveBeenCalledTimes(2); + preview.frame(sender, jpeg); + expect(overlay.webContents.send).toHaveBeenCalledWith("camera-preview:frame", jpeg); + }); + + it("restores a cancelled start without restoring an older or active capture", () => { + const { preview, sender, overlay, owner } = setup(); + preview.configure(sender, true, true); + overlay.emit("ready-to-show"); + const oldToken = preview.prepareCapture(); + const newToken = preview.prepareCapture(); + preview.restoreCapture(oldToken); + expect(overlay.showInactive).toHaveBeenCalledOnce(); + preview.restoreCapture(newToken); + expect(overlay.showInactive).toHaveBeenCalledTimes(2); + expect(owner.send).toHaveBeenLastCalledWith("camera-preview:availability", true); + const token = preview.prepareCapture(); + preview.setCaptureAllowed(false); // A newer native capture owns the suspension. + preview.restoreCapture(token); + expect(overlay.showInactive).toHaveBeenCalledTimes(2); + }); + + function setupWithHud(hudVisible = true) { + const owner = Object.assign(new EventEmitter(), { send: vi.fn() }); + const hud = Object.assign(new EventEmitter(), { + isVisible: () => hudVisible, + isMinimized: () => false, + }); + const overlay = Object.assign(new EventEmitter(), { + webContents: { send: vi.fn() }, + showInactive: vi.fn(), + hide: vi.fn(), + destroy: vi.fn(), + }); + const preview = new CameraPreviewController( + () => overlay as unknown as BrowserWindow, + vi.fn(), + undefined, + () => hud as unknown as BrowserWindow, + ); + return { owner, hud, overlay, preview, sender: owner as unknown as WebContents }; + } + + it("stays hidden when the HUD is already hidden at the first configure", () => { + const { hud, overlay, preview, sender } = setupWithHud(false); + preview.configure(sender, true, true); + overlay.emit("ready-to-show"); + expect(overlay.showInactive).not.toHaveBeenCalled(); + hud.emit("show"); + expect(overlay.showInactive).toHaveBeenCalledOnce(); + }); + + it("removes its HUD listeners on dispose", () => { + const { hud, preview, sender } = setupWithHud(); + preview.ensure(sender); + expect(hud.listenerCount("minimize")).toBe(1); + preview.dispose(); + for (const event of ["minimize", "hide", "restore", "show"]) { + expect(hud.listenerCount(event)).toBe(0); + } + }); + + it("hides with the HUD (Hide recording bar) and comes back with it", () => { + const owner = Object.assign(new EventEmitter(), { send: vi.fn() }); + const hud = Object.assign(new EventEmitter(), { + isVisible: () => true, + isMinimized: () => false, + }); + const overlay = Object.assign(new EventEmitter(), { + webContents: { send: vi.fn() }, + showInactive: vi.fn(), + hide: vi.fn(), + }); + const preview = new CameraPreviewController( + () => overlay as unknown as BrowserWindow, + vi.fn(), + undefined, + () => hud as unknown as BrowserWindow, + ); + const sender = owner as unknown as WebContents; + preview.configure(sender, true, true); + overlay.emit("ready-to-show"); + expect(overlay.showInactive).toHaveBeenCalledOnce(); + + hud.emit("minimize"); + expect(overlay.hide).toHaveBeenCalledOnce(); + preview.configure(sender, true, true); // A HUD re-render must not bring it back early. + expect(overlay.showInactive).toHaveBeenCalledOnce(); + hud.emit("restore"); + expect(overlay.showInactive).toHaveBeenCalledTimes(2); + + hud.emit("hide"); // Linux hides instead of minimizing. + hud.emit("show"); + expect(overlay.showInactive).toHaveBeenCalledTimes(3); + + preview.configure(sender, false, false); + hud.emit("minimize"); + hud.emit("restore"); + expect(overlay.showInactive).toHaveBeenCalledTimes(3); // Camera off stays off. + }); + + it("does not flash a blocked preview on ready or restore one the user hid", () => { + const { preview, sender, overlay, overlaySender } = setup(); + preview.setCaptureAllowed(false); + preview.configure(sender, true, true); + overlay.emit("ready-to-show"); + expect(overlay.showInactive).not.toHaveBeenCalled(); + preview.hide(overlaySender); + preview.setCaptureAllowed(true); + expect(overlay.showInactive).not.toHaveBeenCalled(); + }); +}); + +describe("floating camera preview parking (Wayland)", () => { + function parkingSetup() { + const base = setup(); + let mapped = false; + Object.assign(base.overlay, { + isVisible: () => mapped, + showInactive: vi.fn(() => { + mapped = true; + }), + hide: vi.fn(() => { + mapped = false; + }), + }); + const park = vi.fn(); + const preview = new CameraPreviewController(base.create, base.resize, park); + return { ...base, preview, park }; + } + + it("parks a shown preview instead of unmapping it, and restores its size on show", () => { + // An unmapped Wayland window is placed anew by the compositor when it comes back, so + // hiding lost wherever the user had dragged the preview. + const { preview, sender, overlay, overlaySender, park, resize } = parkingSetup(); + preview.configure(sender, true, true); + overlay.emit("ready-to-show"); + resize.mockClear(); + + preview.hide(overlaySender); + preview.configure(sender, true, false); + expect(park).toHaveBeenCalledOnce(); + expect(overlay.hide).not.toHaveBeenCalled(); + + preview.configure(sender, true, true); + expect(resize).toHaveBeenCalledOnce(); + expect(overlay.showInactive).toHaveBeenCalledTimes(2); + }); + + it("parks for a capture that cannot exclude it too", () => { + const { preview, sender, overlay, park, resize } = parkingSetup(); + preview.configure(sender, true, true); + overlay.emit("ready-to-show"); + resize.mockClear(); + const token = preview.prepareCapture(); + expect(park).toHaveBeenCalledOnce(); + preview.restoreCapture(token); + expect(resize).toHaveBeenCalledOnce(); + }); + + it("keeps a parked preview parked when a resize arrives, and applies it on show", () => { + const { preview, sender, overlay, overlaySender, resize } = parkingSetup(); + preview.configure(sender, true, true); + overlay.emit("ready-to-show"); + resize.mockClear(); + preview.prepareCapture(); + preview.resize(overlaySender, CAMERA_PREVIEW_SIZES[0]); + expect(resize).not.toHaveBeenCalled(); + preview.setCaptureAllowed(true); + expect(resize).toHaveBeenCalledWith(overlay, CAMERA_PREVIEW_SIZES[0], expect.any(Number)); + }); + + it("parks, rather than unmaps, the preview when the recording bar is hidden", () => { + const { create, resize, overlay, sender } = parkingSetup(); + const park = vi.fn(); + const hud = Object.assign(new EventEmitter(), { + isVisible: () => true, + isMinimized: () => false, + }); + const preview = new CameraPreviewController( + create, + resize, + park, + () => hud as unknown as BrowserWindow, + ); + preview.configure(sender, true, true); + overlay.emit("ready-to-show"); + resize.mockClear(); + + hud.emit("hide"); // Linux hides the HUD instead of minimizing it. + expect(park).toHaveBeenCalledOnce(); + expect(overlay.hide).not.toHaveBeenCalled(); + hud.emit("show"); + expect(resize).toHaveBeenCalledOnce(); + expect(overlay.showInactive).toHaveBeenCalledTimes(2); + }); + + it("keeps the preview parked for the next HUD across an editor round trip", () => { + // The HUD closes when the editor opens. A preview destroyed with it was placed anew by + // the compositor for the next HUD, top-left on GNOME, wherever the user had dragged it. + const { preview, sender, owner, overlay, create, park, resize } = parkingSetup(); + preview.configure(sender, true, true); + overlay.emit("ready-to-show"); + resize.mockClear(); + + owner.emit("destroyed"); + expect(overlay.destroy).not.toHaveBeenCalled(); + expect(park).toHaveBeenCalledOnce(); + preview.frame(sender, jpeg); + expect(overlay.webContents.send).not.toHaveBeenCalled(); + + const nextHud = Object.assign(new EventEmitter(), { send: vi.fn() }); + const nextSender = nextHud as unknown as WebContents; + expect(preview.ensure(nextSender)).toBe(overlay); + expect(preview.ensure(sender)).toBeNull(); + preview.configure(nextSender, true, true); + expect(create).toHaveBeenCalledOnce(); + expect(resize).toHaveBeenCalledOnce(); + expect(overlay.showInactive).toHaveBeenCalledTimes(2); + preview.frame(nextSender, jpeg); + expect(overlay.webContents.send).toHaveBeenCalledWith("camera-preview:frame", jpeg); + + // The next HUD's own destruction is still heard. + nextHud.emit("destroyed"); + expect(park).toHaveBeenCalledTimes(2); + }); + + it("keeps the size the user picked across the round trip", () => { + const { preview, sender, owner, overlay, overlaySender, resize } = parkingSetup(); + preview.configure(sender, true, true); + overlay.emit("ready-to-show"); + preview.resize(overlaySender, CAMERA_PREVIEW_SIZES[0]); + owner.emit("destroyed"); + resize.mockClear(); + + const nextSender = Object.assign(new EventEmitter(), { + send: vi.fn(), + }) as unknown as WebContents; + preview.configure(nextSender, true, true); + expect(resize).toHaveBeenCalledWith(overlay, CAMERA_PREVIEW_SIZES[0], expect.any(Number)); + }); + + it("hides a preview that was never shown: there is no position to keep", () => { + const { preview, sender, overlay, park } = parkingSetup(); + preview.configure(sender, true, false); + expect(park).not.toHaveBeenCalled(); + expect(overlay.hide).toHaveBeenCalled(); + }); +}); diff --git a/electron/cameraPreview.ts b/electron/cameraPreview.ts new file mode 100644 index 0000000000..663fc4914e --- /dev/null +++ b/electron/cameraPreview.ts @@ -0,0 +1,237 @@ +import type { BrowserWindow, WebContents } from "electron"; +import { + CAMERA_PREVIEW_DEFAULT_ASPECT_RATIO, + CAMERA_PREVIEW_DEFAULT_SIZE, + CAMERA_PREVIEW_SIZES, +} from "../src/lib/cameraPreview"; + +/** One stable window per HUD: hiding it must not change the native capture exclusion id. */ +export class CameraPreviewController { + private window: BrowserWindow | null = null; + private owner: WebContents | null = null; + private visible = false; + private captureAllowed = true; + /** False while the HUD is minimized or hidden ("Hide recording bar"): the preview goes with it. */ + private ownerShown = true; + private captureGeneration = 0; + private width: number = CAMERA_PREVIEW_DEFAULT_SIZE; + private aspectRatio = CAMERA_PREVIEW_DEFAULT_ASPECT_RATIO; + private ready = false; + private onOwnerDestroyed: (() => void) | null = null; + private parked = false; + private hud: BrowserWindow | null = null; + private hudListeners: Array<[string, () => void]> = []; + + /** + * `parkWindow`, when given, stands in for hiding a preview already on screen. Wayland needs + * it: a client cannot position its window there, so a hidden preview is placed anew by the + * compositor when it comes back (top-left on GNOME, over the HUD), losing wherever the user + * had dragged it. A parked window stays mapped where it was and draws nothing. + */ + constructor( + private readonly createWindow: () => BrowserWindow, + private readonly resizeWindow: ( + window: BrowserWindow, + width: number, + aspectRatio: number, + ) => void, + private readonly parkWindow?: (window: BrowserWindow) => void, + private readonly windowOf: (owner: WebContents) => BrowserWindow | null = () => null, + ) {} + + private shouldShow(): boolean { + return this.visible && this.captureAllowed && this.ownerShown; + } + + private conceal(window: BrowserWindow): void { + if (this.parkWindow && window.isVisible()) { + if (!this.parked) this.parkWindow(window); + this.parked = true; + } else { + window.hide(); + } + } + + private reveal(window: BrowserWindow): void { + if (this.parked) { + this.parked = false; + this.resizeWindow(window, this.width, this.aspectRatio); + } + window.showInactive(); + } + + /** Reserve the capture-exclusion window before renderer configuration arrives. */ + ensure(owner: WebContents): BrowserWindow | null { + if (this.owner && this.owner !== owner) return null; + if (!this.owner) { + this.owner = owner; + owner.send("camera-preview:availability", this.captureAllowed); + this.onOwnerDestroyed = () => { + if (this.owner !== owner) return; + // Wayland keeps the window for the next HUD, so an editor round trip does not + // hand it back to the compositor to place anew. + if (this.parkWindow) this.release(); + else this.dispose(); + }; + owner.once("destroyed", this.onOwnerDestroyed); + // Windows and macOS minimize the HUD, Linux hides it; the tray brings it back. + const hud = this.windowOf(owner); + if (hud) { + this.hud = hud; + this.ownerShown = hud.isVisible() && !hud.isMinimized(); + const hidden = () => this.setOwnerShown(false); + const shown = () => this.setOwnerShown(true); + this.hudListeners = [ + ["minimize", hidden], + ["hide", hidden], + ["restore", shown], + ["show", shown], + ]; + for (const [event, listener] of this.hudListeners) hud.on(event as "show", listener); + } + } + if (!this.window) { + this.window = this.createWindow(); + const previewWindow = this.window; + // A system-menu close is the same action as the preview's hide button. + // Retain the native ID for capture exclusions and the renderer's size state. + previewWindow.on("close", (event) => { + event.preventDefault(); + this.hide(previewWindow.webContents); + }); + previewWindow.once("closed", () => { + if (this.window !== previewWindow) return; + this.window = null; + this.ready = false; + this.visible = false; + this.parked = false; + this.owner?.send("camera-preview:hidden"); + }); + this.window.once("ready-to-show", () => { + if (this.window !== previewWindow) return; + this.ready = true; + this.resizeWindow(previewWindow, this.width, this.aspectRatio); + if (this.shouldShow()) this.reveal(previewWindow); + }); + } + return this.window; + } + + configure(owner: WebContents, enabled: boolean, visible: boolean): void { + const previewWindow = this.ensure(owner); + if (!previewWindow) return; + this.visible = enabled && visible; + if (this.shouldShow() && this.ready) this.reveal(previewWindow); + else if (!this.shouldShow()) this.conceal(previewWindow); + } + + /** Linux portal monitor captures cannot exclude app windows. */ + setCaptureAllowed(allowed: boolean): void { + this.captureGeneration++; + this.captureAllowed = allowed; + const previewWindow = this.window; + if (previewWindow && !allowed) this.conceal(previewWindow); + else if (previewWindow && this.shouldShow() && this.ready) this.reveal(previewWindow); + this.owner?.send("camera-preview:availability", allowed); + } + + setOwnerShown(shown: boolean): void { + if (this.ownerShown === shown) return; + this.ownerShown = shown; + const previewWindow = this.window; + if (previewWindow && !shown) this.conceal(previewWindow); + else if (previewWindow && this.shouldShow() && this.ready) this.reveal(previewWindow); + } + + /** A cancelled browser start may restore only its own preview suspension. */ + prepareCapture(): number { + this.setCaptureAllowed(false); + return this.captureGeneration; + } + + restoreCapture(token: unknown): void { + if (token === this.captureGeneration) this.setCaptureAllowed(true); + } + + hide(sender: WebContents): void { + if (sender !== this.window?.webContents) return; + this.visible = false; + this.conceal(this.window); + this.owner?.send("camera-preview:hidden"); + } + + resize(sender: WebContents, width: unknown): void { + if ( + sender !== this.window?.webContents || + typeof width !== "number" || + !CAMERA_PREVIEW_SIZES.some((size) => size === width) + ) + return; + this.width = width; + // A parked preview takes its new width when it comes back (reveal). + if (!this.parked) this.resizeWindow(this.window, this.width, this.aspectRatio); + } + + frame(sender: WebContents, jpeg: unknown, aspectRatio?: unknown): void { + if (sender !== this.owner || !this.visible || !this.captureAllowed || !this.window) return; + // A 320px JPEG is usually under 20KB. Never relay arbitrary URLs or unbounded IPC data. + if ( + typeof jpeg !== "string" || + jpeg.length > 128_000 || + !jpeg.startsWith("data:image/jpeg;base64,") + ) + return; + if (aspectRatio !== undefined) { + if ( + typeof aspectRatio !== "number" || + !Number.isFinite(aspectRatio) || + aspectRatio < 1 / 8 || + aspectRatio > 8 + ) + return; + if (this.aspectRatio !== aspectRatio) { + this.aspectRatio = aspectRatio; + this.resizeWindow(this.window, this.width, aspectRatio); + } + } + this.window.webContents.send("camera-preview:frame", jpeg); + } + + private detachOwner(): void { + if (this.owner && this.onOwnerDestroyed) { + this.owner.removeListener("destroyed", this.onOwnerDestroyed); + } + this.onOwnerDestroyed = null; + for (const [event, listener] of this.hudListeners) { + this.hud?.removeListener(event as "show", listener); + } + this.hud = null; + this.hudListeners = []; + this.owner = null; + this.visible = false; + this.captureAllowed = true; + this.ownerShown = true; + this.captureGeneration++; + } + + /** + * The HUD is gone (the editor opened) but its window stays, parked where the user left it, + * with its size, for the next HUD to adopt: the window outlives HUDs only where it is parked. + * main.ts destroys it if it is all that is left open, so it never holds the app up. + */ + private release(): void { + if (this.window) this.conceal(this.window); + this.detachOwner(); + } + + dispose(): void { + this.detachOwner(); + this.ready = false; + this.parked = false; + const previewWindow = this.window; + this.window = null; + previewWindow?.destroy(); + this.width = CAMERA_PREVIEW_DEFAULT_SIZE; + this.aspectRatio = CAMERA_PREVIEW_DEFAULT_ASPECT_RATIO; + } +} diff --git a/electron/cli/cliMain.test.ts b/electron/cli/cliMain.test.ts index 85ef7e3528..fa22d152c2 100644 --- a/electron/cli/cliMain.test.ts +++ b/electron/cli/cliMain.test.ts @@ -2,17 +2,34 @@ // counts as active. runCli runs for real here; only Electron and the heavy services around // the MCP server are stubbed. -import { mkdtempSync, rmSync } from "node:fs"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import os from "node:os"; import path from "node:path"; +import { app } from "electron"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { sendUsagePing, setUsagePingEnabled } from "../usage-ping"; import { runCli } from "./cliMain"; -const mocks = vi.hoisted(() => ({ userData: "", serve: vi.fn(async () => undefined) })); +const mocks = vi.hoisted(() => ({ + userData: "", + isPackaged: false, + channel: "nsis", + serve: vi.fn(async (_options: { updateNotice?: string; onClose: () => void }) => ({ + isBusy: (): boolean => mocks.mediaBusy, + })), + mediaBusy: false, + netFetch: vi.fn(), + watchUpdate: vi.fn((_identity: { executablePath: string; version: string }, _close: () => void) => + vi.fn(), + ), + shutdownStt: vi.fn(async () => undefined), +})); vi.mock("electron", () => ({ app: { + get isPackaged() { + return mocks.isPackaged; + }, getPath: () => mocks.userData, getVersion: () => "2.1.0", exit: vi.fn(), @@ -23,6 +40,7 @@ vi.mock("electron", () => ({ BrowserWindow: class {}, ipcMain: { handle: vi.fn() }, Menu: { setApplicationMenu: vi.fn(), buildFromTemplate: vi.fn() }, + net: { fetch: mocks.netFetch }, session: { defaultSession: { setPermissionCheckHandler: vi.fn(), @@ -32,14 +50,19 @@ vi.mock("electron", () => ({ }, systemPreferences: {}, })); -vi.mock("../install-channel", () => ({ getInstallChannel: () => "nsis" })); +// The real `ownsItsUpdates`, so the notice tests follow the channel rule rather than a copy of it. +vi.mock("../install-channel", async (importOriginal) => ({ + ...(await importOriginal()), + getInstallChannel: () => mocks.channel, +})); vi.mock("../ipc/handlers", () => ({ getSelectedDesktopSource: () => null, registerIpcHandlers: () => ({ agentServices: {} }), })); vi.mock("../mcp/hidden-video", () => ({ openVideoInHiddenWindow: vi.fn() })); vi.mock("../mcp/stdio-server", () => ({ serveMcpOverStdio: mocks.serve })); -vi.mock("../stt", () => ({ registerSttIpc: vi.fn(), shutdownStt: vi.fn() })); +vi.mock("../mcp/mac-update-shutdown", () => ({ watchForMacMcpShutdown: mocks.watchUpdate })); +vi.mock("../stt", () => ({ registerSttIpc: vi.fn(), shutdownStt: mocks.shutdownStt })); vi.mock("../windows", () => ({ createRunnerWindow: vi.fn() })); vi.mock("./runnerHost", () => ({ createRunnerHost: () => ({}) })); vi.mock("./sessionData", () => ({ isolateCliSessionData: vi.fn() })); @@ -47,10 +70,21 @@ vi.mock("./sessionData", () => ({ isolateCliSessionData: vi.fn() })); const fetchMock = vi.fn(async (_url: string, _init: RequestInit) => ({ ok: true })); const savedConsole = { ...console }; let stdoutWrite: ReturnType; +let streamErrorListeners: Map>; beforeEach(() => { + streamErrorListeners = new Map( + [process.stdout, process.stderr].map((stream) => [stream, new Set(stream.listeners("error"))]), + ); mocks.userData = mkdtempSync(path.join(os.tmpdir(), "cli-usage-ping-")); + mocks.isPackaged = false; + mocks.channel = "nsis"; mocks.serve.mockClear(); + mocks.mediaBusy = false; + mocks.watchUpdate.mockClear(); + mocks.shutdownStt.mockClear(); + vi.mocked(app.exit).mockClear(); + mocks.netFetch.mockReset(); fetchMock.mockClear(); vi.stubGlobal("fetch", fetchMock); vi.stubEnv("DO_NOT_TRACK", ""); @@ -63,6 +97,13 @@ beforeEach(() => { }); afterEach(() => { + for (const [stream, listeners] of streamErrorListeners) { + for (const listener of stream.listeners("error")) { + if (!listeners.has(listener)) { + stream.removeListener("error", listener as (error: Error) => void); + } + } + } vi.unstubAllGlobals(); vi.unstubAllEnvs(); vi.restoreAllMocks(); @@ -107,3 +148,90 @@ describe("openscreen mcp usage ping", () => { expect(fetchMock).not.toHaveBeenCalled(); }); }); + +describe("macOS MCP update shutdown", () => { + it("refuses update-triggered exit until an active media job is finalized", async () => { + mocks.isPackaged = true; + mocks.channel = "dmg"; + mocks.mediaBusy = true; + vi.spyOn(process, "platform", "get").mockReturnValue("darwin"); + await runMcp(); + expect(mocks.watchUpdate.mock.calls[0][1]()).toBe(false); + expect(mocks.shutdownStt).not.toHaveBeenCalled(); + expect(app.exit).not.toHaveBeenCalled(); + mocks.mediaBusy = false; + expect(mocks.watchUpdate.mock.calls[0][1]()).toBe(true); + await Promise.resolve(); + expect(mocks.shutdownStt).toHaveBeenCalledOnce(); + expect(app.exit).toHaveBeenCalledWith(0); + }); + it("watches packaged macOS sessions and shuts the server down once", async () => { + mocks.isPackaged = true; + mocks.channel = "dmg"; + vi.spyOn(process, "platform", "get").mockReturnValue("darwin"); + await runMcp(); + expect(mocks.watchUpdate).toHaveBeenCalledWith( + { executablePath: process.execPath, version: "2.1.0" }, + expect.any(Function), + ); + mocks.watchUpdate.mock.calls[0][1](); + mocks.serve.mock.calls[0][0].onClose(); + await Promise.resolve(); + expect(mocks.shutdownStt).toHaveBeenCalledOnce(); + expect(app.exit).toHaveBeenCalledWith(0); + expect(mocks.watchUpdate.mock.results[0].value).toHaveBeenCalledOnce(); + }); + + it.each(["linux", "win32"] as const)("does not watch %s sessions", async (platform) => { + mocks.isPackaged = true; + vi.spyOn(process, "platform", "get").mockReturnValue(platform); + await runMcp(); + expect(mocks.watchUpdate).not.toHaveBeenCalled(); + }); + + it("does not watch macOS development sessions", async () => { + vi.spyOn(process, "platform", "get").mockReturnValue("darwin"); + await runMcp(); + expect(mocks.watchUpdate).not.toHaveBeenCalled(); + }); +}); + +// The user who only reaches OpenScreen through an agent: the notice the cache holds has to reach +// the server it starts, or it is never said. +describe("openscreen mcp update notice", () => { + function cacheLookup(latestVersion: string, checkedAt = Date.now()) { + writeFileSync( + path.join(mocks.userData, "mcp-update-check.json"), + JSON.stringify({ latestVersion, checkedAt }), + ); + } + + it("hands the cached newer version to the server on an install that updates itself", async () => { + mocks.isPackaged = true; + cacheLookup("9.9.9"); + await runMcp(); + expect(mocks.serve.mock.calls[0][0].updateNotice).toContain( + "OpenScreen 9.9.9 is available; this is 2.1.0.", + ); + // The cache is fresh, so the background lookup did not go out. + expect(mocks.netFetch).not.toHaveBeenCalled(); + }); + + it("says nothing from a dev build, which a package manager may own", async () => { + cacheLookup("9.9.9"); + await runMcp(); + expect(mocks.serve.mock.calls[0][0].updateNotice).toBeUndefined(); + }); + + // The Store updates its copy: telling the user to open the app would offer a button that is not + // there, and looking the release up would be wasted. + it("says nothing, and looks nothing up, on a Store install", async () => { + mocks.isPackaged = true; + mocks.channel = "store"; + // Stale, so a lookup started by mistake would reach the network and fail the assertion. + cacheLookup("9.9.9", 0); + await runMcp(); + expect(mocks.serve.mock.calls[0][0].updateNotice).toBeUndefined(); + expect(mocks.netFetch).not.toHaveBeenCalled(); + }); +}); diff --git a/electron/cli/cliMain.ts b/electron/cli/cliMain.ts index 410ba97458..b8d600a65b 100644 --- a/electron/cli/cliMain.ts +++ b/electron/cli/cliMain.ts @@ -5,19 +5,21 @@ import fs from "node:fs/promises"; import path from "node:path"; import readline from "node:readline"; -import { app, BrowserWindow, ipcMain, Menu, session, systemPreferences } from "electron"; +import { app, BrowserWindow, ipcMain, Menu, net, session, systemPreferences } from "electron"; import type { CliProgressEvent, CliSourcesResult } from "../../src/lib/cliContracts"; import { renameWithRetry } from "../atomic-rename"; -import { isDiagnosticModeEnabled } from "../diagnostics/main-log-buffer"; -import { getInstallChannel } from "../install-channel"; +import { isDiagnosticModeEnabled } from "../diagnostics/main-log-file"; +import { getInstallChannel, ownsItsUpdates } from "../install-channel"; import { getSelectedDesktopSource, registerIpcHandlers } from "../ipc/handlers"; import { openVideoInHiddenWindow } from "../mcp/hidden-video"; +import { watchForMacMcpShutdown } from "../mcp/mac-update-shutdown"; import { serveMcpOverStdio } from "../mcp/stdio-server"; +import { readReleaseLookup, refreshReleaseLookup, updateNotice } from "../mcp/update-notice"; import { registerSttIpc, shutdownStt } from "../stt"; import { sendUsagePing, USAGE_PING_INTERVAL_MS } from "../usage-ping"; import { createRunnerWindow } from "../windows"; import { CLI_USAGE, type CliCommand } from "./args"; -import { openGuiFromCli } from "./openGui"; +import { onCliActivate } from "./openGui"; import { runInfoCommand, runPackCommand } from "./projectCommands"; import { createRunnerHost } from "./runnerHost"; import { isolateCliSessionData } from "./sessionData"; @@ -281,7 +283,7 @@ export function runCli(command: CliCommand): void { // to this process on to the GUI (see openGui.ts). if (process.platform === "darwin") { app.dock?.hide(); - app.on("activate", () => openGuiFromCli(process.execPath, Date.now(), () => app.dock?.hide())); + app.on("activate", () => onCliActivate(app.dock)); // No application menu either. That menu shows whenever this hidden process comes to the // front, as it does while it hands a launch on to the GUI, and its Quit (Cmd+Q) would end // the agent's MCP server rather than anything the user knowingly started. On macOS @@ -402,15 +404,55 @@ export function runCli(command: CliCommand): void { }); if (command.kind === "mcp") { - await serveMcpOverStdio({ + // Only where the app updates itself: elsewhere a package manager already does, and + // opening the app would not change that. + const tellsAboutUpdates = app.isPackaged && ownsItsUpdates(getInstallChannel()); + let closing = false; + let stopWatchingUpdate: (() => void) | undefined; + const closeMcp = () => { + if (closing) return; + closing = true; + stopWatchingUpdate?.(); + // The Whisper helper must not outlive the server. + void shutdownStt().finally(() => app.exit(0)); + }; + const mcp = await serveMcpOverStdio({ ...agentServices, userDataDir, runners, openVideo: openVideoInHiddenWindow, - // The Whisper helper is a process of its own: it must not outlive this one. - onClose: () => void shutdownStt().finally(() => app.exit(0)), + updateNotice: tellsAboutUpdates + ? (updateNotice(app.getVersion(), readReleaseLookup(userDataDir)) ?? undefined) + : undefined, + onClose: closeMcp, }); + if (process.platform === "darwin" && tellsAboutUpdates) { + stopWatchingUpdate = watchForMacMcpShutdown( + { executablePath: process.execPath, version: app.getVersion() }, + () => { + // Keep recording, project-save and export runners alive until the client + // finishes them. Squirrel can wait; an unfinished take cannot be recovered. + if (mcp.isBusy()) return false; + safeWrite( + process.stderr, + "OpenScreen is updating. Restart the MCP server after the app returns.\n", + ); + closeMcp(); + return true; + }, + ); + } milestone("mcp server listening on stdio"); + if (tellsAboutUpdates) { + refreshReleaseLookup({ + userData: userDataDir, + currentVersion: app.getVersion(), + fetchLatest: (url, init) => net.fetch(url, init), + }).catch(() => { + // Offline or rate-limited: the next session looks again. Nothing to report, + // and stdout is the protocol. + }); + } // An install driven only by an agent is an active install too. Same ping, state // file and opt-outs as the GUI's, and it writes nothing: stdout is the protocol. // The other commands stay silent: they are scripts and CI more often than people. diff --git a/electron/cli/openGui.test.ts b/electron/cli/openGui.test.ts index e0fe782f1f..17ea636494 100644 --- a/electron/cli/openGui.test.ts +++ b/electron/cli/openGui.test.ts @@ -3,7 +3,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; const spawn = vi.hoisted(() => vi.fn()); vi.mock("node:child_process", () => ({ spawn })); -import { appBundleOf, openGuiFromCli } from "./openGui"; +import { appBundleOf, onCliActivate, openGuiFromCli } from "./openGui"; describe("appBundleOf", () => { it("climbs from Contents/MacOS to the .app bundle", () => { @@ -77,3 +77,26 @@ describe("openGuiFromCli", () => { expect(spawn).toHaveBeenCalledTimes(2); }); }); + +describe("onCliActivate", () => { + const exe = "/Applications/Openscreen.app/Contents/MacOS/Openscreen"; + + beforeEach(() => { + spawn.mockReset(); + spawn.mockImplementation(() => ({ on: vi.fn(), unref: vi.fn() })); + }); + + it("ignores a reopen sent to a hidden CLI process", () => { + onCliActivate({ isVisible: () => false, hide: vi.fn() }, exe, 4_000_000); + onCliActivate(undefined, exe, 4_100_000); + expect(spawn).not.toHaveBeenCalled(); + }); + + it("opens the GUI when the process is in the Dock, where macOS routes launches", () => { + onCliActivate({ isVisible: () => true, hide: vi.fn() }, exe, 5_000_000); + expect(spawn).toHaveBeenCalledWith("open", ["-n", "/Applications/Openscreen.app"], { + detached: true, + stdio: "ignore", + }); + }); +}); diff --git a/electron/cli/openGui.ts b/electron/cli/openGui.ts index 6fe114d1c8..d391989aaa 100644 --- a/electron/cli/openGui.ts +++ b/electron/cli/openGui.ts @@ -7,6 +7,12 @@ // CLI process as `activate`, so it starts the GUI itself, as a new instance. When a GUI is // already running, that instance loses the single-instance lock and brings it forward through // `second-instance`. +// +// LaunchServices only does that to a CLI process it counts as a regular app, one that shows in +// the Dock, as a run that has recorded does. A hidden one (`UIElement`) is left alone: Finder, +// the Dock and `open` start the GUI themselves. All it ever gets is a stray `reopen` Apple Event, +// and answering that put OpenScreen in front, out of nowhere, once per agent session still open +// (#1488, measured on macOS 27 with `tell application id … to reopen`). import { spawn } from "node:child_process"; import path from "node:path"; @@ -58,3 +64,13 @@ export function openGuiFromCli( if (afterLaunch) child.on("exit", afterLaunch); child.unref(); } + +/** `activate` on a CLI run: open the GUI only when it is a launch macOS handed to this process. */ +export function onCliActivate( + dock: { isVisible(): boolean; hide(): void } | undefined, + execPath: string = process.execPath, + now: number = Date.now(), +): void { + if (!dock?.isVisible()) return; + openGuiFromCli(execPath, now, () => dock.hide()); +} diff --git a/electron/diagnostics/main-log-buffer.test.ts b/electron/diagnostics/main-log-buffer.test.ts deleted file mode 100644 index 488d4cb32c..0000000000 --- a/electron/diagnostics/main-log-buffer.test.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { afterEach, describe, expect, it } from "vitest"; -import { MainLogBuffer } from "./main-log-buffer"; - -describe("MainLogBuffer", () => { - const buffers: MainLogBuffer[] = []; - - function make(capacity: number) { - const b = new MainLogBuffer(capacity); - buffers.push(b); - return b; - } - - afterEach(() => { - for (const b of buffers) b.uninstall(); - buffers.length = 0; - }); - - it("captures every console level and routes to original", () => { - const buf = make(10); - const captured: string[] = []; - const original = { - log: (...args: unknown[]) => captured.push(`log:${args.join(",")}`), - info: (...args: unknown[]) => captured.push(`info:${args.join(",")}`), - warn: (...args: unknown[]) => captured.push(`warn:${args.join(",")}`), - error: (...args: unknown[]) => captured.push(`error:${args.join(",")}`), - }; - Object.assign(console, original); - buf.install(); - console.log("hello"); - console.info("world"); - console.warn("watch"); - console.error("bad"); - const snap = buf.snapshot(); - expect(snap.map((e) => e.level)).toEqual(["log", "info", "warn", "error"]); - expect(snap.map((e) => e.text)).toEqual(["hello", "world", "watch", "bad"]); - expect(captured).toEqual(["log:hello", "info:world", "warn:watch", "error:bad"]); - }); - - it("stringifies non-string args", () => { - const buf = make(5); - buf.install(); - console.info({ a: 1 }); - const snap = buf.snapshot(); - expect(snap[0].text).toBe('{"a":1}'); - }); - - it("keeps an Error's message and stack rather than JSON's {}", () => { - const buf = make(5); - Object.assign(console, { - error: () => { - // The stack this logs would only clutter the test output. - }, - }); - buf.install(); - const error = new Error("helper exited (code=1)"); - console.error("Failed to stop native Linux recording:", error); - const [entry] = buf.snapshot(); - expect(entry.text).toBe(`Failed to stop native Linux recording: ${error.stack}`); - expect(entry.text).toContain("Error: helper exited (code=1)"); - }); - - it("drops oldest entries past capacity", () => { - const buf = make(3); - buf.install(); - for (let i = 0; i < 5; i += 1) console.info(`line ${i}`); - const snap = buf.snapshot(); - expect(snap.map((e) => e.text)).toEqual(["line 2", "line 3", "line 4"]); - }); - - it("uninstall restores originals", () => { - const buf = make(5); - buf.install(); - console.info("captured"); - buf.uninstall(); - console.info("after-uninstall"); - const snap = buf.snapshot(); - expect(snap.map((e) => e.text)).toEqual(["captured"]); - }); - - it("clear empties the buffer", () => { - const buf = make(5); - buf.install(); - console.info("one"); - console.info("two"); - buf.clear(); - expect(buf.snapshot()).toEqual([]); - }); - - it("install is idempotent", () => { - const buf = make(5); - buf.install(); - buf.install(); - console.info("once"); - const snap = buf.snapshot(); - expect(snap.map((e) => e.text)).toEqual(["once"]); - }); -}); diff --git a/electron/diagnostics/main-log-buffer.ts b/electron/diagnostics/main-log-buffer.ts deleted file mode 100644 index 8a9c8b5d64..0000000000 --- a/electron/diagnostics/main-log-buffer.ts +++ /dev/null @@ -1,107 +0,0 @@ -/** - * Ring buffer for main-process console output. - * - * Captures the last `capacity` lines written via console.info / console.warn / - * console.error / console.log into a single in-memory buffer, which "Save - * Diagnostics" writes as `mainProcessLogs`. main.ts installs it on every app - * launch: the users who save a bundle start the app from a desktop entry, the - * Dock or the Start menu, where no OPENSCREEN_DIAGNOSTIC can be set, and gating - * it on that flag left every one of their bundles with no log at all. - * - * Cost when installed: one array.push + occasional shift per console call, - * negligible against the rest of the app. Cost when not installed (the CLI): - * zero, the original console methods are kept untouched. - */ - -const DEFAULT_CAPACITY = 500; - -export interface MainLogEntry { - timestampMs: number; - level: "info" | "warn" | "error" | "log"; - text: string; -} - -export class MainLogBuffer { - private readonly capacity: number; - private readonly entries: MainLogEntry[] = []; - private installed = false; - private readonly originals: Partial< - Record<"log" | "info" | "warn" | "error", (...args: unknown[]) => void> - > = {}; - - constructor(capacity = DEFAULT_CAPACITY) { - this.capacity = Math.max(1, capacity); - } - - install(): void { - if (this.installed) return; - this.installed = true; - const console_ = console as unknown as Record< - "log" | "info" | "warn" | "error", - (...args: unknown[]) => void - >; - for (const level of ["log", "info", "warn", "error"] as const) { - this.originals[level] = console_[level].bind(console); - console_[level] = (...args: unknown[]) => { - this.push(level, args); - this.originals[level]?.(...args); - }; - } - } - - uninstall(): void { - if (!this.installed) return; - this.installed = false; - const console_ = console as unknown as Record< - "log" | "info" | "warn" | "error", - (...args: unknown[]) => void - >; - for (const level of ["log", "info", "warn", "error"] as const) { - if (this.originals[level]) { - console_[level] = this.originals[level] as (...args: unknown[]) => void; - } - } - this.originals.log = undefined; - this.originals.info = undefined; - this.originals.warn = undefined; - this.originals.error = undefined; - } - - snapshot(): MainLogEntry[] { - return this.entries.slice(); - } - - clear(): void { - this.entries.length = 0; - } - - private push(level: MainLogEntry["level"], args: unknown[]): void { - const text = args - .map((arg) => { - if (typeof arg === "string") return arg; - // An Error's message and stack are not enumerable, so JSON.stringify - // writes `{}`: every `console.error("…failed:", error)` lost its cause. - // The stack is what the terminal shows for it. - if (arg instanceof Error) return arg.stack ?? `${arg.name}: ${arg.message}`; - try { - return JSON.stringify(arg); - } catch { - return String(arg); - } - }) - .join(" "); - this.entries.push({ timestampMs: Date.now(), level, text }); - if (this.entries.length > this.capacity) { - this.entries.splice(0, this.entries.length - this.capacity); - } - } -} - -export const mainLogBuffer = new MainLogBuffer(); - -export function isDiagnosticModeEnabled(): boolean { - const raw = process.env.OPENSCREEN_DIAGNOSTIC; - if (!raw) return false; - const lowered = raw.trim().toLowerCase(); - return lowered === "1" || lowered === "true" || lowered === "yes"; -} diff --git a/electron/diagnostics/main-log-file.test.ts b/electron/diagnostics/main-log-file.test.ts new file mode 100644 index 0000000000..2cb1cebc19 --- /dev/null +++ b/electron/diagnostics/main-log-file.test.ts @@ -0,0 +1,367 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { MainLogFile } from "./main-log-file"; + +describe("MainLogFile", () => { + const logs: MainLogFile[] = []; + let dir: string; + + function make(maxFileBytes?: number) { + const log = new MainLogFile(maxFileBytes); + logs.push(log); + return log; + } + + function silenceConsole(captured: string[] = []) { + Object.assign(console, { + log: (...args: unknown[]) => captured.push(`log:${args.join(",")}`), + info: (...args: unknown[]) => captured.push(`info:${args.join(",")}`), + warn: (...args: unknown[]) => captured.push(`warn:${args.join(",")}`), + error: (...args: unknown[]) => captured.push(`error:${args.join(",")}`), + }); + return captured; + } + + function lines(file = "main.log") { + return fs.readFileSync(path.join(dir, file), "utf-8").trimEnd().split("\n"); + } + + beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), "openscreen-main-log-")); + }); + + afterEach(() => { + for (const log of logs) log.uninstall(); + logs.length = 0; + vi.restoreAllMocks(); + fs.rmSync(dir, { recursive: true, force: true }); + }); + + it("writes a launch header, then every console level, and still calls the console", () => { + const captured = silenceConsole(); + make().install(dir, "launch 1.2.3 pid 42"); + console.log("hello"); + console.info("world"); + console.warn("watch"); + console.error("bad"); + + const written = lines(); + expect(written[0]).toMatch(/^\S+ --- launch 1\.2\.3 pid 42 ---$/); + expect(written.slice(1).map((line) => line.split(" ").slice(1).join(" "))).toEqual([ + "log hello", + "info world", + "warn watch", + "error bad", + ]); + expect(captured).toEqual(["log:hello", "info:world", "warn:watch", "error:bad"]); + }); + + it("appends to the previous launch's file", () => { + silenceConsole(); + const first = make(); + first.install(dir, "launch one"); + console.info("from one"); + first.uninstall(); + make().install(dir, "launch two"); + + expect(lines().map((line) => line.split(" ").slice(1).join(" "))).toEqual([ + "--- launch one ---", + "info from one", + "--- launch two ---", + ]); + }); + + it("writes an Error as its stack and an object as JSON", () => { + silenceConsole(); + make().install(dir, "launch"); + const error = new Error("helper exited (code=1)"); + console.error("Failed to stop:", error); + console.info({ a: 1 }); + + const text = fs.readFileSync(path.join(dir, "main.log"), "utf-8"); + expect(text).toContain(`error Failed to stop: ${error.stack}`); + expect(text).toContain('info {"a":1}'); + }); + + it("writes values with no JSON form as text", () => { + silenceConsole(); + make().install(dir, "launch"); + console.log("value:", undefined); + + const text = fs.readFileSync(path.join(dir, "main.log"), "utf-8"); + expect(text).toContain("log value: undefined"); + }); + + it("moves a full file to main.old.log and never keeps more than two files", () => { + silenceConsole(); + make(200).install(dir, "launch"); + for (let i = 0; i < 30; i += 1) console.info(`line ${String(i).padStart(2, "0")}`); + + expect(fs.readdirSync(dir).sort()).toEqual(["main.log", "main.old.log"]); + expect(fs.statSync(path.join(dir, "main.log")).size).toBeLessThanOrEqual(200); + expect(fs.statSync(path.join(dir, "main.old.log")).size).toBeLessThanOrEqual(200); + expect(lines().at(-1)).toMatch(/info line 29$/); + }); + + it("rotates a file already over the cap when it is installed", () => { + silenceConsole(); + fs.writeFileSync(path.join(dir, "main.log"), "x".repeat(300)); + make(200).install(dir, "launch"); + + expect(fs.readFileSync(path.join(dir, "main.old.log"), "utf-8")).toBe("x".repeat(300)); + expect(lines()).toHaveLength(1); + }); + + it("reads the tail across main.old.log and main.log, from a line start", () => { + fs.writeFileSync(path.join(dir, "main.old.log"), "old 1\nold 2\nold 3\n"); + fs.writeFileSync(path.join(dir, "main.log"), "new 1\nnew 2\n"); + silenceConsole(); + const log = make(); + log.install(dir, "launch"); + const header = lines().at(-1) as string; + const currentBytes = Buffer.byteLength(`new 1\nnew 2\n${header}\n`); + + expect(log.readTail(10_000)).toBe(`old 1\nold 2\nold 3\nnew 1\nnew 2\n${header}\n`); + // Eight bytes into the old file land mid-line: that partial line is dropped. + expect(log.readTail(currentBytes + 8)).toBe(`old 3\nnew 1\nnew 2\n${header}\n`); + expect(log.readTail(Buffer.byteLength(`${header}\n`) + 3)).toBe(`${header}\n`); + }); + + it("keeps the console working and reports why when the folder cannot be created", () => { + const captured = silenceConsole(); + const blocker = path.join(dir, "not-a-folder"); + fs.writeFileSync(blocker, ""); + const log = make(); + log.install(path.join(blocker, "logs"), "launch"); + console.info("still here"); + + expect(captured).toEqual(["info:still here"]); + expect(log.lastError).toMatch(/^(EEXIST|ENOTDIR)/); + expect(log.readTail(1000)).toBe(""); + }); + + it("stops writing after a failed write and keeps the error", () => { + const captured = silenceConsole(); + const log = make(); + log.install(dir, "launch"); + vi.spyOn(fs, "writeSync").mockImplementationOnce(() => { + throw Object.assign(new Error("ENOSPC: no space left on device, write"), { + code: "ENOSPC", + }); + }); + console.info("lost"); + console.info("also lost"); + + expect(captured).toEqual(["info:lost", "info:also lost"]); + expect(log.lastError).toMatch(/^ENOSPC/); + expect(lines()).toHaveLength(1); + }); + + it("readFiles returns both files whole, oldest first", () => { + silenceConsole(); + const log = make(); + fs.writeFileSync(path.join(dir, "main.old.log"), "old\n"); + log.install(dir, "launch"); + console.info("new"); + + const files = log.readFiles(); + expect(files.map((f) => f.name)).toEqual(["main.old.log", "main.log"]); + expect(files[0].data.toString()).toBe("old\n"); + expect(files[1].data.toString()).toMatch(/--- launch ---\n.* info new\n$/); + expect(log.lastError).toBeNull(); + }); + + it("readFiles leaves out a missing file without an error", () => { + silenceConsole(); + const log = make(); + log.install(dir, "launch"); + + expect(log.readFiles().map((f) => f.name)).toEqual(["main.log"]); + expect(log.lastError).toBeNull(); + }); + + it("reports a failed read and keeps writing", () => { + silenceConsole(); + const log = make(); + log.install(dir, "launch"); + vi.spyOn(fs, "readSync").mockImplementationOnce(() => { + throw Object.assign(new Error("EIO: i/o error, read"), { code: "EIO" }); + }); + + expect(log.readTail(1000)).toBe(""); + expect(log.lastError).toMatch(/^EIO/); + console.info("after"); + expect(lines()).toHaveLength(2); + }); + + // Windows reads a directory as an empty file, so there is nothing to fail there. + it.skipIf(process.platform === "win32")( + "keeps the current log when main.old.log cannot be read", + () => { + silenceConsole(); + // A directory opens but does not read, so only the older half of the tail fails. + fs.mkdirSync(path.join(dir, "main.old.log")); + const log = make(); + log.install(dir, "launch"); + console.info("current"); + + expect(log.readTail(1000)).toContain("info current"); + expect(log.lastError).toBeTruthy(); + }, + ); + + it("still calls the console when an argument cannot be turned into text", () => { + silenceConsole(); + const original = vi.fn(); + console.info = original; + make().install(dir, "launch"); + // No prototype, so no toString, and a toJSON that throws. + const unprintable = Object.assign(Object.create(null), { + toJSON: () => { + throw new Error("no"); + }, + }); + + expect(() => console.info(unprintable)).not.toThrow(); + expect(original).toHaveBeenCalledWith(unprintable); + }); + + it("uninstall restores the console methods", () => { + const captured = silenceConsole(); + const original = console.info; + const log = make(); + log.install(dir, "launch"); + log.uninstall(); + console.info("after"); + + expect(console.info).toBe(original); + expect(captured).toEqual(["info:after"]); + expect(lines()).toHaveLength(1); + }); + + describe("flood guard", () => { + function texts() { + return lines().map((line) => line.split(" ").slice(1).join(" ")); + } + + beforeEach(() => { + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(new Date("2026-10-18T12:00:00Z")); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it("counts an identical line in a row instead of writing it", () => { + const captured = silenceConsole(); + make().install(dir, "launch"); + for (let i = 0; i < 1000; i += 1) console.warn("[live] readback_direct: lost"); + console.info("next"); + + expect(texts()).toEqual([ + "--- launch ---", + "warn [live] readback_direct: lost", + "warn last message repeated 999 times", + "info next", + ]); + expect(captured).toHaveLength(1001); + }); + + it("does not merge the same text at different levels", () => { + silenceConsole(); + make().install(dir, "launch"); + console.info("same"); + console.warn("same"); + + expect(texts()).toEqual(["--- launch ---", "info same", "warn same"]); + }); + + it("writes the repeat count while a flood lasts", () => { + silenceConsole(); + make().install(dir, "launch"); + console.warn("flood"); + console.warn("flood"); + vi.advanceTimersByTime(30_000); + console.warn("flood"); + for (let i = 0; i < 10_000; i += 1) console.warn("flood"); + + expect(texts()).toEqual([ + "--- launch ---", + "warn flood", + "warn last message repeated 2 times", + "warn last message repeated 10000 times", + ]); + }); + + it("rate-limits lines that differ and says how many were dropped", () => { + const captured = silenceConsole(); + make().install(dir, "launch"); + for (let i = 0; i < 300; i += 1) console.warn(i % 2 ? "[annotation] a" : "[annotation] b"); + vi.advanceTimersByTime(1000); + console.info("after"); + + const written = texts(); + expect(written).toHaveLength(1 + 200 + 2); + expect(written.at(-2)).toBe("warn 100 lines suppressed (more than 10 lines/s)"); + expect(written.at(-1)).toBe("info after"); + expect(captured).toHaveLength(301); + }); + + it("writes the suppressed count once lines are let through again", () => { + silenceConsole(); + make().install(dir, "launch"); + for (let i = 0; i < 250; i += 1) console.warn(`frame ${i}`); + vi.advanceTimersByTime(30_000); + // 300 tokens came back over 30 s, capped at 200: the first line of this batch + // is written along with the summary. + console.warn("frame 250"); + + expect(texts().slice(-2)).toEqual([ + "warn 50 lines suppressed (more than 10 lines/s)", + "warn frame 250", + ]); + }); + + it("a clock stepped back does not drain the bucket", () => { + silenceConsole(); + make().install(dir, "launch"); + vi.setSystemTime(new Date("2026-10-18T11:59:00Z")); + for (let i = 0; i < 10; i += 1) console.warn(`line ${i}`); + + expect(texts()).toHaveLength(1 + 10); + }); + + it("writes pending counts before the tail is read", () => { + silenceConsole(); + const log = make(); + log.install(dir, "launch"); + console.warn("flood"); + console.warn("flood"); + + expect(log.readTail(10_000)).toMatch(/warn last message repeated 1 time\n$/); + }); + + it("caps a single line", () => { + silenceConsole(); + make().install(dir, "launch"); + console.info("x".repeat(70 * 1024)); + + const written = lines()[1]; + expect(written).toMatch(/…\[truncated 6144 chars\]$/); + expect(written.length).toBeLessThan(65 * 1024); + }); + }); + + it("install is idempotent", () => { + silenceConsole(); + const log = make(); + log.install(dir, "launch"); + log.install(dir, "launch"); + console.info("once"); + + expect(lines()).toHaveLength(2); + }); +}); diff --git a/electron/diagnostics/main-log-file.ts b/electron/diagnostics/main-log-file.ts new file mode 100644 index 0000000000..5b0b7c557e --- /dev/null +++ b/electron/diagnostics/main-log-file.ts @@ -0,0 +1,324 @@ +import fs from "node:fs"; +import path from "node:path"; + +/** + * Main-process console output, appended to `main.log` in the OS log folder + * (`app.getPath("logs")`), which "Save Diagnostics" reads back as `mainProcessLogs`. + * On disk rather than in memory so a bundle saved after a crash still holds the + * run that crashed. The folder stays small by itself: past `maxFileBytes`, + * `main.log` becomes `main.old.log` (replacing the previous one) and starts over, + * so there are never more than two files. + * + * Writes are synchronous so the last lines before a native crash reach the disk. + * Logging never breaks the app: a failed open or write turns the file off, keeps + * the error as `lastError`, and the console carries on. + * + * A flood cannot push the useful lines out of the folder: an identical line in a row + * is counted instead of written, any line is capped at `MAX_LINE_CHARS`, and the + * rest is rate-limited (`BURST_LINES`, then `LINES_PER_SECOND`). Both counts are + * written as one summary line, at least every `SUMMARY_INTERVAL_MS` while a flood + * lasts. Only the file is affected: the console still gets every line. + */ + +const LOG_FILE = "main.log"; +const OLD_LOG_FILE = "main.old.log"; +const DEFAULT_MAX_FILE_BYTES = 5 * 1024 * 1024; +const MAX_LINE_CHARS = 64 * 1024; +const BURST_LINES = 200; +const LINES_PER_SECOND = 10; +const SUMMARY_INTERVAL_MS = 30_000; +const MAX_REPEATS_PER_SUMMARY = 10_000; + +type Level = "log" | "info" | "warn" | "error"; +const LEVELS = ["log", "info", "warn", "error"] as const; + +export class MainLogFile { + private dir: string | null = null; + private fd: number | null = null; + private bytes = 0; + private error: string | null = null; + private readonly originals: Partial void>> = {}; + + private lastKey: string | null = null; + private lastLevel: Level = "log"; + private repeats = 0; + private repeatsSince = 0; + private tokens = BURST_LINES; + private tokensAt = 0; + private suppressed = 0; + private suppressedSince = 0; + + constructor(private readonly maxFileBytes = DEFAULT_MAX_FILE_BYTES) {} + + get filePath(): string | null { + return this.dir ? path.join(this.dir, LOG_FILE) : null; + } + + get lastError(): string | null { + return this.error; + } + + install(dir: string, header: string): void { + if (this.dir) return; + this.dir = dir; + try { + fs.mkdirSync(dir, { recursive: true }); + this.open(); + } catch (error) { + this.fail(error); + } + this.tokensAt = Date.now(); + this.write(`${new Date().toISOString()} --- ${header} ---\n`); + + const console_ = console as unknown as Record void>; + for (const level of LEVELS) { + const original = console_[level]; + this.originals[level] = original; + console_[level] = (...args: unknown[]) => { + // Logging to the file must never break the console call it rides on. + try { + this.record(level, formatArgs(args)); + } catch { + // An argument that cannot even be turned into text: the console still gets it. + } + original.apply(console, args); + }; + } + } + + /** Writes the pending repeat and suppression counts. */ + flush(): void { + const now = Date.now(); + this.flushRepeats(now); + this.flushSuppressed(now); + } + + uninstall(): void { + if (!this.dir) return; + this.flush(); + const console_ = console as unknown as Record void>; + for (const level of LEVELS) { + const original = this.originals[level]; + if (original) console_[level] = original; + this.originals[level] = undefined; + } + this.close(); + this.dir = null; + this.error = null; + this.lastKey = null; + this.repeats = 0; + this.suppressed = 0; + this.tokens = BURST_LINES; + } + + /** The last `maxBytes` of `main.old.log` + `main.log`, starting on a whole line. */ + readTail(maxBytes: number): string { + if (!this.dir) return ""; + this.flush(); + let data: Buffer; + let whole: boolean; + try { + const current = readLastBytes(path.join(this.dir, LOG_FILE), maxBytes); + data = current.data; + whole = current.whole; + if (current.whole && data.length < maxBytes) { + try { + const old = readLastBytes(path.join(this.dir, OLD_LOG_FILE), maxBytes - data.length); + data = Buffer.concat([old.data, data]); + whole = old.whole; + } catch (error) { + // The current log was read: keep it, and report the older one. + this.error = describeError(error); + } + } + } catch (error) { + // Reported in the bundle as `logFileError`; writing carries on. + this.error = describeError(error); + return ""; + } + const text = data.toString("utf-8"); + return whole ? text : text.slice(text.indexOf("\n") + 1); + } + + /** `main.old.log` and `main.log` whole, for the zipped bundle; a missing one is left out. */ + readFiles(): { name: string; data: Buffer }[] { + if (!this.dir) return []; + this.flush(); + const files: { name: string; data: Buffer }[] = []; + for (const name of [OLD_LOG_FILE, LOG_FILE]) { + try { + files.push({ name, data: fs.readFileSync(path.join(this.dir, name)) }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + this.error = describeError(error); + } + } + } + return files; + } + + private record(level: Level, rawText: string): void { + if (this.fd === null) return; + const now = Date.now(); + const text = + rawText.length > MAX_LINE_CHARS + ? `${rawText.slice(0, MAX_LINE_CHARS)}…[truncated ${rawText.length - MAX_LINE_CHARS} chars]` + : rawText; + const key = `${level} ${text}`; + + if (key === this.lastKey) { + if (this.repeats === 0) this.repeatsSince = now; + this.repeats += 1; + if ( + this.repeats >= MAX_REPEATS_PER_SUMMARY || + now - this.repeatsSince >= SUMMARY_INTERVAL_MS + ) { + this.flushRepeats(now); + } + return; + } + this.flushRepeats(now); + this.lastKey = key; + this.lastLevel = level; + + // A clock stepped back must neither drain nor refill the bucket. + const elapsedMs = now - this.tokensAt; + if (elapsedMs > 0) { + this.tokens = Math.min(BURST_LINES, this.tokens + (elapsedMs / 1000) * LINES_PER_SECOND); + this.tokensAt = now; + } + if (this.tokens < 1) { + // A line that was not written cannot be "repeated": its copies are suppressed too. + this.lastKey = null; + if (this.suppressed === 0) this.suppressedSince = now; + this.suppressed += 1; + if (now - this.suppressedSince >= SUMMARY_INTERVAL_MS) this.flushSuppressed(now); + return; + } + this.tokens -= 1; + this.flushSuppressed(now); + this.write(`${new Date(now).toISOString()} ${level} ${text}\n`); + } + + private flushRepeats(now: number): void { + if (this.repeats === 0) return; + const count = this.repeats; + this.repeats = 0; + this.repeatsSince = now; + this.write( + `${new Date(now).toISOString()} ${this.lastLevel} last message repeated ${count} time${count === 1 ? "" : "s"}\n`, + ); + } + + private flushSuppressed(now: number): void { + if (this.suppressed === 0) return; + const count = this.suppressed; + this.suppressed = 0; + this.suppressedSince = now; + this.write( + `${new Date(now).toISOString()} warn ${count} lines suppressed (more than ${LINES_PER_SECOND} lines/s)\n`, + ); + } + + private open(): void { + if (!this.dir) return; + const file = path.join(this.dir, LOG_FILE); + if (fs.existsSync(file) && fs.statSync(file).size >= this.maxFileBytes) { + fs.renameSync(file, path.join(this.dir, OLD_LOG_FILE)); + } + this.fd = fs.openSync(file, "a"); + this.bytes = fs.fstatSync(this.fd).size; + } + + private rotate(): void { + if (!this.dir) return; + this.close(); + fs.renameSync(path.join(this.dir, LOG_FILE), path.join(this.dir, OLD_LOG_FILE)); + this.open(); + } + + private write(line: string): void { + if (this.fd === null) return; + try { + const data = Buffer.from(line, "utf-8"); + if (this.bytes > 0 && this.bytes + data.length > this.maxFileBytes) this.rotate(); + if (this.fd === null) return; + // writeSync may write less than asked (a full disk, a signal): finish the line. + let written = 0; + while (written < data.length) { + const n = fs.writeSync(this.fd, data, written, data.length - written); + if (n <= 0) throw new Error("main.log: write made no progress"); + written += n; + } + this.bytes += data.length; + } catch (error) { + this.fail(error); + } + } + + private fail(error: unknown): void { + this.close(); + this.error = describeError(error); + } + + private close(): void { + if (this.fd === null) return; + try { + fs.closeSync(this.fd); + } catch { + // Already gone; nothing left to flush. + } + this.fd = null; + } +} + +function describeError(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function formatArgs(args: unknown[]): string { + return args + .map((arg) => { + if (typeof arg === "string") return arg; + // An Error's message and stack are not enumerable, so JSON.stringify writes `{}`. + if (arg instanceof Error) return arg.stack ?? `${arg.name}: ${arg.message}`; + try { + // `undefined`, functions and symbols have no JSON form: write them as the console does. + return JSON.stringify(arg) ?? String(arg); + } catch { + return String(arg); + } + }) + .join(" "); +} + +/** `whole` is true when the read reached the start of the file (or it does not exist). */ +function readLastBytes(file: string, maxBytes: number): { data: Buffer; whole: boolean } { + let fd: number; + try { + fd = fs.openSync(file, "r"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return { data: Buffer.alloc(0), whole: true }; + } + throw error; + } + try { + const size = fs.fstatSync(fd).size; + const length = Math.min(size, maxBytes); + const data = Buffer.alloc(length); + fs.readSync(fd, data, 0, length, size - length); + return { data, whole: length === size }; + } finally { + fs.closeSync(fd); + } +} + +export const mainLogFile = new MainLogFile(); + +export function isDiagnosticModeEnabled(): boolean { + const raw = process.env.OPENSCREEN_DIAGNOSTIC; + if (!raw) return false; + const lowered = raw.trim().toLowerCase(); + return lowered === "1" || lowered === "true" || lowered === "yes"; +} diff --git a/electron/diagnostics/zip.test.ts b/electron/diagnostics/zip.test.ts new file mode 100644 index 0000000000..65ca6324e9 --- /dev/null +++ b/electron/diagnostics/zip.test.ts @@ -0,0 +1,72 @@ +import { execFileSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import zlib from "node:zlib"; +import { describe, expect, it } from "vitest"; +import { createZip } from "./zip"; + +/** Reads the archive back through its central directory, the way an unzip tool does. */ +function readZip(zip: Buffer) { + const end = zip.length - 22; + expect(zip.readUInt32LE(end)).toBe(0x06054b50); + const count = zip.readUInt16LE(end + 10); + let at = zip.readUInt32LE(end + 16); + const files: Record = {}; + for (let i = 0; i < count; i++) { + expect(zip.readUInt32LE(at)).toBe(0x02014b50); + const crc = zip.readUInt32LE(at + 16); + const size = zip.readUInt32LE(at + 20); + const nameLength = zip.readUInt16LE(at + 28); + const local = zip.readUInt32LE(at + 42); + const name = zip.toString("utf-8", at + 46, at + 46 + nameLength); + expect(zip.readUInt32LE(local)).toBe(0x04034b50); + const start = local + 30 + zip.readUInt16LE(local + 26); + const data = zlib.inflateRawSync(zip.subarray(start, start + size)); + expect(zlib.crc32(data)).toBe(crc); + files[name] = data.toString("utf-8"); + at += 46 + nameLength; + } + return files; +} + +describe("createZip", () => { + it("round-trips every entry", () => { + const big = "line\n".repeat(100_000); + const zip = createZip([ + { name: "diagnostic.json", data: Buffer.from('{"a":1}') }, + { name: "main.log", data: Buffer.from(big) }, + { name: "empty.log", data: Buffer.alloc(0) }, + { name: "é.txt", data: Buffer.from("ü") }, + ]); + + expect(readZip(zip)).toEqual({ + "diagnostic.json": '{"a":1}', + "main.log": big, + "empty.log": "", + "é.txt": "ü", + }); + expect(zip.length).toBeLessThan(big.length / 10); + }); + + it("opens with the system unzip, where there is one", () => { + let unzip = true; + try { + execFileSync("unzip", ["-v"], { stdio: "ignore" }); + } catch { + unzip = false; + } + if (!unzip) return; + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "openscreen-zip-")); + try { + const file = path.join(dir, "bundle.zip"); + fs.writeFileSync(file, createZip([{ name: "main.log", data: Buffer.from("hello\n") }])); + execFileSync("unzip", ["-tq", file], { stdio: "ignore" }); + expect(execFileSync("unzip", ["-p", file, "main.log"], { encoding: "utf-8" })).toBe( + "hello\n", + ); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/electron/diagnostics/zip.ts b/electron/diagnostics/zip.ts new file mode 100644 index 0000000000..b1ab4ac07c --- /dev/null +++ b/electron/diagnostics/zip.ts @@ -0,0 +1,63 @@ +import zlib from "node:zlib"; + +export interface ZipEntry { + name: string; + data: Buffer; +} + +/** + * A deflated `.zip` of `entries`, for the diagnostic bundle. Only what that needs: no + * folders, no ZIP64 (every entry and the whole archive stay under 4 GiB), UTF-8 names. + */ +export function createZip(entries: ZipEntry[], date = new Date()): Buffer { + const time = (date.getHours() << 11) | (date.getMinutes() << 5) | (date.getSeconds() >> 1); + const day = ((date.getFullYear() - 1980) << 9) | ((date.getMonth() + 1) << 5) | date.getDate(); + const locals: Buffer[] = []; + const centrals: Buffer[] = []; + let offset = 0; + + for (const entry of entries) { + const name = Buffer.from(entry.name, "utf-8"); + const compressed = zlib.deflateRawSync(entry.data); + const crc = zlib.crc32(entry.data); + + const local = Buffer.alloc(30); + local.writeUInt32LE(0x04034b50, 0); + local.writeUInt16LE(20, 4); // version needed: deflate + local.writeUInt16LE(0x0800, 6); // UTF-8 name + local.writeUInt16LE(8, 8); // deflate + local.writeUInt16LE(time, 10); + local.writeUInt16LE(day, 12); + local.writeUInt32LE(crc, 14); + local.writeUInt32LE(compressed.length, 18); + local.writeUInt32LE(entry.data.length, 22); + local.writeUInt16LE(name.length, 26); + locals.push(local, name, compressed); + + const central = Buffer.alloc(46); + central.writeUInt32LE(0x02014b50, 0); + central.writeUInt16LE(20, 4); // version made by + central.writeUInt16LE(20, 6); + central.writeUInt16LE(0x0800, 8); + central.writeUInt16LE(8, 10); + central.writeUInt16LE(time, 12); + central.writeUInt16LE(day, 14); + central.writeUInt32LE(crc, 16); + central.writeUInt32LE(compressed.length, 20); + central.writeUInt32LE(entry.data.length, 24); + central.writeUInt16LE(name.length, 28); + central.writeUInt32LE(offset, 42); + centrals.push(central, name); + + offset += local.length + name.length + compressed.length; + } + + const directory = Buffer.concat(centrals); + const end = Buffer.alloc(22); + end.writeUInt32LE(0x06054b50, 0); + end.writeUInt16LE(entries.length, 8); + end.writeUInt16LE(entries.length, 10); + end.writeUInt32LE(directory.length, 12); + end.writeUInt32LE(offset, 16); + return Buffer.concat([...locals, directory, end]); +} diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index fc6b5f9118..f0c93ff955 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -46,6 +46,10 @@ interface Window { meta: import("../src/native/contracts").CompositorSharedFrameMeta, ) => void, ) => () => void; + reportRecordingOutcome?: ( + outcome: import("../src/lib/recordingSession").RecordingOutcome, + ) => Promise<{ success: boolean }>; + getSources: (opts: Electron.SourcesOptions) => Promise; switchToEditor: () => Promise; switchToHud: () => Promise; @@ -58,6 +62,15 @@ interface Window { usesSystemSourcePicker?: () => Promise; /** Name of the last source picked in Apple's picker, for display only: it is not selected. */ getLastPickedSource?: () => Promise; + prepareCameraPreviewCapture?: () => Promise; + restoreCameraPreviewCapture?: (token: number) => Promise; + onCameraPreviewAvailability?: (callback: (available: boolean) => void) => () => void; + configureCameraPreview?: (enabled: boolean, visible: boolean) => Promise; + sendCameraPreviewFrame?: (jpeg: string, aspectRatio: number) => Promise; + resizeCameraPreview?: (width: number) => Promise; + hideCameraPreview?: () => Promise; + onCameraPreviewFrame?: (callback: (jpeg: string) => void) => () => void; + onCameraPreviewHidden?: (callback: () => void) => () => void; openNotes: () => Promise<{ opened: boolean; reason?: string; @@ -149,7 +162,8 @@ interface Window { recording: boolean, recordingId?: number, cursorCaptureMode?: import("../src/lib/recordingSession").CursorCaptureMode, - ) => Promise; + reserveOnly?: boolean, + ) => Promise<{ success: boolean; error?: string }>; isNativeWindowsCaptureAvailable: () => Promise<{ success: boolean; available: boolean; @@ -191,6 +205,11 @@ interface Window { startNativeMacRecording: ( request: import("../src/lib/nativeMacRecording").NativeMacRecordingRequest, ) => Promise; + onNativeMacPresenterOverlayChanged: ( + callback: ( + event: import("../src/lib/nativeMacRecording").NativeMacPresenterOverlayEvent, + ) => void, + ) => () => void; onNativeMacSystemAudioUnavailable: (callback: () => void) => () => void; pauseNativeMacRecording: () => Promise<{ success: boolean; @@ -229,6 +248,7 @@ interface Window { isNativeLinuxCaptureAvailable: () => Promise<{ success: boolean; available: boolean; + backend?: "x11" | "pipewire"; helperPath?: string; reason?: "unsupported-platform" | "missing-helper" | string; error?: string; @@ -468,7 +488,13 @@ interface Window { getAppInfo: () => Promise<{ version: string; canCheckForUpdates: boolean }>; checkForUpdates: () => Promise; showAbout: () => Promise; + saveDiagnostics: () => Promise; canCheckForUpdatesNow: () => Promise; + getUpdateBadge: () => Promise; + onUpdateBadgeChanged: ( + callback: (badge: import("./background-update").UpdateBadge | null) => void, + ) => () => void; + applyUpdate: () => Promise; revealInFolder: ( filePath: string, ) => Promise<{ success: boolean; error?: string; message?: string }>; @@ -491,6 +517,8 @@ interface Window { shift?: boolean; alt?: boolean; }) => Promise<{ success: boolean }>; + /** False where the session grants no global shortcut (GNOME Wayland). */ + isGlobalShortcutAvailable: () => Promise; hudOverlayHide: () => void; hudOverlayClose: () => void; setHudOverlayIgnoreMouseEvents: (ignore: boolean) => void; diff --git a/electron/globalShortcut.test.ts b/electron/globalShortcut.test.ts new file mode 100644 index 0000000000..cd40d15105 --- /dev/null +++ b/electron/globalShortcut.test.ts @@ -0,0 +1,73 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const registered = new Set(); +const register = vi.fn((accelerator: string) => { + if (!grants(accelerator)) return false; + registered.add(accelerator); + return true; +}); +let grants: (accelerator: string) => boolean = () => true; + +vi.mock("electron", () => ({ + globalShortcut: { + register: (accelerator: string) => register(accelerator), + unregister: (accelerator: string) => registered.delete(accelerator), + unregisterAll: () => registered.clear(), + }, +})); +vi.mock("./ipc/handlers", () => ({ SHORTCUTS_FILE: "/nonexistent/shortcuts.json" })); + +const OPEN_APP = { key: "o", ctrl: true, shift: true }; + +async function freshModule() { + vi.resetModules(); + return import("./globalShortcut"); +} + +beforeEach(() => { + registered.clear(); + register.mockClear(); + grants = () => true; + vi.spyOn(console, "log").mockImplementation(() => undefined); + vi.spyOn(console, "warn").mockImplementation(() => undefined); +}); + +describe("registerOpenAppShortcut", () => { + // The rc.1 pass on GNOME Wayland: the session grants no global shortcut, so Save in + // the shortcuts dialog failed on Open App every time and no binding could change. + it("does not fail where the session grants no global shortcut, and says Open App is unavailable", async () => { + grants = () => false; + const shortcuts = await freshModule(); + + expect(shortcuts.registerOpenAppShortcut(OPEN_APP, () => undefined)).toBe(true); + expect(shortcuts.isGlobalShortcutAvailable()).toBe(false); + // A later Save asks nothing more of a session that already said no. + register.mockClear(); + expect(shortcuts.registerOpenAppShortcut({ ...OPEN_APP, key: "p" }, () => undefined)).toBe( + true, + ); + expect(register).not.toHaveBeenCalled(); + }); + + it("still fails a key another app holds, where the session grants others", async () => { + grants = (accelerator) => accelerator !== "CommandOrControl+Shift+O"; + const shortcuts = await freshModule(); + + expect(shortcuts.registerOpenAppShortcut(OPEN_APP, () => undefined)).toBe(false); + expect(shortcuts.isGlobalShortcutAvailable()).toBe(true); + // The probe is released, not kept. + expect([...registered]).toEqual([]); + }); + + it("keeps the old binding when the new key is taken", async () => { + const shortcuts = await freshModule(); + expect(shortcuts.registerOpenAppShortcut(OPEN_APP, () => undefined)).toBe(true); + + grants = (accelerator) => accelerator !== "CommandOrControl+Shift+P"; + expect(shortcuts.registerOpenAppShortcut({ ...OPEN_APP, key: "p" }, () => undefined)).toBe( + false, + ); + expect([...registered]).toEqual(["CommandOrControl+Shift+O"]); + expect(shortcuts.isGlobalShortcutAvailable()).toBe(true); + }); +}); diff --git a/electron/globalShortcut.ts b/electron/globalShortcut.ts index 2765bad220..808b7f4996 100644 --- a/electron/globalShortcut.ts +++ b/electron/globalShortcut.ts @@ -37,11 +37,39 @@ function bindingToAccelerator(binding: ShortcutBinding): string { } let currentAccelerator: string | null = null; +/** + * False once a failed registration showed that this session grants no global shortcut + * at all, as on GNOME Wayland: there `Ctrl+Shift+O` failed at every launch and every + * Save of the shortcuts dialog with it, so no binding could ever be changed. + */ +let globalShortcutsGranted = true; +/** + * Whether the session grants global shortcuts at all, asked only once a registration + * failed: a key nothing uses, so a refusal of it means "none", not "this one is taken". + */ +function sessionGrantsGlobalShortcuts(): boolean { + const probe = "CommandOrControl+Alt+Shift+F24"; + if (!globalShortcut.register(probe, () => undefined)) { + return false; + } + globalShortcut.unregister(probe); + return true; +} + +/** Whether Open App can be a global shortcut in this session. */ +export function isGlobalShortcutAvailable(): boolean { + return globalShortcutsGranted; +} + +/** + * Registers Open App. False means the key is taken and the user should pick another; + * a session that grants no global shortcut returns true, since no key would do. + */ export function registerOpenAppShortcut(binding: ShortcutBinding, onTrigger: () => void): boolean { const accelerator = bindingToAccelerator(binding); - if (accelerator === currentAccelerator) { + if (!globalShortcutsGranted || accelerator === currentAccelerator) { return true; } @@ -54,11 +82,17 @@ export function registerOpenAppShortcut(binding: ShortcutBinding, onTrigger: () } currentAccelerator = accelerator; console.log(`Global shortcut registered: ${accelerator}`); - } else { - console.warn(`Failed to register global shortcut: ${accelerator}`); + return true; } - - return success; + if (!currentAccelerator && !sessionGrantsGlobalShortcuts()) { + globalShortcutsGranted = false; + console.warn( + `This session grants no global shortcut; Open App (${accelerator}) is unavailable.`, + ); + return true; + } + console.warn(`Failed to register global shortcut: ${accelerator}`); + return false; } export async function loadAndRegisterGlobalShortcut(onTrigger: () => void): Promise { diff --git a/electron/hudWindowBounds.test.ts b/electron/hudWindowBounds.test.ts index c5c7193941..91b706c274 100644 --- a/electron/hudWindowBounds.test.ts +++ b/electron/hudWindowBounds.test.ts @@ -1,11 +1,16 @@ import { describe, expect, it } from "vitest"; +import { HUD_TOOLTIP_STACK_HEIGHT } from "../src/components/launch/hudGeometry"; import { clampHudBoundsToWorkArea, clampHudContentToWorkArea, hudBarAnchor, hudDragDestination, hudResizeBounds, + NOTES_WINDOW_FLOOR_HEIGHT, + NOTES_WINDOW_HEIGHT, + notesWindowHeight, parseHudContentRect, + WAYLAND_HUD_BAR_TOP, } from "./hudWindowBounds"; // 1920x1080 with a 48px taskbar at the bottom: the work area stops above it. @@ -302,3 +307,29 @@ describe("hud content rect parsing", () => { expect(parseHudContentRect({ x: 0, y: 0, width: 560 })).toBeNull(); }); }); + +describe("Notes opening height (GNOME Wayland)", () => { + it("keeps its height where the app places its own windows", () => { + expect(notesWindowHeight(false)).toBe(NOTES_WINDOW_HEIGHT); + }); + + it("ends above the HUD bar and its tooltips when both start at the work area's top edge", () => { + // Notes opens at the top edge (notesPlacement.ts); the HUD's top is at that edge or below. + expect(notesWindowHeight(true) + HUD_TOOLTIP_STACK_HEIGHT).toBeLessThanOrEqual( + WAYLAND_HUD_BAR_TOP, + ); + // What it replaces: at 540px Notes reached into the bar even from the same corner. + expect(NOTES_WINDOW_HEIGHT).toBeGreaterThan(WAYLAND_HUD_BAR_TOP); + }); + + it("follows the bar when the HUD reports less room above it", () => { + // A 600px screen leaves the bar 374px below the HUD's top (CodeRabbit on #1586). + expect(notesWindowHeight(true, 374)).toBe(374 - HUD_TOOLTIP_STACK_HEIGHT); + expect(notesWindowHeight(true, 900)).toBe(NOTES_WINDOW_HEIGHT); + expect(notesWindowHeight(true, 503.6)).toBe(417); + }); + + it("stops at a readable floor on a screen too short for it", () => { + expect(notesWindowHeight(true, 120)).toBe(NOTES_WINDOW_FLOOR_HEIGHT); + }); +}); diff --git a/electron/hudWindowBounds.ts b/electron/hudWindowBounds.ts index 071c89a109..259579c5ab 100644 --- a/electron/hudWindowBounds.ts +++ b/electron/hudWindowBounds.ts @@ -18,12 +18,47 @@ // Pure geometry on purpose: `screen` and the window are the caller's job, so the // rules can be tested without an Electron runtime. +import { + HUD_EDGE_SLACK, + HUD_MODAL_MAX_HEIGHT, + HUD_MODAL_MIN_HEIGHT, + HUD_POPOVER_GAP, + HUD_TOOLTIP_STACK_HEIGHT, +} from "../src/components/launch/hudGeometry"; import { clampRectToWorkArea, type DisplayWorkArea, type EditorWindowRect, } from "./editorWindowState"; +/** The Notes window's opening height, and the least it can be resized to. */ +export const NOTES_WINDOW_HEIGHT = 540; +export const NOTES_WINDOW_MIN_HEIGHT = 400; + +/** + * Where the bar's top sits in the Wayland HUD window, from the window's top edge: the window + * keeps the tallest panel's room above the bar (`computeHudLinuxStack`), so this is that room + * plus the edge slack. Measured 503px on a 1080p GNOME desktop. + */ +export const WAYLAND_HUD_BAR_TOP = HUD_EDGE_SLACK + HUD_POPOVER_GAP + HUD_MODAL_MAX_HEIGHT; + +/** Below this Notes is too short to read from: a screen with less room above the bar gets the bar covered. */ +export const NOTES_WINDOW_FLOOR_HEIGHT = HUD_MODAL_MIN_HEIGHT; + +/** + * The height Notes settles at. On Wayland, Notes opens at the work area's top edge, which the + * HUD's window cannot rise above (notesPlacement.ts), so the HUD's visible stack is at least + * `stackTop` below Notes' top: the stack's top in the HUD window, as the renderer last reported + * it. The room above the bar shrinks on a short screen or with the tall vertical bar, so the + * reported top is used when there is one, and the full reserve's otherwise. Notes stops short of + * the room the bar's tooltips take above it, so it covers neither; the user can make it taller. + */ +export function notesWindowHeight(wayland: boolean, stackTop: number | null = null): number { + if (!wayland) return NOTES_WINDOW_HEIGHT; + const clearOfBar = (stackTop ?? WAYLAND_HUD_BAR_TOP) - HUD_TOOLTIP_STACK_HEIGHT; + return Math.floor(Math.max(NOTES_WINDOW_FLOOR_HEIGHT, Math.min(NOTES_WINDOW_HEIGHT, clearOfBar))); +} + /** The HUD's own floor, matching `minWidth`/`minHeight` at construction. */ export const HUD_WINDOW_MIN = { width: 120, height: 80 }; diff --git a/electron/i18n.ts b/electron/i18n.ts index 8c8d55bff8..5ebb14825c 100644 --- a/electron/i18n.ts +++ b/electron/i18n.ts @@ -19,6 +19,10 @@ import commonJa from "../src/i18n/locales/ja-JP/common.json"; import dialogsJa from "../src/i18n/locales/ja-JP/dialogs.json"; import commonKo from "../src/i18n/locales/ko-KR/common.json"; import dialogsKo from "../src/i18n/locales/ko-KR/dialogs.json"; +import commonNl from "../src/i18n/locales/nl/common.json"; +import dialogsNl from "../src/i18n/locales/nl/dialogs.json"; +import commonPl from "../src/i18n/locales/pl/common.json"; +import dialogsPl from "../src/i18n/locales/pl/dialogs.json"; import commonPtBr from "../src/i18n/locales/pt-BR/common.json"; import dialogsPtBr from "../src/i18n/locales/pt-BR/dialogs.json"; import commonRu from "../src/i18n/locales/ru/common.json"; @@ -42,6 +46,8 @@ type Locale = | "it" | "ja-JP" | "ko-KR" + | "nl" + | "pl" | "pt-BR" | "ru" | "tr" @@ -61,6 +67,8 @@ const messages: Record> = { it: { common: commonIt, dialogs: dialogsIt }, "ja-JP": { common: commonJa, dialogs: dialogsJa }, "ko-KR": { common: commonKo, dialogs: dialogsKo }, + nl: { common: commonNl, dialogs: dialogsNl }, + pl: { common: commonPl, dialogs: dialogsPl }, "pt-BR": { common: commonPtBr, dialogs: dialogsPtBr }, ru: { common: commonRu, dialogs: dialogsRu }, tr: { common: commonTr, dialogs: dialogsTr }, @@ -82,6 +90,8 @@ export function setMainLocale(locale: string) { locale === "it" || locale === "ja-JP" || locale === "ko-KR" || + locale === "nl" || + locale === "pl" || locale === "pt-BR" || locale === "ru" || locale === "tr" || diff --git a/electron/ipc/cursorTelemetryLifecycle.test.ts b/electron/ipc/cursorTelemetryLifecycle.test.ts new file mode 100644 index 0000000000..ca5e00a160 --- /dev/null +++ b/electron/ipc/cursorTelemetryLifecycle.test.ts @@ -0,0 +1,184 @@ +import fs from "node:fs"; +import path from "node:path"; +import { runInNewContext } from "node:vm"; +import ts from "typescript"; +import { describe, expect, it, vi } from "vitest"; +import { + compactCursorTelemetryPauseRanges, + finalizeSampledCursorTelemetry, +} from "../native-bridge/cursor/recording/cursorPauseRanges"; + +const source = fs.readFileSync(new URL("./handlers.ts", import.meta.url), "utf8"); + +function execute(code: string, context: object) { + runInNewContext( + ts.transpileModule(code, { + compilerOptions: { target: ts.ScriptTarget.ES2022, module: ts.ModuleKind.None }, + }).outputText, + context, + ); +} + +describe("native Linux fallback timing", () => { + it("starts the sampler on the helper pause-range clock, not picker or delayed sampler time", async () => { + const start = source.indexOf("async function startCursorRecording("); + const end = source.indexOf("\nasync function stopCursorRecording", start); + const create = vi.fn((_options: { startTimeMs?: number }) => ({ + start: vi.fn(async () => undefined), + })); + const context = { + linuxNativeCaptureSession: { cursorStartTimeMs: 1000 }, + isX11PointerFallbackEnabled: () => true, + cursorRecordingSession: null, + pendingCursorRecordingData: null, + createX11PointerRecordingSession: create, + getX11CaptureBounds: () => null, + MAX_CURSOR_SAMPLES: 100, + CURSOR_SAMPLE_INTERVAL_MS: 33, + console: { info: vi.fn() }, + startCursorRecording: undefined as unknown as (id: number) => Promise, + }; + execute(source.slice(start, end), context); + await context.startCursorRecording(100); + expect(create.mock.calls[0][0]).toMatchObject({ startTimeMs: 1000 }); + }); + function fixture(mode = "editable-overlay", helperHasSamples = false) { + const start = source.indexOf('ipcMain.handle("stop-native-linux-recording"'); + const end = source.indexOf("\n\t});", start) + 7; + const handle = vi.fn(); + let finish!: () => void; + const done = new Promise((resolve) => { + finish = resolve; + }); + const cursor = { + version: 2 as const, + provider: "none" as const, + assets: [], + samples: (helperHasSamples ? [42] : []).map((timeMs) => ({ timeMs, cx: 0.5, cy: 0.5 })), + }; + const result = { + path: "/take.mp4", + durationMs: 500, + interrupted: false, + cursor, + cursorPauseRanges: [{ startMs: 200, endMs: 400 }], + }; + const session = { + stop: vi.fn(async () => { + await done; + return result; + }), + discard: vi.fn(), + }; + const context = { + ipcMain: { handle }, + linuxNativeCaptureSession: session, + linuxNativeCaptureRecordingId: 7, + linuxNativeCaptureCursorMode: mode, + pendingCursorRecordingData: null as unknown, + stopCursorRecording: vi.fn(async () => { + context.pendingCursorRecordingData = { + ...cursor, + samples: [100, 200, 300, 400, 500, 700, 701, 1500].map((timeMs) => ({ + timeMs, + cx: 0.5, + cy: 0.5, + })), + }; + }), + fs: { + writeFile: vi.fn(async (_path: string, _body: string) => undefined), + rm: vi.fn(async () => undefined), + }, + compactCursorTelemetryPauseRanges, + finalizeSampledCursorTelemetry, + path, + RECORDINGS_DIR: "/recordings", + RECORDING_FILE_PREFIX: "recording-", + RECORDING_SESSION_SUFFIX: ".json", + setCurrentRecordingSessionState: vi.fn(), + currentProjectPath: null, + nativeRecordingWarning: null, + mainT: () => "interrupted", + formatDuration: () => "0:00", + registerRecordingMediaLinks: vi.fn(), + console: { info: vi.fn(), error: vi.fn() }, + linuxNativeCaptureSourceLabel: null, + linuxSourceLabel: () => "screen", + setAppRecording: vi.fn(), + }; + execute(source.slice(start, end), context); + return { + context, + session, + finish, + result, + stop: handle.mock.calls[0][1] as (e: unknown, discard?: boolean) => Promise, + }; + } + it("stops the sampler before helper teardown completes and bounds compacted samples to video", async () => { + const { context, finish, stop } = fixture(); + const pending = stop({}); + const stoppedBeforeHelper = context.stopCursorRecording.mock.calls.length; + finish(); + await pending; + const call = context.fs.writeFile.mock.calls.find(([p]) => p.endsWith(".cursor.json")); + expect(call).toBeDefined(); + expect(JSON.parse(call![1]).samples.map((s: { timeMs: number }) => s.timeMs)).toEqual([ + 100, 300, 500, + ]); + expect(stoppedBeforeHelper).toBe(1); + }); + it("bounds interrupted takes and closes a pause still active at stop", async () => { + const { context, finish, stop, result } = fixture(); + result.interrupted = true; + result.cursorPauseRanges = [{ startMs: 200, endMs: 1500 }]; + result.durationMs = 200; + const pending = stop({}); + finish(); + await pending; + const call = context.fs.writeFile.mock.calls.find(([p]) => p.endsWith(".cursor.json")); + expect(JSON.parse(call![1]).samples.map((s: { timeMs: number }) => s.timeMs)).toEqual([100]); + }); + it.each([ + "system", + "editable-overlay", + ])("starts a sampler only for editable native mode (%s)", async (mode) => { + const nativeStart = source.indexOf('ipcMain.handle(\n\t\t"start-native-linux-recording"'); + expect(nativeStart).toBeGreaterThan(-1); + const start = source.indexOf('if (cursorCaptureMode === "editable-overlay")', nativeStart); + const end = source.indexOf("\n\t\t\t\treturn { success: true, recordingId", start); + expect(end).toBeGreaterThan(start); + const startCursorRecording = vi.fn(); + const context = { cursorCaptureMode: mode, startCursorRecording, recordingId: 7 }; + const code = `(async () => {${source.slice(start, end)}})()`; + await runInNewContext( + ts.transpileModule(code, { compilerOptions: { target: ts.ScriptTarget.ES2022 } }).outputText, + context, + ); + expect(startCursorRecording).toHaveBeenCalledTimes(mode === "editable-overlay" ? 1 : 0); + }); + + it("prefers helper telemetry without re-compacting it", async () => { + const { context, finish, stop } = fixture("editable-overlay", true); + const pending = stop({}); + finish(); + await pending; + const call = context.fs.writeFile.mock.calls.find(([p]) => p.endsWith(".cursor.json")); + expect(JSON.parse(call![1]).samples.map((s: { timeMs: number }) => s.timeMs)).toEqual([42]); + }); + it("system cursor publishes no editable sidecar", async () => { + const { context, finish, stop } = fixture("system"); + const pending = stop({}); + finish(); + await pending; + expect(context.fs.writeFile.mock.calls.some(([p]) => p.endsWith(".cursor.json"))).toBe(false); + }); + it("discard publishes no sidecar and does not stop the helper for storage", async () => { + const { context, session, stop } = fixture(); + await stop({}, true); + expect(session.discard).toHaveBeenCalledOnce(); + expect(session.stop).not.toHaveBeenCalled(); + expect(context.fs.writeFile).not.toHaveBeenCalled(); + }); +}); diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index 75c3a23df7..478f135eac 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -18,6 +18,7 @@ import { shell, systemPreferences, } from "electron"; +import { z } from "zod"; import { DEFAULT_WEBCAM_QUALITY, type WebcamQualityId } from "../../src/hooks/webcamCaptureTarget"; import { type AxcutDocument, @@ -26,13 +27,16 @@ import { } from "../../src/lib/ai-edition/schema"; import type { CliSourcesResult } from "../../src/lib/cliContracts"; import { pickRecordingSource } from "../../src/lib/cliSourcePick"; +import { desktopSourceTimeoutMessage } from "../../src/lib/desktopSourceTimeout"; import { + type NativeLinuxCaptureSource, type NativeLinuxRecordingRequest, portalCursorMode, } from "../../src/lib/nativeLinuxRecording"; import { collectMacCaptureExcludedWindowIds, type NativeMacRecordingRequest, + updateMacPresenterOverlayState, } from "../../src/lib/nativeMacRecording"; import type { NativeWindowsRecordingRequest } from "../../src/lib/nativeWindowsRecording"; import { @@ -69,9 +73,13 @@ import { DocumentService } from "../ai-edition/document-service"; import { LlmConfigStore } from "../ai-edition/llm-config-store"; import { StylePresetService } from "../ai-edition/style-preset-service"; import { AppSettingsStore } from "../app-settings"; +import { writeFileReplacing } from "../atomic-rename"; +import { bringToFront } from "../bringToFront"; +import { CameraPreviewController } from "../cameraPreview"; import { createRunnerHost } from "../cli/runnerHost"; import { hideCountdownOverlay } from "../countdownOverlay"; -import { isDiagnosticModeEnabled, mainLogBuffer } from "../diagnostics/main-log-buffer"; +import { isDiagnosticModeEnabled, mainLogFile } from "../diagnostics/main-log-file"; +import { createZip } from "../diagnostics/zip"; import { mainT } from "../i18n"; import { getInstallChannel } from "../install-channel"; import { @@ -88,6 +96,7 @@ import { McpController } from "../mcp/mcp-controller"; import { McpSettingsStore } from "../mcp/mcp-settings-store"; import { createMediaTools, type MediaToolDeps, type RecordRequest } from "../mcp/media-tools"; import { createToolRunner, type McpToolDeps } from "../mcp/openscreen-mcp-server"; +import { RecordingCompletion } from "../mcp/recording-completion"; import { type AudioPeaksResult, getAudioPeaks } from "../media/audioPeaks"; import { readCursorRecordingFile as readCursorRecordingFileFrom, @@ -98,17 +107,26 @@ import { findMediaLinksByFingerprint, registerMediaLinks } from "../media/mediaL import { relinkProjectMedia } from "../media/projectMediaRelinker"; import { showOpenDialogOver, showSaveDialogOver } from "../messageBox"; import { + isPortalCancelled, type LinuxCaptureSourceKind, LinuxNativeCaptureSession, } from "../native-bridge/capture/linuxNativeCaptureSession"; -import { compactCursorTelemetryPauseRanges } from "../native-bridge/cursor/recording/cursorPauseRanges"; -import { createCursorRecordingSession } from "../native-bridge/cursor/recording/factory"; +import { + compactCursorTelemetryPauseRanges, + finalizeSampledCursorTelemetry, +} from "../native-bridge/cursor/recording/cursorPauseRanges"; +import { + createCursorRecordingSession, + createX11PointerRecordingSession, + isX11PointerFallbackEnabled, +} from "../native-bridge/cursor/recording/factory"; import { isMacCursorHelperUnavailable, requestMacCursorAccessibilityAccess, } from "../native-bridge/cursor/recording/macNativeCursorRecordingSession"; -import { findPipeWireCursorHelperPath } from "../native-bridge/cursor/recording/pipeWireCursorRecordingSession"; +import { findLinuxCaptureHelperPath } from "../native-bridge/cursor/recording/pipeWireCursorRecordingSession"; import type { CursorRecordingSession } from "../native-bridge/cursor/recording/session"; +import { resolveX11CaptureBounds } from "../native-bridge/cursor/recording/x11CaptureBounds"; import { toHelperRect } from "../native-bridge/helperCoordinates"; import { isMacPickerSourceId, @@ -123,8 +141,15 @@ import { CompositorViewService } from "../native-bridge/services/compositorViewS import { getMacPermissions, showPermissionsWindow } from "../permissions"; import { scoreDeviceNameMatch } from "../recording/deviceNameMatching"; import { recordingStorageProblem } from "../recording/diskSpaceCheck"; +import { + linuxCaptureBackend, + linuxPortalOwnsSources, + resetLinuxCaptureBackendProbe, + resolveX11CaptureSource, +} from "../recording/linuxCaptureBackend"; import { describeSalvagedTake, + formatDuration, nativeMacSalvageTarget, salvageNativeMacCapture, } from "../recording/nativeMacCaptureSalvage"; @@ -141,8 +166,11 @@ import { readMicrophoneNotFound, readMicrophoneUnavailable, readSecondaryWindowsApplied, + readSoftwareEncoderAdapters, + readStalledEncoderAdapter, readWebcamFormat, readWebcamUnavailable, + rememberSoftwareEncoderAdapter, terminateNativeWindowsCapture, waitForNativeWindowsCaptureStop, } from "../recording/nativeWindowsCaptureStop"; @@ -161,7 +189,23 @@ import { shouldPersistSelectedSource, } from "../recording-source-settings"; import { getSttManager } from "../stt"; -import { createRunnerWindow } from "../windows"; +import { usesWaylandBackend } from "../wayland"; +import { + createCameraPreviewWindow, + createRunnerWindow, + getCameraPreviewWindow, + parkCameraPreviewWindow, + resizeCameraPreviewWindow, +} from "../windows"; +import { + assertWindowsUpdateIdle, + isWindowsUpdatePending, + WINDOWS_UPDATE_PENDING_MESSAGE, +} from "../windows-update"; +import { + createWindowsCameraPreviewMailbox, + watchWindowsCameraPreview, +} from "../windowsCameraPreview"; import { getSourcesByType, shouldEnumerateSourceTypesSeparately } from "./desktopSourceTypes"; import { registerNativeBridgeHandlers } from "./nativeBridge"; import { createNativeMacMidCaptureErrorWatch } from "./nativeMacMidCaptureErrorWatch"; @@ -185,6 +229,11 @@ const ALLOWED_IMPORT_VIDEO_EXTENSIONS = new Set([ ".ts", ]); const PREVIEW_AUDIO_DIR = path.join(app.getPath("userData"), "preview-audio"); +/** Adapters whose hardware H.264 encoder stalled on stop (getopenscreen/openscreen#1544). */ +const WINDOWS_SOFTWARE_ENCODER_ADAPTERS_FILE = path.join( + app.getPath("userData"), + "windows-software-encoder-adapters.json", +); // See the save-recorded-voiceover handler: an upper bound on renderer-supplied // bytes written to disk, well past any plausible take. const MAX_RECORDED_VOICEOVER_BYTES = 512 * 1024 * 1024; @@ -872,11 +921,12 @@ const nativeMacCaptureExits = new WeakMap(); /** - * Why the last macOS take ended before it was stopped, keyed by the file it was - * kept in. Handed to whatever opens that recording next — the editor or the CLI — + * What the user must hear about the last native take, keyed by the file it was kept + * in: why a macOS take ended before it was stopped, or that a Linux take lost its audio. Handed to whatever opens that recording next — the editor or the CLI — * because the HUD that ran the stop closes as the editor opens. */ -let nativeMacRecordingWarning: { screenVideoPath: string; message: string } | null = null; +let nativeRecordingWarning: { screenVideoPath: string; message: string } | null = null; +let nativeMacPresenterOverlay = { active: false, cameraInterrupted: false }; /** True while stop-native-mac-recording runs: the take is ending on purpose. */ let nativeMacStopInFlight = false; // Global frame the macOS cursor is normalised against (see getSelectedSourceBounds). @@ -1137,6 +1187,19 @@ function isMacWindowCaptureSourceId(sourceId: string | null | undefined) { ); } +// Electron pointer positions and display bounds are DIP; helper origins may +// instead be X11 pixels. Resolve conservatively, never mix those coordinate spaces. +// Only native Linux capture runs this sampler, so the portal grant is the source of truth. +function getX11CaptureBounds() { + if (!linuxNativeCaptureSession) return null; + return resolveX11CaptureBounds({ + displays: screen.getAllDisplays(), + sourceKind: linuxNativeCaptureSession.grantedSourceKind, + origin: linuxNativeCaptureSession.grantedCaptureOrigin, + size: linuxNativeCaptureSession.grantedCaptureSize, + }); +} + function getSelectedSourceBounds() { // A window take is normalised against the rect the helper is recording. That // rect is seeded from the source frame before the first cursor sample and @@ -1334,11 +1397,40 @@ async function resolveDirectShowWebcamClsid(deviceName?: string) { async function startCursorRecording(recordingId?: number) { // On Linux the native capture helper already produces cursor samples, from // the SAME portal session that produces the pixels. Spawning the cursor-only - // helper alongside it would open a SECOND portal session — and since + // PipeWire helper alongside it would open a SECOND portal session — and since // SelectSources may be called once per session, that means a second picker // in front of the user, for a stream nothing consumes. The double prompt is // precisely what the native path exists to remove. + // + // X11 is the exception: the portal often has no METADATA cursor mode at all, + // so the helper records pixels with an empty cursor track. Sampling the + // pointer through X11 (Electron / xinput) does not talk to the portal, + // so it can run in parallel and fill `