diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 936b1c0945..c472c1282f 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -15,6 +15,7 @@ "skills": [ "./skills/hyperframes", "./skills/hyperframes-animation", + "./skills/hyperframes-audio", "./skills/hyperframes-cli", "./skills/hyperframes-core", "./skills/hyperframes-creative", diff --git a/.fallowrc.jsonc b/.fallowrc.jsonc index f5b1617118..3a9511683b 100644 --- a/.fallowrc.jsonc +++ b/.fallowrc.jsonc @@ -176,22 +176,6 @@ "withLane", ], }, - // automationShapes is part of the audio-automation stack: its consumer is - // the UI layer that uses shape generators one PR upstack, so a per-PR audit - // diffing against the merge base sees these as unused. Consumed for real once - // the stack merges; safe to drop this entry then. - { - "file": "packages/studio/src/player/components/automationShapes.ts", - "exports": ["AUTOMATION_SHAPES"], - }, - // automationSimplify is part of the audio-automation stack: its consumer is - // the UI layer one PR upstack, so a per-PR audit diffing against the merge - // base sees these as unused. Consumed for real once the stack merges; safe - // to drop this entry then. - { - "file": "packages/studio/src/player/components/automationSimplify.ts", - "exports": ["simplifyPoints"], - }, // propertyPanelAutomation is the shared reader for both panel sections; the // FX group that consumes these two lands one PR upstack, so a per-PR audit // against the merge base sees them as unused. diff --git a/CLAUDE.md b/CLAUDE.md index 638926508f..31edd2a183 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,12 +4,12 @@ Open-source video rendering framework: write HTML, render video. ## Skills -This repo ships 19 AI agent skills via [vercel-labs/skills](https://github.com/vercel-labs/skills). Install them before writing compositions — they encode framework-specific patterns that generic docs don't cover. **Default to the core set**: the `/hyperframes` router installs each creation workflow on demand; install all 19 only when the user explicitly asks for the full set. +This repo ships 20 AI agent skills via [vercel-labs/skills](https://github.com/vercel-labs/skills). Install them before writing compositions — they encode framework-specific patterns that generic docs don't cover. **Default to the core set**: the `/hyperframes` router installs each creation workflow on demand; install all 20 only when the user explicitly asks for the full set. ```bash npx hyperframes skills update # default: installs/refreshes the core set — workflows install on demand -npx skills add heygen-com/hyperframes --full-depth # interactive picker (terminal only — non-interactive without --skill installs all 19) -npx skills add heygen-com/hyperframes --all --full-depth # all 19 at once — only on explicit request +npx skills add heygen-com/hyperframes --full-depth # interactive picker (terminal only — non-interactive without --skill installs all 20) +npx skills add heygen-com/hyperframes --all --full-depth # all 20 at once — only on explicit request npx skills add heygen-com/hyperframes --skill --full-depth # just one (bare name, no leading slash) ``` @@ -39,6 +39,7 @@ Atomic capabilities the creation workflows compose against — pull one when you - `/hyperframes-keyframes` — seek-safe keyframe authoring across runtimes: GSAP timelines, CSS keyframes, Anime.js, WAAPI, FLIP, paths, masks, SVG morph/draw, text trails, 3D depth; plus `hyperframes keyframes` diagnostics for surfacing and verifying rendered motion. - `/hyperframes-creative` — non-animation creative direction: `frame.md` / `design.md` handling, palettes, typography, narration, beat planning, audio-reactive visuals, composition patterns. - `/media-use` — the media OS: resolve any media need (BGM, SFX, image, icon, logo, voice, color grade, LUT) into a frozen local file or paste-ready block + ledger record; generate via TTS / music / image models when the catalog misses; transcribe, caption, remove backgrounds, and reuse assets across projects. One shared `scripts/audio.mjs` engine + manifest tracking; keeps search noise on disk. +- `/hyperframes-audio` — mix the audio already placed in a composition: voiceover carve (dip a music bed only in the bands the voice occupies, static or dynamic, level match included), the effect chain (EQ, compressor, limiter, gate, saturation, delay, reverb, chorus, phaser, bitcrush), and automation envelopes on volume or any effect parameter. Sourcing the audio is `/media-use`; this is what happens to it afterwards. - `/hyperframes-cli` — CLI dev loop: `init`, `add`, `lint`, `check`, `snapshot`, `preview`, `render`, `publish`, `doctor`, `lambda` (AWS Lambda cloud rendering). - `/hyperframes-registry` — install and wire registry blocks and components into compositions via `hyperframes add`. Covers authoring a new block or component to contribute upstream. - `/figma` — import Figma assets, tokens, components, and storyboard sections → reconstructed motion (frames read as states, not slides) (REST/CLI) plus Motion animations (MCP) and shaders (MCP source / native export) into a composition. @@ -51,7 +52,7 @@ When adding a new skill, or substantially renaming / repurposing an existing one 2. The scaffolded project template `packages/cli/src/templates/_shared/CLAUDE.md` + `AGENTS.md` — written into every `hyperframes init` project, so a stale entry there ships to users. The two template files must stay byte-identical. 3. If the skill changes the routing surface for "make a video" requests, also update the routing table + intent layer in `skills/hyperframes/SKILL.md` AND that workflow's own route file, `skills/hyperframes/references/routes/.md`. One file carries both halves: the input/output/trigger contract the router reads before the workflow is installed, and its interview entry (must-haves, conditionals, deferred asks, run-shape). The older `references/workflow-catalog.md` and `references/route-briefs.md` are now "moved" stubs pointing at `routes/` — don't edit them. 4. Mirror the Router / Creation workflows / Domain skills grouping across all surfaces so a skill always lives in the same column. -5. Skill count appears in the README and CLAUDE.md intro lines ("19 AI agent skills…") — update on add/remove. The `docs/guides/skills.mdx` page and the CLI templates deliberately omit a count to avoid drift; keep them count-free. +5. Skill count appears in the README and CLAUDE.md intro lines ("20 AI agent skills…") — update on add/remove. The `docs/guides/skills.mdx` page and the CLI templates deliberately omit a count to avoid drift; keep them count-free. The skill's own `SKILL.md` frontmatter `description:` is the source of truth for the one-line "use when" blurb; copy from there into the catalog rather than paraphrasing. diff --git a/README.md b/README.md index 2a34005a40..df182d6b42 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ Install the HyperFrames skills, then describe the video you want: npx skills add heygen-com/hyperframes --full-depth ``` -> The picker opens with nothing pre-selected — the **Core Skills** group is all you need: the `/hyperframes` router installs each creation workflow on demand. Agents and non-interactive runs should use `npx hyperframes skills update` instead — it installs exactly the core set, whereas a non-interactive `skills add` without `--skill` installs all 19. +> The picker opens with nothing pre-selected — the **Core Skills** group is all you need: the `/hyperframes` router installs each creation workflow on demand. Agents and non-interactive runs should use `npx hyperframes skills update` instead — it installs exactly the core set, whereas a non-interactive `skills add` without `--skill` installs all 20. > > `--full-depth` does a full clone of the repo's current `main`. Without it, `skills add` fetches the skills.sh registry blob, which lags `main` by hours — you'd get an older copy of a skill. (`hyperframes skills update` already installs full-depth.) @@ -53,9 +53,9 @@ The skills teach agents the HyperFrames production loop: plan the video, write v ## Skills -HyperFrames ships 19 skills agents load on demand. Read `/hyperframes` first — it's the router and capability map; it picks a workflow for any "make me a…" request — video, deck, or composition port — and points to the domain skills below. +HyperFrames ships 20 skills agents load on demand. Read `/hyperframes` first — it's the router and capability map; it picks a workflow for any "make me a…" request — video, deck, or composition port — and points to the domain skills below. -Default to the **core set** — the router installs each creation workflow on demand. `npx hyperframes skills update` installs exactly that from anywhere; the interactive picker (`npx skills add heygen-com/hyperframes --full-depth`) lists it as the "Core Skills" group, nothing pre-selected. The picker is interactive-only — a non-interactive or agent run without `--skill` installs all 19. Use `npx skills add heygen-com/hyperframes --all --full-depth` to install all 19 deliberately (skips the picker), or `npx skills add heygen-com/hyperframes --skill --full-depth` for just one (bare name, no leading `/`). Keep `--full-depth` — it installs the current `main`; without it `skills add` fetches the skills.sh blob, which lags by hours. +Default to the **core set** — the router installs each creation workflow on demand. `npx hyperframes skills update` installs exactly that from anywhere; the interactive picker (`npx skills add heygen-com/hyperframes --full-depth`) lists it as the "Core Skills" group, nothing pre-selected. The picker is interactive-only — a non-interactive or agent run without `--skill` installs all 20. Use `npx skills add heygen-com/hyperframes --all --full-depth` to install all 20 deliberately (skips the picker), or `npx skills add heygen-com/hyperframes --skill --full-depth` for just one (bare name, no leading `/`). Keep `--full-depth` — it installs the current `main`; without it `skills add` fetches the skills.sh blob, which lags by hours. Installs stay lean after that: `npx hyperframes init` keeps the **core set** fresh (the router, the `hyperframes-*` domain skills, and `media-use` — plus whatever is already installed; `/figma` stays on demand) and never expands a partial install; the creation workflows install **on demand** — the router runs `npx hyperframes skills update ` before entering one. Nothing re-pulls the full set behind your back. @@ -102,6 +102,7 @@ Atomic capabilities the creation workflows compose against — pull one when you | `/hyperframes-creative` | Non-animation creative direction — `frame.md` / `design.md`, palettes, typography, narration, beat planning, audio-reactive visuals, composition patterns. | | `/media-use` | The media OS — resolve any media need (BGM, SFX, image, icon, logo, voice, color grade, LUT) into a frozen local file or paste-ready block + ledger record, generate via TTS/music/image models when the catalog misses, transcribe, caption, remove backgrounds, and reuse assets across projects. One shared audio engine + manifest tracking. | | `/hyperframes-cli` | CLI dev loop — `init`, `lint`, `check`, `snapshot`, `preview`, `render`, `publish`, `doctor`, plus HeyGen-hosted cloud rendering (`cloud render`) and AWS Lambda rendering (`lambda deploy / render / progress`). | +| `/hyperframes-audio` | Mix the audio already placed in a composition — voiceover carve (dip a music bed only in the bands the voice occupies, static or dynamic, level match included), the effect chain (EQ, compressor, limiter, gate, saturation, delay, reverb, chorus, phaser, bitcrush), and automation envelopes on volume or any effect parameter. Sourcing the audio is `/media-use`. | | `/hyperframes-registry` | Install and wire registry blocks and components into compositions via `hyperframes add`. Authoring a new block or component to contribute upstream. | | `/figma` | Import Figma assets, tokens, components, and storyboard sections → reconstructed motion (frames read as states, not slides) (REST/CLI) plus Motion animations (MCP) and shaders (MCP source / native export) into a composition. | diff --git a/packages/cli/src/templates/_shared/AGENTS.md b/packages/cli/src/templates/_shared/AGENTS.md index 5c8126aeb2..1762ea7a66 100644 --- a/packages/cli/src/templates/_shared/AGENTS.md +++ b/packages/cli/src/templates/_shared/AGENTS.md @@ -18,7 +18,7 @@ **Porting an existing composition?** `/remotion-to-hyperframes` translates a Remotion (React) composition into HyperFrames HTML — a source migration, separate from the creation workflows above. -The domain skills (`/hyperframes-core`, `/hyperframes-animation`, `/hyperframes-keyframes`, `/hyperframes-creative`, `/hyperframes-cli`, `/media-use`, `/hyperframes-registry`, `/figma`) and the full capability map live inside `/hyperframes` — it is the single source of truth for which skill handles which intent. +The domain skills (`/hyperframes-core`, `/hyperframes-animation`, `/hyperframes-keyframes`, `/hyperframes-creative`, `/hyperframes-cli`, `/media-use`, `/hyperframes-audio`, `/hyperframes-registry`, `/figma`) and the full capability map live inside `/hyperframes` — it is the single source of truth for which skill handles which intent. **Changing how real footage or images look or reveal?** Load `/media-use` and read its `references/media-treatments.md` before editing, even when the request only says dark, flat, boring, retro, private, or “make the reveal cooler.” It governs how footage is treated, never whether media may be used. Use canonical media treatments and seek-safe motion; do not improvise equivalent CSS/SVG filters or overlays. diff --git a/packages/cli/src/templates/_shared/CLAUDE.md b/packages/cli/src/templates/_shared/CLAUDE.md index 5c8126aeb2..1762ea7a66 100644 --- a/packages/cli/src/templates/_shared/CLAUDE.md +++ b/packages/cli/src/templates/_shared/CLAUDE.md @@ -18,7 +18,7 @@ **Porting an existing composition?** `/remotion-to-hyperframes` translates a Remotion (React) composition into HyperFrames HTML — a source migration, separate from the creation workflows above. -The domain skills (`/hyperframes-core`, `/hyperframes-animation`, `/hyperframes-keyframes`, `/hyperframes-creative`, `/hyperframes-cli`, `/media-use`, `/hyperframes-registry`, `/figma`) and the full capability map live inside `/hyperframes` — it is the single source of truth for which skill handles which intent. +The domain skills (`/hyperframes-core`, `/hyperframes-animation`, `/hyperframes-keyframes`, `/hyperframes-creative`, `/hyperframes-cli`, `/media-use`, `/hyperframes-audio`, `/hyperframes-registry`, `/figma`) and the full capability map live inside `/hyperframes` — it is the single source of truth for which skill handles which intent. **Changing how real footage or images look or reveal?** Load `/media-use` and read its `references/media-treatments.md` before editing, even when the request only says dark, flat, boring, retro, private, or “make the reveal cooler.” It governs how footage is treated, never whether media may be used. Use canonical media treatments and seek-safe motion; do not improvise equivalent CSS/SVG filters or overlays. diff --git a/packages/cli/src/utils/skillsManifest.ts b/packages/cli/src/utils/skillsManifest.ts index 5145cc0348..ad613270f4 100644 --- a/packages/cli/src/utils/skillsManifest.ts +++ b/packages/cli/src/utils/skillsManifest.ts @@ -150,6 +150,7 @@ export function isCoreSkill(name: string): boolean { export const FALLBACK_CORE_SKILLS: readonly string[] = [ "hyperframes", "hyperframes-animation", + "hyperframes-audio", "hyperframes-cli", "hyperframes-core", "hyperframes-creative", diff --git a/packages/core/src/audio/audioFxAutomation.test.ts b/packages/core/src/audio/audioFxAutomation.test.ts index d8748f9f45..e73c14ede9 100644 --- a/packages/core/src/audio/audioFxAutomation.test.ts +++ b/packages/core/src/audio/audioFxAutomation.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest"; import { cancelParamLane, + clearParamLane, scheduleChainAutomation, scheduleParamLane, volumeLane, @@ -68,7 +69,11 @@ describe("scheduleParamLane", () => { const { target, param } = fake(); scheduleParamLane([target], ramp, "linear", at(0)); expect(param.calls).toEqual([ - { op: "cancel", time: 10 }, + // Cleared from zero, not held: holding leaves the span of a running curve + // booked, and Chrome refuses the next curve — or a plain `.value` write — + // that lands inside it. The seed below restores the value in the same pass, + // so clearing costs nothing audible. + { op: "cancel", time: 0 }, // Before the first point the envelope holds that point's value. { op: "set", value: 0.2, time: 10 }, { op: "ramp", value: 0.8, time: 13 }, @@ -87,7 +92,7 @@ describe("scheduleParamLane", () => { const { target, param } = fake(); scheduleParamLane([target], ramp, "linear", at(9)); expect(param.calls).toEqual([ - { op: "cancel", time: 10 }, + { op: "cancel", time: 0 }, { op: "set", value: 0.8, time: 10 }, ]); }); @@ -122,7 +127,7 @@ describe("scheduleParamLane", () => { at(0), ); expect(param.calls).toEqual([ - { op: "cancel", time: 10 }, + { op: "cancel", time: 0 }, { op: "set", value: 0.4, time: 10 }, ]); }); @@ -152,6 +157,31 @@ describe("scheduleParamLane", () => { expect(curve.values[Math.floor(curve.values.length / 2)]).toBeLessThan(0.4); }); + it("samples a segment bent by a via point, not just one bent by `curve`", () => { + // Both express the same thing — a segment that is not a straight line — and + // the via form is what the timeline writes when a bend is dragged. Read as + // straight, the whole bend was played as a linear ramp: the envelope drawn in + // the lane and the envelope heard were different shapes. + const { target, param } = fake(); + scheduleParamLane( + [target], + { + target: "volume", + points: [ + { t: 0, v: 0, viaX: 0.5, viaY: 0.9 }, + { t: 2, v: 1 }, + ], + }, + "linear", + at(0), + ); + const curve = param.calls.find((c) => c.op === "curve"); + expect(curve, "a via-bent segment must be sampled, not ramped").toBeTruthy(); + if (curve?.op !== "curve") throw new Error("expected a curve"); + // Through the via point: 90% of the way up at the halfway mark. + expect(curve.values[Math.floor(curve.values.length / 2)]).toBeGreaterThan(0.8); + }); + it("samples a log-scaled sweep, which a linear ramp would get wrong", () => { const { target, param } = fake(); scheduleParamLane( @@ -232,6 +262,238 @@ describe("scheduleParamLane", () => { }); }); +/** + * An AudioParam with Chrome's own overlap rule, and its own cancel semantics. + * + * The distinction that matters: `cancelScheduledValues(t)` drops events at or + * after `t` but leaves a value curve that is already running — the spec only + * special-cases an in-progress curve for `cancelAndHoldAtTime`, which truncates + * it. Schedule a curve inside one that is still live and the browser throws. + */ +class OverlapAwareParam { + curves: { time: number; duration: number }[] = []; + value = 0; + setValueAtTime(): void {} + linearRampToValueAtTime(): void {} + setValueCurveAtTime(_values: Float32Array, time: number, duration: number): void { + const clash = this.curves.find((c) => time < c.time + c.duration && time + duration > c.time); + if (clash) { + throw new Error( + `Failed to execute 'setValueCurveAtTime' on 'AudioParam': ` + + `setValueCurveAtTime(..., ${time}, ${duration}) overlaps ` + + `setValueCurveAtTime(..., ${clash.time}, ${clash.duration})`, + ); + } + this.curves.push({ time, duration }); + } + cancelScheduledValues(time: number): void { + // Events at or after the cancel; a curve already under way is untouched. + this.curves = this.curves.filter((c) => c.time < time); + } + cancelAndHoldAtTime(time: number): void { + this.curves = this.curves + .filter((c) => c.time < time) + .map((c) => ({ time: c.time, duration: Math.min(c.duration, time - c.time) })); + } +} + +describe("rescheduling over a curve that is still playing", () => { + const bent: HfAutomationLane = { + target: "volume", + points: [ + { t: 0, v: 1, curve: 1 }, + { t: 8, v: 0.2, curve: 1 }, + { t: 12, v: 1 }, + ], + }; + + it("takes over from an in-progress curve instead of throwing", () => { + // Two schedule passes a few milliseconds apart is ordinary: an attribute edit + // lands, the graph is re-parameterised, and the envelope is re-aimed at the + // live playhead. The second pass has to displace the curve the first one + // started, which `cancelScheduledValues` does not do for a curve that is + // already running — the browser then refuses the new curve outright. + const param = new OverlapAwareParam(); + const target = { param: param as unknown as AudioParam }; + scheduleParamLane([target], bent, "linear", at(0, 1, 9.109333)); + expect(() => + scheduleParamLane([target], bent, "linear", at(0.005334, 1, 9.114667)), + ).not.toThrow(); + }); + + it("survives a burst of reschedules, as a dragged knob produces", () => { + const param = new OverlapAwareParam(); + const target = { param: param as unknown as AudioParam }; + for (let i = 0; i < 12; i++) { + const when = 9.1 + i * 0.005; + expect(() => + scheduleParamLane([target], bent, "linear", at(i * 0.005, 1, when)), + ).not.toThrow(); + } + }); +}); + +describe("a curve the browser refuses", () => { + /** + * A param that rejects any curve overlapping one it has been given, and — like + * Chrome — keeps counting a held curve's original span. Holding stops what is + * audible; it does not free the slot for overlap checking. + */ + class UnforgivingParam { + curves: { time: number; duration: number }[] = []; + ramps: { time: number; value: number }[] = []; + sets: { time: number; value: number }[] = []; + value = 0; + setValueAtTime(value: number, time: number): void { + this.sets.push({ time, value }); + } + linearRampToValueAtTime(value: number, time: number): void { + this.ramps.push({ time, value }); + } + setValueCurveAtTime(_v: Float32Array, time: number, duration: number): void { + const clash = this.curves.find((c) => time < c.time + c.duration && time + duration > c.time); + if (clash) { + throw new Error( + `Failed to execute 'setValueCurveAtTime' on 'AudioParam': ` + + `setValueCurveAtTime(..., ${time}, ${duration}) overlaps ` + + `setValueCurveAtTime(..., ${clash.time}, ${clash.duration})`, + ); + } + this.curves.push({ time, duration }); + } + // Measured against Chrome, in a live running context and in an offline one + // suspended mid-curve: either cancel frees the span of a curve already under + // way, so only a write with no cancel at all is refused. + cancelScheduledValues(time: number): void { + this.curves = this.curves.filter((c) => c.time + c.duration < time); + } + cancelAndHoldAtTime(time: number): void { + this.cancelScheduledValues(time); + } + } + + const bent: HfAutomationLane = { + target: "volume", + points: [ + { t: 0, v: 1, curve: 1 }, + { t: 0.25, v: 0.4, curve: 1 }, + { t: 0.6, v: 1, curve: 1 }, + { t: 1.2, v: 0.3 }, + ], + }; + + it("reschedules mid-playback with its curves intact", () => { + // Applying a carve while the transport runs reschedules into curves that are + // already playing — one render quantum apart is the common case. Cancelling + // the parameter first is what lets the new pass keep its shape instead of + // being refused; this does not discriminate the strength of the cancel, which + // the no-cancel test below is for. + const param = new UnforgivingParam(); + const target = { param: param as unknown as AudioParam }; + scheduleParamLane([target], bent, "linear", at(0, 1, 1492.016)); + const first = param.curves.length; + expect(() => + scheduleParamLane([target], bent, "linear", at(0.005333, 1, 1492.021333)), + ).not.toThrow(); + expect(param.curves.length).toBeGreaterThan(0); + // Curves, not ramps: nothing had to be degraded. + expect(param.ramps).toHaveLength(0); + expect(first).toBeGreaterThan(0); + }); + + it("still degrades to a ramp where the span cannot be freed at all", () => { + // The last line of defence, for a hypothetical param that refuses to give the + // span up. No engine has been measured behaving this way; it stands in for the + // unexplained curve-over-curve refusals reported from the field. The envelope + // loses a bend rather than the exception escaping and abandoning the rest. + class ImmovableParam extends UnforgivingParam { + override cancelScheduledValues(): void { + // Nothing is ever freed. + } + } + const param = new ImmovableParam(); + const target = { param: param as unknown as AudioParam }; + scheduleParamLane([target], bent, "linear", at(0, 1, 100)); + expect(() => + scheduleParamLane([target], bent, "linear", at(0.005333, 1, 100.005333)), + ).not.toThrow(); + expect(param.ramps.length).toBeGreaterThan(0); + }); +}); + +describe("a curve that starts at this very instant", () => { + /** + * Chrome's behaviour at the boundary, which is where this bit: holding at a + * time does not free a curve that *begins* at that time, so both a fresh curve + * and a plain `.value` write at the same instant are refused. Only a cancel + * clears it. + */ + class BoundaryParam { + curves: { time: number; duration: number }[] = []; + #value = 0; + constructor(private clock: { currentTime: number }) {} + get value(): number { + return this.#value; + } + set value(v: number) { + const t = this.clock.currentTime; + const clash = this.curves.find((c) => t >= c.time && t <= c.time + c.duration); + if (clash) { + throw new Error( + `Failed to set the 'value' property on 'AudioParam': setValueAtTime(${v}, ${t}) ` + + `overlaps setValueCurveAtTime(..., ${clash.time}, ${clash.duration})`, + ); + } + this.#value = v; + } + setValueAtTime(v: number): void { + this.#value = v; + } + linearRampToValueAtTime(): void {} + setValueCurveAtTime(_v: Float32Array, time: number, duration: number): void { + this.curves.push({ time, duration }); + } + // As Chrome behaves: either cancel frees a running curve's span. + cancelScheduledValues(time: number): void { + this.curves = this.curves.filter((c) => c.time + c.duration < time); + } + cancelAndHoldAtTime(time: number): void { + this.cancelScheduledValues(time); + } + } + + const bent: HfAutomationLane = { + target: "volume", + points: [ + { t: 0, v: 1, curve: 1 }, + { t: 0.04, v: 0.2, curve: 1 }, + { t: 4, v: 1 }, + ], + }; + + it("clears the booked span so the next pass can schedule at all", () => { + const clock = { currentTime: 91.069 }; + const param = new BoundaryParam(clock); + const target = { param: param as unknown as AudioParam }; + scheduleParamLane([target], bent, "linear", at(0, 1, 91.069)); + expect(() => scheduleParamLane([target], bent, "linear", at(0, 1, 91.069))).not.toThrow(); + }); + + it("leaves the parameter writable, which is what a chain edit does next", () => { + // The re-parameterise after an edit writes every knob straight onto its param. + // Landing inside a span the scheduler booked at this instant is the error the + // console kept reporting, with the gain stage's own unity value in it. + const clock = { currentTime: 91.069 }; + const param = new BoundaryParam(clock); + const target = { param: param as unknown as AudioParam }; + scheduleParamLane([target], bent, "linear", at(0, 1, 91.069)); + clearParamLane([target]); + expect(() => { + (param as unknown as { value: number }).value = 1; + }).not.toThrow(); + }); +}); + describe("cancelParamLane", () => { it("holds the value the envelope had reached rather than snapping back", () => { const { target, param } = fake(); diff --git a/packages/core/src/audio/audioFxAutomation.ts b/packages/core/src/audio/audioFxAutomation.ts index 766c14ece8..94e7cca6b2 100644 --- a/packages/core/src/audio/audioFxAutomation.ts +++ b/packages/core/src/audio/audioFxAutomation.ts @@ -14,6 +14,7 @@ import { sampleAutomationLane, type HfAutomation, type HfAutomationLane, + type HfAutomationPoint, } from "../audioAutomation.js"; import { parseAutomationTarget, VOLUME_TARGET } from "../audioAutomation.js"; import type { HfAudioFxChain } from "../audioFx.js"; @@ -46,16 +47,23 @@ type Op = | { kind: "curve"; time: number; duration: number; values: Float32Array }; /** - * A segment is a straight line only when nothing bends it: no curvature on the - * point it leaves, a linear parameter scale, and no unit mapping that could be - * non-linear. Everything else is sampled. + * A segment is a straight line only when nothing bends it: no curvature and no + * via point on the point it leaves, a linear parameter scale, and no unit + * mapping that could be non-linear. Everything else is sampled. + * + * Both bend forms have to be checked. `curve` is an exponent; a via point names + * an interior point the segment passes through, and it is what the timeline + * writes when a bend is dragged. Testing only the exponent read every dragged + * bend as straight and played it as a linear ramp — the lane drew one shape and + * the ear got another. */ function isStraight( - curve: number | undefined, + point: Pick, scale: "linear" | "log", map: FxParamTarget["map"], ): boolean { - return !curve && scale === "linear" && !map; + const bent = Boolean(point.curve) || (point.viaX !== undefined && point.viaY !== undefined); + return !bent && scale === "linear" && !map; } function curvePointCount(duration: number): number { @@ -86,7 +94,7 @@ function segmentOp( const duration = toTime - fromTime; if (duration <= 0) return null; - if (isStraight(a.curve, scale, map)) { + if (isStraight(a, scale, map)) { return { kind: "ramp", time: toTime, value: apply(b.v) }; } // Sample from wherever the segment is entered, which is mid-segment when the @@ -128,11 +136,66 @@ function planOps( return ops; } +/** + * Hand the plan to the AudioParam, degrading a curve the browser will not take. + * + * A value curve may not overlap another, and this has been seen refused in the + * field while rescheduling mid-playback — two passes landing a render quantum + * apart, which is what applying a carve while the transport runs does. Every + * caller here cancels the parameter first, and measured against Chrome that is + * enough to free even a curve already under way, so the mechanism behind those + * reports is not understood and does not reproduce. This catch is the backstop + * for it: unhandled, the exception reached the console and abandoned every + * remaining segment of the envelope. + * + * A ramp to the same destination is always accepted, since it books no span. The + * segment loses its bend until the next full reschedule, which is a far better + * failure than the rest of the envelope never being scheduled at all. + */ function emit(param: AudioParam, ops: readonly Op[]): void { for (const op of ops) { if (op.kind === "set") param.setValueAtTime(op.value, op.time); else if (op.kind === "ramp") param.linearRampToValueAtTime(op.value, op.time); - else param.setValueCurveAtTime(op.values, op.time, op.duration); + else { + try { + param.setValueCurveAtTime(op.values, op.time, op.duration); + } catch { + const end = op.values[op.values.length - 1] ?? 0; + param.linearRampToValueAtTime(end, op.time + op.duration); + } + } + } +} + +/** + * Drop every event booked on a parameter, including a curve already under way. + * + * Anything landing inside a running curve is refused — a fresh curve, or the + * plain `.value` write that re-parameterising a chain performs — so the param has + * to be cancelled first. Measured against Chrome, any cancel frees the span; this + * takes the strongest form on purpose, because curve-over-curve refusals were + * reported from the field with a cancel at the new schedule time already in place + * and have never reproduced. Cancelling from zero cannot leave a stale event + * behind whatever that mechanism is. + * + * Safe on these parameters because this scheduler is their only writer: they + * belong to effect nodes it built, and the caller re-seeds the value in the same + * breath, so nothing clicks. + */ +export function clearParamLane(targets: readonly FxParamTarget[]): void { + for (const target of targets) target.param.cancelScheduledValues(0); +} + +/** Stop an envelope, leaving the parameter wherever it currently sits. */ +export function cancelParamLane(targets: readonly FxParamTarget[], from: number): void { + for (const target of targets) { + // Keep the value the ramp had reached rather than snapping back to the + // last explicitly-set one, which would click. + if (typeof target.param.cancelAndHoldAtTime === "function") { + target.param.cancelAndHoldAtTime(from); + } else { + target.param.cancelScheduledValues(from); + } } } @@ -150,7 +213,11 @@ export function scheduleParamLane( ): void { if (lane.points.length === 0) return; for (const target of targets) { - target.param.cancelScheduledValues(timing.scheduledAt); + // Cleared from zero rather than from this instant: a reschedule mid-playback + // lands inside the previous pass's still-running curve, and the field reports + // of that being refused had a cancel at this instant already in place. The + // plan below re-seeds the value here, so clearing costs nothing audible. + clearParamLane([target]); if (isConstantLane(lane)) { const value = (target.map ?? identity)(lane.points[0]!.v); target.param.setValueAtTime(value, timing.scheduledAt); @@ -160,19 +227,6 @@ export function scheduleParamLane( } } -/** Stop an envelope, leaving the parameter wherever it currently sits. */ -export function cancelParamLane(targets: readonly FxParamTarget[], from: number): void { - for (const target of targets) { - // Keep the value the ramp had reached rather than snapping back to the - // last explicitly-set one, which would click. - if (typeof target.param.cancelAndHoldAtTime === "function") { - target.param.cancelAndHoldAtTime(from); - } else { - target.param.cancelScheduledValues(from); - } - } -} - /** One built effect, paired with the chain node id lanes address it by. */ export interface AutomatableNode { id?: string; diff --git a/packages/core/src/audio/audioFxGraph.test.ts b/packages/core/src/audio/audioFxGraph.test.ts index 6339ba5f23..9c54f12bc9 100644 --- a/packages/core/src/audio/audioFxGraph.test.ts +++ b/packages/core/src/audio/audioFxGraph.test.ts @@ -342,6 +342,39 @@ describe("levels and per-channel state", () => { }); }); +describe("gain stage", () => { + /** + * A level control the chain owns, so a carve can make broadband room for a + * voice without touching the track's own volume lane — which belongs to the + * author, and which a carve re-run would otherwise have to overwrite. + */ + it("sets a GainNode from a value in dB", () => { + const c = ctx(); + buildFxNode(c as unknown as BaseAudioContext, "gain", { gain: -6 }); + const node = c.created.find((x) => x.kind === "gain"); + expect(node).toBeTruthy(); + expect(node!.gain.value).toBeCloseTo(0.501, 3); + }); + + it("passes unity through at 0 dB", () => { + const c = ctx(); + buildFxNode(c as unknown as BaseAudioContext, "gain", { gain: 0 }); + expect(c.created.find((x) => x.kind === "gain")!.gain.value).toBeCloseTo(1, 6); + }); + + it("maps an automation lane's dB into the AudioParam's linear units", () => { + // The lane is authored in the knob's own unit — dB, like every other gain in + // the registry — but a GainNode's param is a linear multiplier. Without the + // mapping, a lane point of -12 would schedule a gain of -12: a phase flip + // twelve times too loud, not a cut. + const handle = buildFxNode(ctx() as unknown as BaseAudioContext, "gain", { gain: 0 }); + const target = handle.automation?.gain?.[0]; + expect(target?.map).toBeTypeOf("function"); + expect(target!.map!(-12)).toBeCloseTo(0.251, 3); + expect(target!.map!(0)).toBeCloseTo(1, 6); + }); +}); + describe("automatable parameters", () => { /** * The registry's `automatable` flag is what the panel and the scheduler both diff --git a/packages/core/src/audio/audioFxGraph.ts b/packages/core/src/audio/audioFxGraph.ts index 67cfe04565..a366b823e7 100644 --- a/packages/core/src/audio/audioFxGraph.ts +++ b/packages/core/src/audio/audioFxGraph.ts @@ -111,6 +111,26 @@ function simple( */ const USES_Q: ReadonlySet = new Set(["peaking", "highpass", "lowpass"]); +/** dB on the knob, a linear multiplier on the AudioParam. */ +const dbToLinear = (db: number): number => Math.pow(10, db / 20); + +/** + * A plain level stage. + * + * Every other gain in the registry sits on a BiquadFilterNode, whose `gain` + * param is already in dB. A GainNode's is a linear multiplier, so both the + * initial value and anything an automation lane schedules have to be converted — + * which is what `FxParamTarget.map` is for. + */ +const gainStage: Builder = (ctx, p) => { + const g = ctx.createGain(); + const apply = (v: HfAudioFxParamValues): void => { + g.gain.value = dbToLinear(n(v.gain)); + }; + apply(p); + return simple(g, apply, { gain: [{ param: g.gain, map: dbToLinear }] }); +}; + function biquad(type: BiquadFilterType, useGain: boolean): Builder { return (ctx, p) => { const f = ctx.createBiquadFilter(); @@ -390,6 +410,7 @@ const convolver: Builder = (ctx, p) => { }; const BUILDERS: Record = { + "gain-node": gainStage, "biquad-peaking": biquad("peaking", true), "biquad-lowshelf": biquad("lowshelf", true), "biquad-highshelf": biquad("highshelf", true), diff --git a/packages/core/src/audioAutomation.test.ts b/packages/core/src/audioAutomation.test.ts index 4a59caae8a..7b9af2c747 100644 --- a/packages/core/src/audioAutomation.test.ts +++ b/packages/core/src/audioAutomation.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { applyCurve, + shapeProgress, fxAutomationTarget, isConstantLane, parseAutomation, @@ -10,6 +11,7 @@ import { sampleAutomationCurve, sampleAutomationLane, serializeAutomation, + steadyViaPoint, VOLUME_RANGE, type HfAutomationLane, } from "./audioAutomation.js"; @@ -152,6 +154,54 @@ describe("normalisation", () => { }; expect(parseAutomation(serializeAutomation(source))).toEqual(source); }); + + it("round-trips a via point, and keeps it as a pair", () => { + const source = { + version: 1, + lanes: [ + lane([ + // A via point with no `curve` at all: the bend is entirely described by + // where the segment goes. It has to survive on its own, or a bend + // dragged near a breakpoint silently straightens on the next load. + { t: 0, v: 0.8, viaX: 0.7, viaY: 0.6 }, + { t: 2, v: 0.2, curve: 0.5, viaX: 0.3, viaY: 0.44 }, + { t: 3, v: 0.5 }, + ]), + ], + }; + expect(parseAutomation(serializeAutomation(source))).toEqual(source); + }); + + it("drops a via point that says nothing, and clamps one off the segment", () => { + const parsed = parseAutomation( + JSON.stringify({ + version: 1, + lanes: [ + { + target: "volume", + points: [ + // Half a via point describes no shape; taking one coordinate on its + // own would leave the segment's shape depending on which half + // survived a hand edit. + { t: 0, v: 0.5, viaX: 0.4 }, + // On the diagonal: this IS the straight line, so storing it would + // claim a bend the segment does not have. + { t: 1, v: 0.6, viaX: 0.4, viaY: 0.4 }, + // Outside the segment entirely: pulled back to the steady region + // rather than describing a shape no curve can draw. + { t: 2, v: 0.7, viaX: 5, viaY: -3 }, + { t: 3, v: 1 }, + ], + }, + ], + }), + ); + const points = parsed.lanes[0]!.points; + expect(points[0]).not.toHaveProperty("viaX"); + expect(points[1]).not.toHaveProperty("viaX"); + expect(points[2]!.viaX).toBeCloseTo(0.999, 6); + expect(points[2]!.viaY).toBeCloseTo(0.001, 6); + }); }); describe("resolveAutomation", () => { @@ -230,6 +280,216 @@ describe("sampling", () => { expect(applyCurve(0.5, 0)).toBe(0.5); }); + it("leaves a legacy curve-only point sampling exactly as it always did", () => { + for (const x of [0, 0.1, 0.25, 0.5, 0.75, 0.9, 1]) { + expect(shapeProgress(x, { curve: 0.5 })).toBe(Math.pow(x, Math.pow(2, 1))); + } + expect(shapeProgress(0.5, {})).toBe(0.5); + }); + + it("passes exactly through any via point, however deep or off-centre", () => { + // The contract the drag depends on: the pointer's position IS the shape. No + // depth cap and no position cap — an earlier version held bends to a maximum + // slope, which capped how far the line could be pulled and, worse, could clamp a + // bend onto the straight line and flatten it mid-drag. + for (const [vx, vy] of [ + [0.5, 0.7], + [0.5, 0.95], + [0.2, 0.8], + [0.1, 0.9], + [0.05, 0.95], + [0.9, 0.15], + [0.3, 0.05], + [0.95, 0.55], + ] as const) { + expect(shapeProgress(vx, { viaX: vx, viaY: vy })).toBeCloseTo(vy, 6); + } + }); + + it("puts the curve's furthest point from straight exactly at the via point", () => { + // "The cursor is the apex": not near it, at it. The conic passes through the via + // point at its own midparameter, and its two halves are symmetric in parameter, + // so the deepest departure from the straight line lands there by construction. + for (const [vx, vy] of [ + [0.1, 0.9], + [0.2, 0.8], + [0.5, 0.95], + [0.8, 0.3], + [0.9, 0.15], + [0.95, 0.55], + ] as const) { + let deepest = 0; + let at = 0; + for (let k = 1; k < 1000; k++) { + const x = k / 1000; + const gap = Math.abs(shapeProgress(x, { viaX: vx, viaY: vy }) - x); + if (gap > deepest) { + deepest = gap; + at = x; + } + } + expect(at).toBeCloseTo(vx, 2); + expect(deepest).toBeCloseTo(Math.abs(vy - vx), 2); + } + }); + + it("reaches a bend far deeper than a plain quadratic could", () => { + // A plain quadratic needs its control point at 2Q - M, which leaves the segment + // once the via point is past the middle half — so it cannot pass through a deep + // point at all. The conic's weight is what buys the reach. + expect(shapeProgress(0.1, { viaX: 0.1, viaY: 0.9 })).toBeCloseTo(0.9, 6); + expect(shapeProgress(0.05, { viaX: 0.05, viaY: 0.95 })).toBeCloseTo(0.95, 6); + }); + + it("never flattens a bend that is pulled harder", () => { + // The regression. Holding a bend's depth inside the steady region by clamping + // its coordinates one at a time snapped the curve flat mid-drag: near the ends + // of a segment almost every legal value sits on ONE side of the straight line, + // so a bend pulled the other way got clamped onto the line itself and vanished. + // Pulling further has to keep bending the way the pointer asked, always. + for (const viaX of [0.1, 0.3, 0.5, 0.7, 0.9]) { + for (const viaY of [0.9, 0.6, 0.45, 0.3, 0.05, -0.5, 1.5]) { + if (Math.abs(viaY - viaX) < 0.02) continue; + const via = steadyViaPoint(viaX, viaY); + expect(via).not.toBeNull(); + if (!via) continue; + // Same side of the straight line as the pointer asked for. + expect(Math.sign(via.viaY - via.viaX)).toBe(Math.sign(viaY - viaX)); + // And a bend worth seeing, not a hair off straight. + expect(Math.abs(via.viaY - via.viaX)).toBeGreaterThan(0.01); + } + } + }); + + it("stays a steady curve through the via point, with no corner to see", () => { + // A shape built from two curves meeting AT the via point can only be as smooth + // as that join, and keeping such a join monotone forces its tangent down to a + // fraction of the slope the curve arrives with — which reads as a sharp corner + // exactly where the pointer is. One conic arc has no join: slope is continuous + // by construction, so the ratio across the via point stays near 1 even for the + // extreme bends where a spliced curve kinked hardest. + const slopeAt = (x: number, viaX: number, viaY: number): number => { + const h = 1e-4; + return ( + (shapeProgress(x + h, { viaX, viaY }) - shapeProgress(x - h, { viaX, viaY })) / (2 * h) + ); + }; + for (const [viaX, viaY] of [ + [0.1, 0.9], + [0.9, 0.1], + [0.15, 0.6], + [0.8, 0.95], + [0.5, 0.95], + [0.5, 0.05], + ] as const) { + const before = slopeAt(viaX - 0.02, viaX, viaY); + const after = slopeAt(viaX + 0.02, viaX, viaY); + expect(before).toBeGreaterThan(0); + expect(after).toBeGreaterThan(0); + // Within a factor of 2.2 across a 4% window either side — that much is real + // curvature on a tight bend. Two cubics spliced at the via point measured + // 27.6 on the first of these, and 2.4-3.6 on the gentler ones. + const ratio = before > after ? before / after : after / before; + expect(ratio).toBeLessThan(2.2); + } + }); + + it("never changes slope abruptly anywhere along the segment", () => { + // The same property swept rather than probed at the via point, so a corner + // introduced anywhere else would fail too. + for (const [viaX, viaY] of [ + [0.2, 0.8], + [0.85, 0.35], + [0.5, 0.9], + [0.1, 0.9], + [0.9, 0.08], + ] as const) { + let previous: number | null = null; + for (let k = 2; k < 98; k++) { + const x = k / 100; + const h = 1e-3; + const slope = + (shapeProgress(x + h, { viaX, viaY }) - shapeProgress(x - h, { viaX, viaY })) / (2 * h); + if (previous !== null) { + const ratio = slope > previous ? slope / previous : previous / slope; + // 1% of the segment at a time: a steady curve changes slope gradually. + // The spliced version measured 2.2 here, and 14.5 with the via point + // dragged into a corner. + expect(ratio).toBeLessThan(1.7); + } + previous = slope; + } + } + }); + + it("keeps the segment pinned at both breakpoints", () => { + for (const [vx, vy] of [ + [0.2, 0.7], + [0.85, 0.3], + ] as const) { + expect(shapeProgress(0, { viaX: vx, viaY: vy })).toBeCloseTo(0, 9); + expect(shapeProgress(1, { viaX: vx, viaY: vy })).toBeCloseTo(1, 9); + } + }); + + it("stays monotone for every via point, so a render never sags mid-segment", () => { + // Not cosmetic: a segment is baked into setValueCurveAtTime, and progress + // that dipped backwards is a rising fader audibly dropping. This is the + // property the Fritsch-Carlson tangent limit is there to guarantee, so it is + // swept rather than spot-checked. + for (let ix = 1; ix < 20; ix++) { + for (let iy = 1; iy < 20; iy++) { + const viaX = ix / 20; + const viaY = iy / 20; + let previous = -Infinity; + for (let k = 0; k <= 60; k++) { + const y = shapeProgress(k / 60, { viaX, viaY }); + expect(y).toBeGreaterThanOrEqual(previous - 1e-9); + previous = y; + } + } + } + }); + + it("puts the bend where the via point is, not always on the same side", () => { + // The complaint this replaced: every upward bend a single exponent could draw + // deviated most in the first fifth of the segment. Now the peak deviation + // tracks the via point across the whole span. + const apexOf = (viaX: number, viaY: number): number => { + let best = 0; + let at = 0; + for (let k = 1; k < 100; k++) { + const x = k / 100; + const gap = Math.abs(shapeProgress(x, { viaX, viaY }) - x); + if (gap > best) { + best = gap; + at = x; + } + } + return at; + }; + // Near, not exactly at: the deviation is smooth through the knot, so its peak + // can sit slightly inside. What matters is that it tracks the via point + // across the whole segment instead of parking in the first fifth. + for (const viaX of [0.3, 0.4, 0.5, 0.6, 0.7]) { + expect(Math.abs(apexOf(viaX, viaX + 0.06) - viaX)).toBeLessThan(0.12); + } + expect(apexOf(0.3, 0.36)).toBeLessThan(apexOf(0.7, 0.76)); + }); + + it("bends either way from the same via progress", () => { + const at = (viaY: number) => + sampleAutomationLane( + lane([ + { t: 0, v: 0, viaX: 0.7, viaY }, + { t: 1, v: 1 }, + ]), + 0.7, + ); + expect(at(0.56)).toBeCloseTo(0.56, 6); + expect(at(0.73)).toBeCloseTo(0.73, 6); + }); + it("walks a dense lane by bisection, not by scanning", () => { const points = Array.from({ length: 200 }, (_, i) => ({ t: i, v: i % 2 })); const dense = lane(points); diff --git a/packages/core/src/audioAutomation.ts b/packages/core/src/audioAutomation.ts index f6363f5426..a2cdef1c92 100644 --- a/packages/core/src/audioAutomation.ts +++ b/packages/core/src/audioAutomation.ts @@ -35,6 +35,25 @@ export interface HfAutomationPoint { * straight line; positive holds low then rises late, negative rises early. */ curve?: number; + /** + * An interior point the segment *leaving* this point passes through, in + * normalised segment space: `viaX` is progress 0..1 between the two + * breakpoints, `viaY` is how far the value has travelled by then. Both are + * needed for either to mean anything; without them the segment falls back to + * `curve` above, so everything authored before this existed reads unchanged. + * + * Two numbers rather than one because `curve` answers only "how hard", and an + * exponent cannot say "and where". Every upward bend a single exponent can + * draw has its deepest deviation in the first fifth of the segment, whatever + * the author aimed at, and an exponent steep enough to reach a point near + * either end runs past the ±1 the model accepts — so a bend dragged near the + * right-hand breakpoint bulged on the left and then stopped following the + * pointer at all. Naming the point the curve goes through makes both the + * height and the position of the bend the author's to choose, and takes the + * saturation with it: any interior point is reachable exactly. + */ + viaX?: number; + viaY?: number; } export interface HfAutomationLane { @@ -153,6 +172,67 @@ function clampCurve(v: unknown): number { return Math.min(1, Math.max(-1, n)); } +/** + * A via point, or null when there isn't a usable one. + * + * Only two things disqualify one: a coordinate outside the segment, which names a + * point the curve does not travel through, and a point ON the straight line, which + * is a straight line — storing that would claim a bend the segment does not have. + * + * Depth is deliberately NOT limited. An earlier version held bends to a maximum + * slope so a segment could never leave a breakpoint steeply, and it was wrong twice + * over: it capped how far a curve could be pulled, and clamping the two coordinates + * separately could land a bend exactly on the straight line and flatten it mid-drag. + * A steep departure is a smooth curve doing what it was asked; what must never + * happen is a crease, and that is the sampler's job below, not a limit here. + */ +function clampVia(rawX: unknown, rawY: unknown): { x: number; y: number } | null { + const x = numberOrNull(rawX); + const y = numberOrNull(rawY); + if (x === null || y === null) return null; + const inside = (n: number): number => Math.min(0.999, Math.max(0.001, n)); + const cx = inside(x); + const cy = inside(y); + if (Math.abs(cx - cy) < 1e-6) return null; + return { x: cx, y: cy }; +} + +/** + * The via point a segment will actually be drawn with, given one that was asked + * for — pulled into the steady region, or null when it describes no bend. + * + * Exported so an editor can write what the model will honour rather than what the + * pointer happened to ask for. Without it a lane's attribute claims a shape the + * renderer quietly declines to draw, and the two only agree once the file makes a + * round trip. + */ +export function steadyViaPoint( + viaX: number | undefined, + viaY: number | undefined, +): { viaX: number; viaY: number } | null { + const via = clampVia(viaX, viaY); + return via ? { viaX: via.x, viaY: via.y } : null; +} + +/** + * The shape fields a point carries, cleaned: whichever of `curve` and the via + * pair survive, each omitted when it describes nothing. + * + * The via pair is kept or dropped together. Half a via point says nothing, and + * letting one coordinate through would leave the segment's shape depending on + * which half survived a hand edit. + */ +function shapeFields( + p: HfAutomationPoint | undefined, +): Pick { + const curve = clampCurve(p?.curve); + const via = clampVia(p?.viaX, p?.viaY); + return { + ...(curve ? { curve } : {}), + ...(via ? { viaX: via.x, viaY: via.y } : {}), + }; +} + /** * Clean one point, or reject it. * @@ -167,8 +247,7 @@ function cleanPoint( const v = numberOrNull(p?.v); if (t === null || v === null) return null; const clamped = range ? Math.min(range.max, Math.max(range.min, v)) : v; - const curve = clampCurve(p?.curve); - return { t: Math.max(0, t), v: clamped, ...(curve ? { curve } : {}) }; + return { t: Math.max(0, t), v: clamped, ...shapeFields(p) }; } /** @@ -281,6 +360,7 @@ export function serializeAutomation(automation: HfAutomation): string { t: p.t, v: p.v, ...(p.curve ? { curve: p.curve } : {}), + ...(p.viaX !== undefined && p.viaY !== undefined ? { viaX: p.viaX, viaY: p.viaY } : {}), })), })), }); @@ -298,6 +378,94 @@ export function applyCurve(x: number, curve: number | undefined): number { return Math.pow(x, Math.pow(2, 2 * curve)); } +/** + * The conic that carries a segment through its via point: control point and weight + * of a rational quadratic Bezier from (0,0) to (1,1). + * + * A rational quadratic at its own midparameter is `(P0 + 2wC + P1) / (2 + 2w)`, so + * demanding that equal the via point `Q` fixes the control point for any weight: + * `C = Q + (Q - M) / w`, with `M` the straight midpoint. The curve therefore passes + * exactly through `Q` whatever the weight, and — because it does so at the + * midparameter, with the two halves symmetric in parameter — its furthest departure + * from the straight line is at `Q` too. The pointer is the apex, by construction. + * + * The weight is what buys reach. A plain quadratic (`w = 1`) needs its control point + * at `2Q - M`, which leaves the segment as soon as `Q` is past the middle half — so + * a plain quadratic simply cannot pass through a deep or off-centre point. Raising + * the weight draws the control point back toward `Q`, so take the smallest weight + * that keeps `C` inside the segment: inside is what makes progress monotone, and + * smallest-that-fits is the broadest curve available through that point. + * + * Deep bends therefore come out tight and steep near the breakpoint they lean on. + * That is the shape being asked for, and it stays a single smooth arc while doing + * it — no join, no inflection, no crease. + */ +function viaConic(viaX: number, viaY: number): { cx: number; cy: number; w: number } { + const dx = viaX - 0.5; + const dy = viaY - 0.5; + const edge = 0.999; + // Per axis, `C` stays inside when the weight is at least the pull divided by the + // room left in the pull's own direction. + const needX = dx > 0 ? dx / (edge - viaX) : dx < 0 ? -dx / (viaX - (1 - edge)) : 0; + const needY = dy > 0 ? dy / (edge - viaY) : dy < 0 ? -dy / (viaY - (1 - edge)) : 0; + const w = Math.max(1, needX, needY); + return { cx: viaX + dx / w, cy: viaY + dy / w, w }; +} + +/** + * Progress through a segment that passes through its via point: one conic arc, from + * breakpoint to breakpoint. + * + * One curve, not two joined at the via point, because anything joined there can only + * be as smooth as the join — and keeping such a join monotone forces its tangent + * down to a fraction of the slope the curve arrives with, creasing the segment + * exactly where the pointer is. A conic has no join and no inflection: its slope + * moves one way from end to end, which is what makes it read as one curve rather + * than a shape with features in it. + */ +function shapeVia(x: number, viaX: number, viaY: number): number { + const { cx, cy, w } = viaConic(viaX, viaY); + // Both coordinates are quadratics over one shared denominator, so `x(t) = x` + // rearranges into an ordinary quadratic in `t`: closed form, and the same answer on + // every machine, which a render that has to match its own preview depends on. + const spread = 2 - 2 * w; + const a = x * spread - 1 + 2 * w * cx; + const b = -x * spread - 2 * w * cx; + const t = conicParam(a, b, x); + const rest = 1 - t; + const denominator = rest * rest + 2 * w * t * rest + t * t; + if (denominator <= 0) return x; + return (2 * w * cy * t * rest + t * t) / denominator; +} + +/** The root of `a*t^2 + b*t + c` that lies on the arc the segment travels. */ +function conicParam(a: number, b: number, c: number): number { + if (Math.abs(a) < 1e-12) return Math.abs(b) < 1e-12 ? c : -c / b; + const root = Math.sqrt(Math.max(0, b * b - 4 * a * c)); + const first = (-b + root) / (2 * a); + const second = (-b - root) / (2 * a); + const onArc = (t: number): boolean => t >= -1e-9 && t <= 1 + 1e-9; + if (onArc(first)) return Math.min(1, Math.max(0, first)); + if (onArc(second)) return Math.min(1, Math.max(0, second)); + return c; +} + +/** + * The progress a segment leaving `point` has reached at normalised `x`. + * + * A via point wins when it is there, since it says strictly more than an + * exponent can. Everything authored before via points existed carries only + * `curve` and takes the exponent path unchanged. + */ +export function shapeProgress( + x: number, + point: { curve?: number | undefined; viaX?: number | undefined; viaY?: number | undefined }, +): number { + const via = clampVia(point.viaX, point.viaY); + if (via) return shapeVia(Math.min(1, Math.max(0, x)), via.x, via.y); + return applyCurve(x, point.curve); +} + function lerpValue(a: number, b: number, x: number, scale: "linear" | "log"): number { // A frequency sweep interpolated linearly spends almost all its time in the // top octave. Log-scaled parameters interpolate in log space so a 200 Hz to @@ -338,7 +506,7 @@ export function sampleAutomationLane( const b = pts[hi]!; const span = b.t - a.t; if (span <= 0) return b.v; - return lerpValue(a.v, b.v, applyCurve((t - a.t) / span, a.curve), scale); + return lerpValue(a.v, b.v, shapeProgress((t - a.t) / span, a), scale); } /** True when the lane is a single value, i.e. worth setting once and not scheduling. */ diff --git a/packages/core/src/audioCarve.test.ts b/packages/core/src/audioCarve.test.ts index 5778a1731d..519afc875e 100644 --- a/packages/core/src/audioCarve.test.ts +++ b/packages/core/src/audioCarve.test.ts @@ -1,11 +1,21 @@ import { describe, expect, it } from "vitest"; +import { MAX_AUTOMATION_POINTS, sampleAutomationLane } from "./audioAutomation.js"; import { analyseCarveBands, + analyseCarveDuck, + analyseCarveDynamics, carveBandsToChain, + carveProfile, + classifyAudioName, + mixCarveSources, + couldBeCarveSource, DEFAULT_CARVE, normalizeCarveSettings, } from "./audioCarve.js"; +/** The profile the panel's default strength produces — what the separate controls defaulted to. */ +const PROFILE = carveProfile(DEFAULT_CARVE.strength); + const SR = 48000; /** A tone-plus-harmonics stand-in for a voice, centred on `f0`. */ @@ -25,28 +35,99 @@ function voiceLike(f0: number, seconds = 0.5): Float32Array { describe("normalizeCarveSettings", () => { it("fills defaults and clamps nonsense", () => { expect(normalizeCarveSettings(undefined)).toEqual(DEFAULT_CARVE); - const v = normalizeCarveSettings({ maxCutDb: 500, bands: 99, q: 0, intelligibilityBias: -3 }); - expect(v.maxCutDb).toBe(24); - expect(v.bands).toBe(6); - expect(v.q).toBe(0.3); - expect(v.intelligibilityBias).toBe(0); + expect(normalizeCarveSettings({ strength: 40 }).strength).toBe(1); + expect(normalizeCarveSettings({ strength: -3 }).strength).toBe(0); + expect(normalizeCarveSettings({ strength: Number.NaN }).strength).toBe(DEFAULT_CARVE.strength); + }); + + it("reads a carve written before the controls collapsed into one", () => { + // Depth is the number that said most about intent, so it maps back onto the + // strength that reproduces it rather than being dropped. + expect(normalizeCarveSettings({ maxCutDb: 6, bands: 3, q: 1.4 }).strength).toBe(0.25); + expect(carveProfile(normalizeCarveSettings({ maxCutDb: 6 }).strength).maxCutDb).toBe(6); + expect(normalizeCarveSettings({ maxCutDb: 10 }).strength).toBe(0.5); + expect(normalizeCarveSettings({ maxCutDb: 24 }).strength).toBe(1); + }); + + it("keeps an explicit strength over a legacy depth", () => { + expect(normalizeCarveSettings({ strength: 0.25, maxCutDb: 10 }).strength).toBe(0.25); + }); +}); + +describe("carveProfile", () => { + it("lands on the old defaults at the strength the panel starts at", () => { + // The separate controls shipped 6 dB, 3 bands, Q 1.4, bias 0.7, and the duck + // pair defaulted to 6/9 — a project carved before the collapse must sound the + // same after it. That point is a quarter of the way up, since the knob's range + // was later doubled above it. + expect(DEFAULT_CARVE.strength).toBe(0.25); + expect(PROFILE).toEqual({ + maxCutDb: 6, + bands: 3, + q: 1.4, + intelligibilityBias: 0.7, + duckDb: 6, + headroomDb: 9, + }); + }); + + it("moves every number the same direction, monotonically", () => { + const steps = [0, 0.25, 0.5, 0.75, 1].map(carveProfile); + for (let i = 1; i < steps.length; i++) { + const a = steps[i - 1]!; + const b = steps[i]!; + expect(b.maxCutDb).toBeGreaterThan(a.maxCutDb); + expect(b.bands).toBeGreaterThanOrEqual(a.bands); + expect(b.q).toBeGreaterThan(a.q); + expect(b.duckDb).toBeGreaterThan(a.duckDb); + expect(b.headroomDb).toBeGreaterThan(a.headroomDb); + } + }); + + it("carves frequencies only at zero, and goes hard at the top", () => { + expect(carveProfile(0).duckDb).toBe(0); + expect(carveProfile(0).bands).toBe(1); + expect(carveProfile(1).maxCutDb).toBe(18); + expect(carveProfile(1).bands).toBe(6); + expect(carveProfile(1).duckDb).toBe(24); + // The bias is a 0..1 weight, so the top of the knob is all the way over + // rather than past it. + expect(carveProfile(1).intelligibilityBias).toBe(1); + }); + + it("puts the old top of the knob at the halfway mark", () => { + // The range was doubled because full strength could not sit a bed under a + // loud voice. Everything the knob used to reach is now the bottom half. + expect(carveProfile(0.5)).toEqual({ + maxCutDb: 10, + bands: 4, + q: 1.7, + intelligibilityBias: 0.8, + duckDb: 12, + headroomDb: 12, + }); + }); + + it("clamps a strength outside the knob's own range", () => { + expect(carveProfile(9)).toEqual(carveProfile(1)); + expect(carveProfile(-9)).toEqual(carveProfile(0)); }); }); describe("analyseCarveBands", () => { it("returns nothing for silence-length input", () => { - expect(analyseCarveBands(new Float32Array(0), SR, DEFAULT_CARVE)).toEqual([]); + expect(analyseCarveBands(new Float32Array(0), SR, PROFILE)).toEqual([]); }); it("returns the requested number of bands, ascending, all cuts", () => { - const bands = analyseCarveBands(voiceLike(250), SR, { ...DEFAULT_CARVE, bands: 3 }); + const bands = analyseCarveBands(voiceLike(250), SR, { ...PROFILE, bands: 3 }); expect(bands).toHaveLength(3); expect(bands.map((b) => b.freq)).toEqual([...bands.map((b) => b.freq)].sort((a, b) => a - b)); for (const b of bands) expect(b.gainDb).toBeLessThan(0); }); it("never cuts deeper than the configured maximum", () => { - const bands = analyseCarveBands(voiceLike(400), SR, { ...DEFAULT_CARVE, maxCutDb: 5 }); + const bands = analyseCarveBands(voiceLike(400), SR, { ...PROFILE, maxCutDb: 5 }); for (const b of bands) expect(Math.abs(b.gainDb)).toBeLessThanOrEqual(5); }); @@ -121,7 +202,7 @@ describe("analyseCarveBands", () => { // not where the voice is loudest. DEFAULT_CARVE is what a user gets from // clicking "Analyse and apply", so the default has to do this — a knob that // only works at its extreme does not work. - const bands = analyseCarveBands(tiltedVoice(), SR, { ...DEFAULT_CARVE, bands: 3 }); + const bands = analyseCarveBands(tiltedVoice(), SR, { ...PROFILE, bands: 3 }); expect(bands.map((b) => b.freq).some((f) => f >= 1000)).toBe(true); }); @@ -151,7 +232,7 @@ describe("analyseCarveBands", () => { } return out; })(); - const one = { ...DEFAULT_CARVE, bands: 1 }; + const one = { ...PROFILE, bands: 1 }; expect(analyseCarveBands(twoTone, SR, { ...one, intelligibilityBias: 0 })[0]!.freq).toBe(250); expect(analyseCarveBands(twoTone, SR, one)[0]!.freq).toBe(1600); }); @@ -180,3 +261,299 @@ describe("carveBandsToChain", () => { expect(carveBandsToChain([]).nodes).toEqual([]); }); }); + +describe("analyseCarveDuck", () => { + const SR8 = 8000; + + /** Steady tone at a given amplitude — stands in for a bed or a voice. */ + function tone(amp: number, seconds: number, f = 300, sr = SR8): Float32Array { + const out = new Float32Array(Math.floor(sr * seconds)); + for (let i = 0; i < out.length; i++) out[i] = amp * Math.sin(2 * Math.PI * f * (i / sr)); + return out; + } + + /** Silence, then speech-level tone, then silence. */ + function voiceWithPause(amp: number, seconds: number, from: number, to: number): Float32Array { + const out = new Float32Array(Math.floor(SR8 * seconds)); + for (let i = 0; i < out.length; i++) { + const t = i / SR8; + out[i] = t >= from && t < to ? amp * Math.sin(2 * Math.PI * 700 * t) : 0; + } + return out; + } + + const at = (points: { t: number; v: number }[], t: number): number => + sampleAutomationLane({ target: "fx.n1.gain", points }, t); + + const settings = (over: Partial = {}) => ({ + ...PROFILE, + duckDb: 12, + headroomDb: 9, + ...over, + }); + + it("leaves the bed alone where the voice is silent", () => { + // The whole point of a follower: no voice, nothing to make room for. Before + // the first word it is exactly flat; afterwards it eases back rather than + // snapping, so the tail is checked by the release cases below. + const duck = analyseCarveDuck(voiceWithPause(0.5, 16, 3, 6), tone(0.5, 16), SR8, settings(), 0); + expect(at(duck, 1)).toBe(0); + // A window is half a second wide and its value is stamped at its centre, so + // the duck begins up to half a window before the first word — checked clear + // of that rather than right against it. + expect(at(duck, 2)).toBe(0); + // Back to inaudible by the end of a long tail rather than held down for the + // clip; the envelope's last point is pinned at exactly no cut. + expect(at(duck, 15)).toBeCloseTo(0, 1); + expect(duck.at(-1)?.v).toBe(0); + }); + + it("ducks a loud bed under a quiet voice", () => { + // Bed 12 dB above the voice with 9 dB of headroom asked for: the bed has to + // come down by about 21 dB, which the cap holds at 12. + const duck = analyseCarveDuck( + voiceWithPause(0.12, 8, 2, 7), + tone(0.5, 8), + SR8, + settings({ duckDb: 24 }), + 0, + ); + expect(at(duck, 4.5)).toBeLessThan(-15); + }); + + it("barely touches a bed already sitting under the voice", () => { + const duck = analyseCarveDuck(voiceWithPause(0.8, 8, 2, 7), tone(0.05, 8), SR8, settings(), 0); + expect(at(duck, 4.5)).toBeGreaterThan(-1); + }); + + it("lets the bed come back slowly after the voice stops", () => { + // A bed that jumps back to full the instant a word ends is heard as the + // effect switching off. It has to ease back over seconds — and the envelope's + // own step is ~0.3s on a long voice, so a release near that lands the whole + // recovery inside one step, which is exactly the flip this pins against. + const voice = voiceWithPause(0.5, 14, 2, 6); + const duck = analyseCarveDuck(voice, tone(0.5, 14), SR8, settings({ duckDb: 12 }), 0); + const atSpeech = at(duck, 5.5); + expect(atSpeech).toBeLessThan(-6); + + // Still most of the way down a third of a second after the last word... + expect(at(duck, 6.3)).toBeLessThan(atSpeech * 0.6); + // ...noticeably recovered by a second and a half... + expect(at(duck, 7.5)).toBeGreaterThan(atSpeech * 0.5); + // ...and within a hair of flat a few seconds later, rather than held down. + expect(at(duck, 11)).toBeGreaterThan(-0.6); + }); + + it("finishes the release past the last word instead of snapping at the end", () => { + // A lane's last point has to be no cut, or the bed stays dipped for the rest + // of the clip. Jumping there from full duck is the same flip the slow release + // exists to prevent, just moved to the end of the narration — so the envelope + // keeps decaying past the voice and pins once it is inaudible. + const voice = voiceWithPause(0.5, 8, 1, 7.9); + const duck = analyseCarveDuck(voice, tone(0.5, 12), SR8, settings({ duckDb: 12 }), 0); + const steps = duck.slice(1).map((p, i) => Math.abs(p.v - duck[i]!.v)); + expect(Math.max(...steps.slice(-4))).toBeLessThan(2); + // Still ducked a moment after the voice stops, and flat well after. + expect(at(duck, 8.2)).toBeLessThan(-2); + expect(duck.at(-1)?.v).toBe(0); + expect(duck.at(-1)!.t).toBeGreaterThan(8); + }); + + it("comes back up more slowly than it ducks", () => { + // Asymmetric on purpose: the cut has to be there by the time a word is + // audible, and gone slowly enough that the bed does not pump between + // sentences. + const voice = voiceWithPause(0.5, 14, 4, 8); + const duck = analyseCarveDuck(voice, tone(0.5, 14), SR8, settings({ duckDb: 12 }), 0); + const floor = Math.min(...duck.map((p) => p.v)); + const toHalfDown = duck.find((p) => p.v <= floor * 0.5)?.t ?? Infinity; + const backToHalfUp = + [...duck].reverse().find((p) => p.v <= floor * 0.5)?.t ?? Number.NEGATIVE_INFINITY; + const attackSpan = toHalfDown - 4; + const releaseSpan = backToHalfUp - 8; + expect(releaseSpan).toBeGreaterThan(attackSpan * 2); + }); + + it("never exceeds the configured maximum", () => { + const duck = analyseCarveDuck( + voiceWithPause(0.02, 8, 2, 7), + tone(0.9, 8), + SR8, + settings({ duckDb: 6 }), + 0, + ); + expect(Math.min(...duck.map((p) => p.v))).toBeGreaterThanOrEqual(-6); + }); + + it("reads the bed at the offset between the two clips", () => { + // The bed starts before the voice, so voice-local 0 is bed-local `offset`. + // Given a bed that is loud only in its first half, a voice that starts 4s + // into it must see the quiet half and duck less than the same voice at 0. + const bed = new Float32Array(Math.floor(SR8 * 8)); + for (let i = 0; i < bed.length; i++) { + const t = i / SR8; + bed[i] = (t < 4 ? 0.9 : 0.02) * Math.sin(2 * Math.PI * 300 * t); + } + const voice = voiceWithPause(0.3, 4, 0.5, 3.5); + const aligned = analyseCarveDuck(voice, bed, SR8, settings({ duckDb: 24 }), 0); + const shifted = analyseCarveDuck(voice, bed, SR8, settings({ duckDb: 24 }), 4); + expect(at(aligned, 2)).toBeLessThan(at(shifted, 2)); + }); + + it("returns nothing without a voice, and stays inside the point budget", () => { + expect(analyseCarveDuck(new Float32Array(0), tone(0.5, 2), SR8, settings(), 0)).toEqual([]); + const long = analyseCarveDuck( + voiceWithPause(0.4, 200, 5, 195), + tone(0.5, 200), + SR8, + settings(), + 0, + ); + expect(long.length).toBeLessThanOrEqual(MAX_AUTOMATION_POINTS); + }); +}); + +describe("analyseCarveDynamics", () => { + const BAND = { freq: 1000, gainDb: -8, q: 1.4 }; + + /** Silence, then a 1 kHz burst, then silence again. */ + function burst(sampleRate: number, seconds: number, from: number, to: number): Float32Array { + const out = new Float32Array(Math.floor(sampleRate * seconds)); + for (let i = 0; i < out.length; i++) { + const t = i / sampleRate; + out[i] = t >= from && t < to ? 0.7 * Math.sin(2 * Math.PI * 1000 * t) : 0; + } + return out; + } + + const at = (points: { t: number; v: number }[], t: number): number => + sampleAutomationLane({ target: "fx.n1.gain", points }, t); + + it("holds the filter flat where the voice is silent and cuts where it speaks", () => { + const [lane] = analyseCarveDynamics(burst(SR, 6, 2, 4), SR, [BAND]); + expect(lane!.freq).toBe(1000); + expect(at(lane!.points, 0.7)).toBeCloseTo(0, 1); + expect(at(lane!.points, 3)).toBeLessThan(-6); + expect(at(lane!.points, 5.3)).toBeCloseTo(0, 1); + }); + + it("pins both ends of the envelope to no cut", () => { + const [lane] = analyseCarveDynamics(burst(SR, 6, 2, 4), SR, [BAND]); + const points = lane!.points; + expect(points[0]).toMatchObject({ t: 0, v: 0 }); + expect(points.at(-1)!.v).toBe(0); + // Held past the end, so the bed comes back up rather than staying dipped. + expect(points.at(-1)!.t).toBeGreaterThanOrEqual(5.9); + }); + + it("never exceeds the point budget a lane can hold", () => { + const long = burst(8000, 200, 10, 190); + const [lane] = analyseCarveDynamics(long, 8000, [BAND]); + expect(lane!.points.length).toBeLessThanOrEqual(MAX_AUTOMATION_POINTS); + expect(lane!.points.length).toBeGreaterThan(2); + }); + + it("returns one envelope per band, and nothing for no input", () => { + const sig = burst(SR, 3, 0.5, 2.5); + expect(analyseCarveDynamics(sig, SR, [BAND, { freq: 2500, gainDb: -4, q: 1.4 }])).toHaveLength( + 2, + ); + expect(analyseCarveDynamics(new Float32Array(0), SR, [BAND])).toEqual([]); + }); +}); + +describe("classifyAudioName", () => { + it("reads a track's kind from its id and its filename together", () => { + // Either can be the informative one: elements named a1/a2 may still have + // narration.mp3 and bgm.mp3 behind them. + expect(classifyAudioName("narration")).toBe("voice"); + expect(classifyAudioName("a1", "voiceover-take3.wav")).toBe("voice"); + expect(classifyAudioName("music-bed")).toBe("music"); + expect(classifyAudioName("a2", "bgm_loop.m4a")).toBe("music"); + expect(classifyAudioName("sfx-explosion")).toBe("sfx"); + expect(classifyAudioName("whoosh-01")).toBe("sfx"); + }); + + it("says nothing about a name that says nothing", () => { + // The common case, and the reason nothing downstream may treat "unknown" as + // "not a voice": it would hide the one track somebody needs to pick. + expect(classifyAudioName("a1")).toBe("unknown"); + expect(classifyAudioName("clip-2", "0f9c1a.mp3")).toBe("unknown"); + expect(classifyAudioName(undefined, null)).toBe("unknown"); + }); + + it("prefers voice when a name carries both hints", () => { + // A file called voiceover-over-music-bed.wav is the voiceover, and a track + // matching both is better offered than hidden. + expect(classifyAudioName("voiceover-over-music-bed.wav")).toBe("voice"); + }); + + it("offers speech and unnamed tracks as carve sources, never music or effects", () => { + expect(couldBeCarveSource("recap-audio")).toBe(true); + expect(couldBeCarveSource("a1")).toBe(true); + expect(couldBeCarveSource("music-bed")).toBe(false); + expect(couldBeCarveSource("sfx-explosion")).toBe(false); + }); +}); + +describe("mixCarveSources", () => { + const tone = (seconds: number, level: number, sampleRate = 48000) => + new Float32Array(Math.round(seconds * sampleRate)).fill(level); + + it("places every voice where it starts on the bed's clock", () => { + // Three people talking at different times is still one question — where and + // when is speech masking this bed — so they become one signal. + const mixed = mixCarveSources( + [ + { samples: tone(1, 0.5), offsetSeconds: 1 }, + { samples: tone(1, 0.25), offsetSeconds: 3 }, + ], + 48000, + ); + expect(mixed.length).toBe(4 * 48000); + const at = (t: number) => mixed[Math.round(t * 48000)]; + expect(at(0.5)).toBe(0); // before anyone speaks + expect(at(1.5)).toBeCloseTo(0.5, 5); + expect(at(2.5)).toBe(0); // the gap between them + expect(at(3.5)).toBeCloseTo(0.25, 5); + }); + + it("sums voices that overlap, because two at once mask more than one", () => { + const mixed = mixCarveSources( + [ + { samples: tone(1, 0.3), offsetSeconds: 0 }, + { samples: tone(1, 0.3), offsetSeconds: 0 }, + ], + 48000, + ); + expect(mixed[0]).toBeCloseTo(0.6, 5); + }); + + it("drops the part of a voice that plays before the bed starts", () => { + // It masks nothing there, and folding it in at zero would put a cut where + // there is no voice. + const mixed = mixCarveSources([{ samples: tone(1, 0.5), offsetSeconds: -0.5 }], 48000); + expect(mixed.length).toBe(0.5 * 48000); + expect(mixed[0]).toBeCloseTo(0.5, 5); + }); + + it("has nothing to mix when there are no voices", () => { + expect(mixCarveSources([], 48000)).toHaveLength(0); + }); +}); + +describe("carve settings written before this took a list of voices", () => { + it("reads a single `source` as a one-voice list, and forgets `dynamic`", () => { + // Every carve is dynamic now: a static one thinned the bed through every pause, + // and nobody wanted that once they had heard both. + const read = normalizeCarveSettings({ source: "vo", strength: 0.4, dynamic: false } as never); + expect(read.sources).toEqual(["vo"]); + expect(read.strength).toBe(0.4); + expect("dynamic" in read).toBe(false); + }); + + it("drops empty ids rather than carrying a source that names nothing", () => { + expect(normalizeCarveSettings({ sources: ["", "vo", ""] } as never).sources).toEqual(["vo"]); + expect(normalizeCarveSettings({ source: "" } as never).sources).toEqual([]); + }); +}); diff --git a/packages/core/src/audioCarve.ts b/packages/core/src/audioCarve.ts index 386aadf64b..1a0fac18a0 100644 --- a/packages/core/src/audioCarve.ts +++ b/packages/core/src/audioCarve.ts @@ -32,47 +32,267 @@ export interface HfCarveBand { q: number; } +/** + * What the author sets: which voice to listen to, how hard to work, and whether + * the work follows the voice moment to moment. + * + * One number for the strength of the effect, not six for its mechanism. The + * mechanism has six numbers — how deep to cut, how many bands, how wide, how far + * to favour intelligibility over raw energy, how far the level may drop, how far + * under the voice to aim — and every one of them was a control nobody could set + * without knowing what the analysis does with it. They move together anyway: a + * gentle carve is a shallow cut in few bands with little ducking, a hard one is + * deeper in more bands with more. `carveProfile` is that relationship, written + * once. + */ export interface HfCarveSettings { - /** Element id of the voice track to analyse. */ - source: string; + /** + * Element ids of every voice track this bed makes room for. + * + * More than one because a bed usually runs under a whole sequence: a narrator, an + * interview answer, a second presenter. Each occupies its own stretch of the bed, + * and carving against only one of them leaves the others fighting it. They are + * analysed together — see `mixCarveSources` — so the cuts follow whoever is + * speaking rather than averaging strangers. + */ + sources: string[]; + /** How hard to carve, 0..1. */ + strength: number; + /** + * Whether the carve is applied at all. + * + * A bed under a voice wants carving, so a track that has never been configured + * is treated as on and carved without being asked. That default needs an off + * switch that survives: with "off" represented by having no settings at all, + * selecting the clip again would read it as never-configured and re-apply. So + * switching it off writes `enabled: false` and the default stops applying. + */ + enabled: boolean; +} + +/** The numbers the analysis actually works in, all derived from `strength`. */ +export interface HfCarveProfile { /** Deepest cut applied to the strongest band. */ maxCutDb: number; /** How many bands to dip. */ bands: number; q: number; /** - * Weight selection toward intelligibility rather than raw voice energy. + * Weight band selection toward intelligibility rather than raw voice energy. * * Ranking purely by voice power lands on the fundamental almost every time, - * because that is where a voice is loudest — but masking that actually hurts - * a voiceover happens higher up, and dipping 160 Hz mostly just thins the - * bed. Weighting pushes selection toward 1-3 kHz where intelligibility lives. + * because that is where a voice is loudest — but masking that actually hurts a + * voiceover happens higher up, and dipping 160 Hz mostly just thins the bed. */ intelligibilityBias: number; + /** How far the bed's whole level may come down to make room, in dB. */ + duckDb: number; + /** How far under the voice the bed should sit while the voice speaks, in dB. */ + headroomDb: number; +} + +/** + * What a track's name suggests it holds. + * + * Only ever a hint — a name is what the author called something, not what is in the + * file — so this is used to order and to filter a list of candidates, never to + * decide alone. `unknown` is deliberately common: a track called `a1` could be + * anything, and treating an unrecognised name as "not a voice" would hide the one + * track somebody needs to pick. + */ +export type HfAudioNameKind = "voice" | "music" | "sfx" | "unknown"; + +/** Short, deliberately dull effects. Nothing here is ever a voiceover. */ +const SFX_NAME = + /sfx|foley|whoosh|impact|riser|stinger|swoosh|thud|boom|click|ding|beep|ambien|room[-_ ]?tone/i; +/** A bed, which is the thing being carved rather than the thing carving it. */ +const MUSIC_NAME = /music|bgm|\bbed\b|soundtrack|score|\bsong\b|theme|instrumental|track\d/i; +/** Speech. */ +const VOICE_NAME = + /voice|\bvo\b|\bvox\b|narrat|speech|dialog|monolog|announce|\btts\b|talk|interview|podcast|recap|script/i; + +/** + * Classify a track from its id and filename together. + * + * Both, because either can be the informative one: an author naming elements `a1` + * and `a2` may still have `narration.mp3` and `bgm.mp3` as their sources, and one + * naming them `voice` and `music` may have opaque hashes for filenames. + * + * Voice is tested first: a file called `voiceover-music-bed.wav` is more likely the + * voiceover than the bed, and a track matching both hints is better offered than + * hidden. + */ +export function classifyAudioName( + ...parts: readonly (string | null | undefined)[] +): HfAudioNameKind { + const text = parts.filter(Boolean).join(" "); + if (VOICE_NAME.test(text)) return "voice"; + if (SFX_NAME.test(text)) return "sfx"; + if (MUSIC_NAME.test(text)) return "music"; + return "unknown"; +} + +/** A clip's place on the timeline. A duration that is not a number is unbounded. */ +export interface HfClipSpan { + start: number; + duration?: number | null; +} + +/** + * Do these two clips share any time at all? + * + * A voice that never plays while the bed does cannot mask it, so it has no business + * in the carve: it would contribute silence to the analysis and, worse, invite the + * author to wonder why including it changed nothing. + * + * An unknown duration counts as unbounded rather than as zero. Refusing a track + * because its length is not written down would drop the commonest case there is — a + * clip whose duration the composition leaves to the media itself. + */ +export function clipsOverlap(a: HfClipSpan, b: HfClipSpan): boolean { + const end = (clip: HfClipSpan): number => + typeof clip.duration === "number" && Number.isFinite(clip.duration) + ? clip.start + clip.duration + : Number.POSITIVE_INFINITY; + return a.start < end(b) && b.start < end(a); +} + +/** + * Could this track be the voice a carve listens to? + * + * Music and SFX are out: a bed is the thing being carved, and a 200 ms whoosh has + * no speech to make room for. Everything else stays in, including names that say + * nothing — see `HfAudioNameKind`. + */ +export function couldBeCarveSource(...parts: readonly (string | null | undefined)[]): boolean { + const kind = classifyAudioName(...parts); + return kind === "voice" || kind === "unknown"; } export const DEFAULT_CARVE: HfCarveSettings = { - source: "", - maxCutDb: 6, - bands: 3, - q: 1.4, - intelligibilityBias: 0.7, + enabled: true, + sources: [], + // A quarter, because the knob's range was doubled and this is the point on the + // new scale that produces what the panel has always defaulted to. Switching + // carve on sounds the same as it did; the extra range is above, not under. + strength: 0.25, }; -export function normalizeCarveSettings(raw: Partial | undefined): HfCarveSettings { - const clamp = (v: unknown, lo: number, hi: number, dflt: number): number => { +/** + * Strength as the six numbers the analysis needs. + * + * Each is a straight line from "barely there" to as far as the effect goes. The + * slopes are twice what they first were: the top of the knob was not strong + * enough to sit a bed under a loud voice, so what used to be full strength is now + * the halfway point and everything above it is new range. Quarter strength is + * therefore where the separate controls' own defaults land, which is what the + * panel defaults to. + */ +export function carveProfile(strength: number): HfCarveProfile { + const s = Number.isFinite(strength) ? Math.min(1, Math.max(0, strength)) : 0.5; + return { + // 2 dB is audible-but-subtle. Past about 10 the hole starts being heard as + // the effect rather than as room for the voice — which is a price worth + // paying at the top of the knob, and why the range now runs past it. + maxCutDb: Number((2 + s * 16).toFixed(2)), + // One band carves the single worst collision; six is every candidate band + // speech meaningfully occupies. + bands: Math.max(1, Math.min(6, Math.round(1 + s * 6))), + // Wider at low strength so a gentle carve is a tilt rather than a notch. + q: Number((1.1 + s * 1.2).toFixed(2)), + // Always weighted toward where masking hurts; at the top, entirely. + intelligibilityBias: Number(Math.min(1, 0.6 + s * 0.4).toFixed(2)), + // No level ducking at all at zero: the carve is then purely spectral. + duckDb: Number((s * 24).toFixed(2)), + // How far under the voice to aim. Deeper targets at higher strength. + headroomDb: Number((6 + s * 12).toFixed(2)), + }; +} + +/** + * Read carve settings from an attribute. + * + * Projects written before the collapse to one knob carry the six mechanism + * numbers instead; their depth is the one that says most about intent, so it maps + * back onto strength rather than being dropped. Everything else about such a + * carve is re-derived, which is the point of having one control. + */ +export function normalizeCarveSettings( + raw: Partial | undefined, +): HfCarveSettings { + // `source` and `dynamic` are gone from the type but still out there in files. + const legacy = raw as (Partial & { source?: unknown }) | undefined; + const num = (v: unknown): number | null => { const n = typeof v === "number" ? v : Number(v); - return Number.isFinite(n) ? Math.min(hi, Math.max(lo, n)) : dflt; + return Number.isFinite(n) ? n : null; }; + // No attribute at all is not a carve to read, it is the absence of one — so the + // defaults apply whole, dynamic included. Only a stored object gets the reading + // below, where a missing `dynamic` means the static carve it was written as. + if (raw === undefined || raw === null) return { ...DEFAULT_CARVE }; + const strength = num(raw?.strength); + const legacyDepth = num(raw?.maxCutDb); + const resolved = + strength !== null + ? strength + : legacyDepth !== null + ? // The inverse of the depth line in `carveProfile`, so the 6 dB default + // reads back as the strength that produces 6 dB. + (legacyDepth - 2) / 16 + : DEFAULT_CARVE.strength; + // A carve written before this took a list names its one voice in `source`. + const stored = Array.isArray(raw?.sources) + ? raw.sources + : typeof legacy?.source === "string" + ? [legacy.source] + : []; return { - source: typeof raw?.source === "string" ? raw.source : "", - maxCutDb: clamp(raw?.maxCutDb, 0, 24, DEFAULT_CARVE.maxCutDb), - bands: Math.round(clamp(raw?.bands, 1, 6, DEFAULT_CARVE.bands)), - q: clamp(raw?.q, 0.3, 8, DEFAULT_CARVE.q), - intelligibilityBias: clamp(raw?.intelligibilityBias, 0, 1, DEFAULT_CARVE.intelligibilityBias), + // Absent means on: every carve written before the flag existed was applied. + enabled: raw?.enabled !== false, + sources: stored.filter((id): id is string => typeof id === "string" && id !== ""), + strength: Math.min(1, Math.max(0, resolved)), }; } +/** + * Every voice as one signal on the BED's clock. + * + * The analysis asks one question — where and when is speech masking this bed — and + * that question has one answer even when three people are talking at different + * times. Summing them onto the bed's timeline first means the existing analysis + * needs no notion of "which voice": bands come out of all the speech there is, and + * the envelopes rise wherever any of it is happening. + * + * `offsetSeconds` is where each voice starts relative to the bed. Audio before the + * bed begins is dropped rather than folded in at zero: it plays over nothing and + * cannot mask anything, and shifting it would put a cut where there is no voice. + * + * Summed, not averaged. Two people speaking at once mask more than either alone, + * which is exactly what the carve should answer to. + */ +export function mixCarveSources( + parts: readonly { samples: Float32Array; offsetSeconds: number }[], + sampleRate: number, +): Float32Array { + const placed = parts.map((part) => ({ + samples: part.samples, + at: Math.round(part.offsetSeconds * sampleRate), + })); + const length = placed.reduce((max, p) => Math.max(max, p.at + p.samples.length), 0); + if (length <= 0) return new Float32Array(0); + const mixed = new Float32Array(length); + for (const { samples, at } of placed) { + // A voice starting before the bed contributes only the part that overlaps it. + const from = at < 0 ? -at : 0; + for (let i = from; i < samples.length; i += 1) { + const target = at + i; + if (target < 0 || target >= length) continue; + mixed[target] = (mixed[target] ?? 0) + (samples[i] ?? 0); + } + } + return mixed; +} + /** Averaged power spectrum, Welch-style. */ function powerSpectrum( mono: Float32Array, @@ -206,7 +426,7 @@ function intelligibilityPenaltyDb(center: number, bias: number): number { export function analyseCarveBands( voice: Float32Array, sampleRate: number, - settings: HfCarveSettings, + profile: HfCarveProfile, ): HfCarveBand[] { if (voice.length === 0) return []; const { freqs, power } = powerSpectrum(voice, sampleRate); @@ -222,12 +442,12 @@ export function analyseCarveBands( scoreDb: bandPowerAt > 0 ? 10 * Math.log10(bandPowerAt) - - intelligibilityPenaltyDb(center, settings.intelligibilityBias) + intelligibilityPenaltyDb(center, profile.intelligibilityBias) : Number.NEGATIVE_INFINITY, }; }).sort((a, b) => b.scoreDb - a.scoreDb); - const selected = scored.slice(0, Math.max(1, settings.bands)).filter((b) => b.hasEnergy); + const selected = scored.slice(0, Math.max(1, profile.bands)).filter((b) => b.hasEnergy); if (selected.length === 0) return []; const topDb = selected[0]!.scoreDb; @@ -237,14 +457,261 @@ export function analyseCarveBands( // under the strongest gets half its cut — read off the dB difference. const relative = Math.pow(10, (scoreDb - topDb) / 10); const depth = Math.min( - settings.maxCutDb, - Math.max(settings.maxCutDb / 2, settings.maxCutDb * relative), + profile.maxCutDb, + Math.max(profile.maxCutDb / 2, profile.maxCutDb * relative), ); - return { freq: center, gainDb: -Number(depth.toFixed(2)), q: settings.q }; + return { freq: center, gainDb: -Number(depth.toFixed(2)), q: profile.q }; }) .sort((a, b) => a.freq - b.freq); } +/** + * How far below a band's own loudest moment counts as nothing to carve. A voice + * 30 dB down on its peak is a room tone or a breath, not speech, and the bed + * should be flat there. + */ +const DYNAMIC_RANGE_DB = 30; + +/** + * Envelope follower time constants, in seconds. Asymmetric on purpose, the way + * a ducker is: the cut has to be there by the time a word is audible, and has to + * leave slowly enough that the bed does not pump between syllables. + */ +const ATTACK_S = 0.05; +const RELEASE_S = 0.25; + +/** + * Release for the level envelope, which is far slower than the filters' own. + * + * A notch closing quickly is inaudible — nothing about the bed's loudness + * changes. The whole bed coming back is not: at a filter's 0.25s the music + * jumped to full the instant a word ended, and what you hear then is the effect + * switching off rather than a mix breathing. Slower than any gap inside a + * sentence, so it rides through the pauses between words and only recovers + * between sentences. + * + * It also has to be well clear of the envelope's own step. A long voice is + * measured about every 0.3s, so a release anywhere near that puts the entire + * recovery inside one step — a jump, whatever the constant claims. + */ +const DUCK_RELEASE_S = 1.6; + +/** + * Windows an envelope may spend. Held under `MAX_AUTOMATION_POINTS` with room + * for the two end points, because `normalizeAutomation` truncates an over-budget + * lane rather than thinning it — losing the tail would leave the last cut held + * for the rest of the bed, which is the one failure this feature must not have. + */ +const POINT_BUDGET = 400; + +/** Below this a cut is inaudible, so it is recorded as no cut at all. */ +const SNAP_DB = 0.2; + +/** + * Smallest move worth its own breakpoint. Coarse enough that a lane stays + * editable by hand — a syllable-rate envelope at the analysis hop draws a point + * every 85 ms, which is a wall of handles nobody can grab — and fine enough that + * the shape is unchanged, since the segments either side interpolate. + */ +const STEP_DB = 0.5; + +export interface HfCarveDynamics { + /** The band this envelope drives, matching a band from `analyseCarveBands`. */ + freq: number; + /** Gain in dB over time, in seconds from the start of the *voice* clip. */ + points: { t: number; v: number }[]; +} + +/** + * Turn each carved band into an envelope that follows the voice's level in that + * same band. + * + * The depth from `analyseCarveBands` becomes the envelope's ceiling rather than a + * constant: full depth where that band is at its loudest in the voice, flat where + * the voice is silent, scaled in dB between the two. Written as automation on the + * filters' gain, so playback and render both already know how to follow it. + * + * Times are relative to the voice clip, since that is what was measured. A caller + * placing these on another element shifts them by the gap between the two clips' + * starts. This assumes the voice plays from its own beginning at its `data-start` + * — a clip with a media offset would need that added. + */ +export function analyseCarveDynamics( + voice: Float32Array, + sampleRate: number, + bands: HfCarveBand[], +): HfCarveDynamics[] { + if (voice.length === 0 || bands.length === 0) return []; + + const window = new Float32Array(FRAME); + for (let i = 0; i < FRAME; i++) window[i] = 0.5 - 0.5 * Math.cos((2 * Math.PI * i) / (FRAME - 1)); + + // One window per emitted point, so the hop is what keeps the lane in budget. + const hop = Math.max(FRAME, Math.ceil(voice.length / POINT_BUDGET)); + const bins = FRAME / 2 + 1; + const freqs: number[] = []; + for (let k = 0; k < bins; k++) freqs.push((k * sampleRate) / FRAME); + + const times: number[] = []; + const perBand = bands.map(() => [] as number[]); + for (let start = 0; start < voice.length; start += hop) { + const re = new Float64Array(FRAME); + const im = new Float64Array(FRAME); + for (let i = 0; i < FRAME; i++) re[i] = (voice[start + i] ?? 0) * window[i]!; + fft(re, im); + const power: number[] = []; + for (let k = 0; k < bins; k++) power.push(re[k]! * re[k]! + im[k]! * im[k]!); + times.push((start + FRAME / 2) / sampleRate); + bands.forEach((band, b) => perBand[b]!.push(bandPower(freqs, power, band.freq))); + } + + const duration = voice.length / sampleRate; + const attack = 1 - Math.exp(-(hop / sampleRate) / ATTACK_S); + const release = 1 - Math.exp(-(hop / sampleRate) / RELEASE_S); + + return bands.map((band, b) => { + const powers = perBand[b]!; + const peak = Math.max(...powers); + let level = 0; + const points: { t: number; v: number }[] = [{ t: 0, v: 0 }]; + let lastKept = 0; + const raw: { t: number; v: number }[] = []; + powers.forEach((p, i) => { + // Relative to this band's own loudest moment, so a quiet band still gets a + // full envelope rather than a permanently shallow one. + const target = + p > 0 && peak > 0 ? Math.max(0, 1 + (10 * Math.log10(p / peak)) / DYNAMIC_RANGE_DB) : 0; + level += (target > level ? attack : release) * (target - level); + const scaled = band.gainDb * level; + const v = Math.abs(scaled) < SNAP_DB ? 0 : Number(scaled.toFixed(1)); + raw.push({ t: Number(times[i]!.toFixed(3)), v }); + }); + let keptIndex = -1; + raw.forEach((pt, i) => { + if (i !== raw.length - 1 && Math.abs(pt.v - lastKept) < STEP_DB) return; + // Anchor the run this point ends. Segments interpolate, so dropping a run + // of equal values does not hold them — it draws a straight line from + // wherever the last kept point was, which turned a silent stretch before + // the first word into a slow 2 dB slide into it. Keeping the last value of + // the run pins the plateau flat and puts the whole move where it belongs. + if (i > 0 && keptIndex !== i - 1) points.push(raw[i - 1]!); + points.push(pt); + lastKept = pt.v; + keptIndex = i; + }); + // The last point's value is held for the rest of the bed, so it has to be + // no cut and it has to be at the end of the voice. + points.push({ t: Number(Math.max(duration, points.at(-1)!.t).toFixed(3)), v: 0 }); + return { freq: band.freq, points }; + }); +} + +/** Level in dB of one window, or -Infinity for silence. */ +function windowDb(samples: Float32Array, from: number, count: number): number { + let sum = 0; + let n = 0; + for (let i = from; i < from + count; i++) { + const s = samples[i]; + if (s === undefined) break; + sum += s * s; + n += 1; + } + if (n === 0) return Number.NEGATIVE_INFINITY; + const rms = Math.sqrt(sum / n); + return rms > 0 ? 20 * Math.log10(rms) : Number.NEGATIVE_INFINITY; +} + +/** + * The level envelope that keeps a bed under a voice. + * + * The spectral carve makes room in the frequency domain, which does nothing + * about a bed that is simply louder than the voice: the notches sit in the right + * places while the level goes on winning. This measures both tracks over the + * same windows and returns the gain the bed needs to sit `headroomDb` under the + * voice — no more than `duckDb`, and nothing at all where the voice is not + * speaking, so pauses stay open rather than being held down. + * + * Times are in seconds from the start of the *voice* clip; `offsetSeconds` is + * how far into the bed the voice's start falls, so the two are read at the same + * moment of the composition even when the clips begin at different times. + */ +export function analyseCarveDuck( + voice: Float32Array, + bed: Float32Array, + sampleRate: number, + profile: HfCarveProfile, + offsetSeconds: number, +): { t: number; v: number }[] { + if (voice.length === 0 || profile.duckDb <= 0) return []; + + const hop = Math.max(FRAME, Math.ceil(voice.length / POINT_BUDGET)); + const bedOffset = Math.round(offsetSeconds * sampleRate); + + const voiceDbs: number[] = []; + const bedDbs: number[] = []; + const times: number[] = []; + for (let start = 0; start < voice.length; start += hop) { + voiceDbs.push(windowDb(voice, start, FRAME)); + bedDbs.push(windowDb(bed, start + bedOffset, FRAME)); + times.push((start + FRAME / 2) / sampleRate); + } + + // Speaking, rather than merely non-zero: a room tone 30 dB under the voice's + // own peak is not something to duck for, and gating on it is what keeps the + // bed up through pauses. + const voicePeak = Math.max(...voiceDbs); + const speakingFloor = voicePeak - DYNAMIC_RANGE_DB; + + const attack = 1 - Math.exp(-(hop / sampleRate) / ATTACK_S); + const release = 1 - Math.exp(-(hop / sampleRate) / DUCK_RELEASE_S); + + let level = 0; + const raw: { t: number; v: number }[] = []; + voiceDbs.forEach((vDb, i) => { + const bDb = bedDbs[i] ?? Number.NEGATIVE_INFINITY; + // How far over the line the bed is, right now. + const over = vDb > speakingFloor && Number.isFinite(bDb) ? bDb - (vDb - profile.headroomDb) : 0; + const target = Math.min(profile.duckDb, Math.max(0, over)); + level += (target > level ? attack : release) * (target - level); + const v = level < SNAP_DB ? 0 : -Number(level.toFixed(1)); + raw.push({ t: Number(times[i]!.toFixed(3)), v }); + }); + + // Let the release finish past the last word. A lane's final point has to be no + // cut — otherwise the bed stays dipped for the rest of the clip — but jumping + // there from a full duck is the same flip the slow release exists to prevent, + // moved to the end of the narration. So the decay carries on at its own rate + // beyond the voice until it is inaudible, and only then pins. + const hopSeconds = hop / sampleRate; + let tailTime = times.at(-1) ?? 0; + while (level >= SNAP_DB) { + level += release * -level; + tailTime += hopSeconds; + raw.push({ + t: Number(tailTime.toFixed(3)), + v: level < SNAP_DB ? 0 : -Number(level.toFixed(1)), + }); + } + + const points: { t: number; v: number }[] = [{ t: 0, v: 0 }]; + let lastKept = 0; + let keptIndex = -1; + raw.forEach((pt, i) => { + if (i !== raw.length - 1 && Math.abs(pt.v - lastKept) < STEP_DB) return; + if (i > 0 && keptIndex !== i - 1) points.push(raw[i - 1]!); + points.push(pt); + lastKept = pt.v; + keptIndex = i; + }); + // Pin the end at no cut, unless the release already got there — a second point + // at the same time and value is just noise in the lane. + const duration = voice.length / sampleRate; + const last = points.at(-1)!; + const endTime = Number(Math.max(duration, last.t).toFixed(3)); + if (!(last.v === 0 && last.t >= endTime)) points.push({ t: endTime, v: 0 }); + return points; +} + /** Carve bands as an ordinary FX chain of peaking filters. */ export function carveBandsToChain(bands: HfCarveBand[]): HfAudioFxChain { const nodes: HfAudioFxNode[] = bands.map((b) => ({ diff --git a/packages/core/src/audioFx.ts b/packages/core/src/audioFx.ts index 2eb2db83d4..1e38c4f298 100644 --- a/packages/core/src/audioFx.ts +++ b/packages/core/src/audioFx.ts @@ -136,6 +136,16 @@ const poles: HfAudioFxEnumParam = { * a value that survives `normalizeAudioFxParams` is always safe to realise. */ export const HF_AUDIO_FX: readonly HfAudioFxDef[] = [ + { + id: "gain", + label: "Gain", + group: "dynamics", + description: "Raise or lower the whole signal. Automate it to duck under something else.", + // Cuts go deep, boosts stay modest: this is a level stage for making room, + // and a chain that can add 40 dB clips long before it is useful. + params: [gainDb(-60, 12, 0)], + web: "gain-node", + }, { id: "peaking", label: "Peaking EQ", diff --git a/packages/core/src/runtime/audioFx.test.ts b/packages/core/src/runtime/audioFx.test.ts index 46ac76139f..9774515810 100644 --- a/packages/core/src/runtime/audioFx.test.ts +++ b/packages/core/src/runtime/audioFx.test.ts @@ -1,6 +1,7 @@ // @vitest-environment happy-dom import { describe, expect, it } from "vitest"; import { attachElementFxChain } from "./audioFx.js"; +import { defaultAudioFxParams, HF_AUDIO_FX } from "../audioFx.js"; /** * The DSP is proven in a real browser by the engine's render tests. What needs @@ -187,6 +188,122 @@ describe("attachElementFxChain", () => { expect(before.disconnected).toBe(true); }); + const violations: string[] = []; + + /** + * An AudioParam with the browser's own scheduling rule: writing `.value` + * while a value curve covers the current time throws, because setting the + * property is a `setValueAtTime` at that instant and a curve may not overlap + * another event. Chrome words it exactly as this does. + */ + class SchedParam { + curves: { time: number; duration: number }[] = []; + sets: number[] = []; + #value = 0; + constructor(private clock: { currentTime: number }) {} + get value(): number { + return this.#value; + } + set value(v: number) { + const t = this.clock.currentTime; + const covering = this.curves.find((c) => t >= c.time && t <= c.time + c.duration); + if (covering) { + // Recorded as well as thrown: the throw happens inside a + // MutationObserver callback, where nothing rethrows it into the test — + // which is exactly why this only ever showed up as an uncaught error + // in the preview console. + const message = + `Failed to set the 'value' property on 'AudioParam': setValueAtTime(${v}, ${t}) ` + + `overlaps setValueCurveAtTime(..., ${covering.time}, ${covering.duration})`; + violations.push(message); + throw new Error(message); + } + this.#value = v; + } + setValueAtTime(v: number, t: number): void { + this.sets.push(t); + this.#value = v; + } + linearRampToValueAtTime(v: number): void { + this.#value = v; + } + setValueCurveAtTime(_v: Float32Array, time: number, duration: number): void { + this.curves.push({ time, duration }); + } + cancelScheduledValues(from: number): void { + this.curves = this.curves.filter((c) => c.time + c.duration < from); + } + cancelAndHoldAtTime(from: number): void { + this.cancelScheduledValues(from); + } + } + + it("edits a chain whose parameters an envelope is mid-curve on", async () => { + // The level stage a carve adds is automated, so a chain edit lands while a + // curve is running. Writing the new value straight onto the param throws + // NotSupportedError, which surfaced in the console as the preview refusing + // every edit after the first. + const clock = { currentTime: 0 }; + class SchedNode extends Node { + override frequency = new SchedParam(clock) as unknown as { value: number }; + override Q = new SchedParam(clock) as unknown as { value: number }; + override gain = new SchedParam(clock) as unknown as { value: number }; + } + class SchedCtx extends Ctx { + get currentTime(): number { + return clock.currentTime; + } + override createGain(): Node { + return new SchedNode(); + } + override createBiquadFilter(): Node { + return new SchedNode(); + } + } + // A gain stage, because its dB-to-linear mapping is what makes the + // scheduler emit a value curve instead of a ramp — and a curve is what a + // `.value` write may not land inside. + const chainWith = (freq: number) => ({ + version: 1, + nodes: [ + { type: "gain", id: "n1", params: { gain: 0 } }, + { type: "peaking", id: "n2", params: { frequency: freq, gain: -6, q: 1 } }, + ], + }); + const node = audioEl(chainWith(1000)); + // A bent segment, so the scheduler emits a curve rather than a ramp. + node.setAttribute( + "data-automation", + JSON.stringify({ + version: 1, + lanes: [ + { + target: "fx.n1.gain", + points: [ + { t: 0, v: 0 }, + { t: 8, v: -12 }, + ], + }, + ], + }), + ); + attachElementFxChain( + new SchedCtx() as unknown as BaseAudioContext, + node, + new Node() as never, + new Node() as never, + { scheduledAt: 0, elapsed: 0, rate: 1 }, + ); + + // Mid-curve, which is when the edit has to survive. + clock.currentTime = 3; + node.setAttribute("data-fx-chain", JSON.stringify(chainWith(1200))); + await settle(); + + expect(violations).toEqual([]); + expect(node.getAttribute("data-fx-chain")).toContain("1200"); + }); + it("keeps playing dry when an edit leaves the chain unreadable", async () => { const src = new Node(); const dst = new Node(); @@ -205,4 +322,370 @@ describe("attachElementFxChain", () => { handle!.dispose(); expect((src.connections[0] as Node).disconnected).toBe(true); }); + + /** + * A limiter, a compressor, a gate and a bitcrush are AudioWorklet processors, + * and a worklet node cannot be constructed before its module has registered. So + * a chain containing one plays dry for a moment and swaps in when registration + * resolves — and that path used to end there: no automation scheduled, no + * observer watching the attribute. Adding a compressor to a carved bed + * therefore killed the carve's envelopes and froze every later edit until the + * composition reloaded. + */ + describe("a chain that needs worklets before it can be built", () => { + /** + * Constructs only once the module has registered on that context, which is + * what Chrome does — and the whole reason this path exists. A stub that always + * succeeds hides the bug: the build then works even when nothing registered. + */ + class WorkletNodeStub extends Node { + port = { postMessage: () => {} }; + constructor(ctx: { workletsReady?: boolean }, name: string) { + super("worklet"); + if (!ctx.workletsReady) { + throw new Error(`AudioWorkletNode: processor "${name}" is not registered`); + } + } + } + + /** A param that records whether an envelope was ever scheduled onto it. */ + class RecordingParam { + scheduled = false; + value = 0; + setValueAtTime(): void { + this.scheduled = true; + } + linearRampToValueAtTime(): void { + this.scheduled = true; + } + setValueCurveAtTime(): void { + this.scheduled = true; + } + cancelScheduledValues(): void {} + cancelAndHoldAtTime(): void {} + } + + /** + * A context whose worklet module registers when asked, and which hands out + * filters whose gain reports being automated. + */ + class WorkletCtx extends Ctx { + currentTime = 0; + workletsReady = false; + audioWorklet = { + addModule: async () => { + this.workletsReady = true; + }, + }; + biquads: { gain: RecordingParam }[] = []; + override createBiquadFilter() { + const node = new Node("biquad") as unknown as Node & { gain: RecordingParam }; + node.gain = new RecordingParam(); + this.biquads.push(node); + return node as unknown as Node; + } + } + + const carved = { + version: 1, + nodes: [ + { + type: "peaking", + id: "n1", + fromCarve: true, + params: { frequency: 1000, gain: -6, q: 1.4 }, + }, + { type: "compressor", id: "n2", params: {} }, + ], + }; + const lane = JSON.stringify({ + version: 1, + lanes: [ + { + target: "fx.n1.gain", + points: [ + { t: 0, v: 0 }, + { t: 4, v: -6 }, + ], + }, + ], + }); + + const audioEl = (): HTMLElement => { + const node = document.createElement("audio"); + node.setAttribute("data-fx-chain", JSON.stringify(carved)); + node.setAttribute("data-automation", lane); + document.body.append(node); + return node; + }; + const settle = async () => { + await Promise.resolve(); + await Promise.resolve(); + await new Promise((r) => setTimeout(r, 0)); + }; + + it("schedules the carve's envelopes once the processors land", async () => { + const original = (globalThis as { AudioWorkletNode?: unknown }).AudioWorkletNode; + (globalThis as { AudioWorkletNode?: unknown }).AudioWorkletNode = WorkletNodeStub; + try { + const worklets = new WorkletCtx(); + const src = new Node(); + const dst = new Node(); + const node = audioEl(); + attachElementFxChain( + worklets as unknown as BaseAudioContext, + node, + src as never, + dst as never, + { + scheduledAt: 0, + elapsed: 0, + rate: 1, + }, + ); + // Dry to begin with, which is the whole reason for this path. + expect(src.connections).toContain(dst); + await settle(); + // Now routed through the chain — and the envelope is playing, which is + // what "the carve still works" means. + expect(src.connections.at(-1)).not.toBe(dst); + const filter = worklets.biquads[0]; + expect(filter, "the carve's filter was never built").toBeTruthy(); + expect(filter!.gain.scheduled).toBe(true); + } finally { + (globalThis as { AudioWorkletNode?: unknown }).AudioWorkletNode = original; + } + }); + + it("keeps the carve alive when a compressor is added to a playing track", async () => { + // The reported case, and the worse of the two paths: adding a worklet effect + // to an already-attached chain rebuilt it, the build threw because nothing + // on that path had registered the module, and the catch wired the source + // straight to its gain — dropping the carve's filters along with the + // compressor. Both symptoms, one cause. + const original = (globalThis as { AudioWorkletNode?: unknown }).AudioWorkletNode; + (globalThis as { AudioWorkletNode?: unknown }).AudioWorkletNode = WorkletNodeStub; + try { + const worklets = new WorkletCtx(); + const src = new Node(); + const dst = new Node(); + // Starts with the carve only: no worklets, so it attaches immediately. + const node = document.createElement("audio"); + node.setAttribute( + "data-fx-chain", + JSON.stringify({ + version: 1, + nodes: [ + { + type: "peaking", + id: "n1", + fromCarve: true, + params: { frequency: 1000, gain: -6, q: 1.4 }, + }, + ], + }), + ); + node.setAttribute("data-automation", lane); + document.body.append(node); + attachElementFxChain( + worklets as unknown as BaseAudioContext, + node, + src as never, + dst as never, + { scheduledAt: 0, elapsed: 0, rate: 1 }, + ); + expect(worklets.biquads[0]!.gain.scheduled).toBe(true); + + // Now add the compressor, as the panel's "Add effect" does. + node.setAttribute("data-fx-chain", JSON.stringify(carved)); + await settle(); + + // Still routed through a chain rather than dry, the carve's filter rebuilt, + // and its envelope scheduled onto the new one. + expect(src.connections.at(-1)).not.toBe(dst); + const rebuilt = worklets.biquads.at(-1)!; + expect(rebuilt.gain.scheduled).toBe(true); + } finally { + (globalThis as { AudioWorkletNode?: unknown }).AudioWorkletNode = original; + } + }); + + it("follows the attribute after the late build, as any other chain does", async () => { + const original = (globalThis as { AudioWorkletNode?: unknown }).AudioWorkletNode; + (globalThis as { AudioWorkletNode?: unknown }).AudioWorkletNode = WorkletNodeStub; + try { + const worklets = new WorkletCtx() as unknown as BaseAudioContext; + const src = new Node(); + const dst = new Node(); + const node = audioEl(); + attachElementFxChain(worklets, node, src as never, dst as never, { + scheduledAt: 0, + elapsed: 0, + rate: 1, + }); + await settle(); + const before = src.connections.at(-1); + node.setAttribute( + "data-fx-chain", + JSON.stringify({ + version: 1, + nodes: [{ type: "lowpass", id: "n9", params: { frequency: 900, q: 1, poles: "2" } }], + }), + ); + await settle(); + expect(src.connections.at(-1)).not.toBe(before); + } finally { + (globalThis as { AudioWorkletNode?: unknown }).AudioWorkletNode = original; + } + }); + }); + + /** + * Every effect in the registry, one at a time. + * + * The limiter/compressor bug was not really about worklets: it was a build that + * threw inside `attach`, whose catch wires the source straight to its gain — + * dropping every other effect in the chain, the carve's filters included. Any + * effect whose construction can fail has the same shape, so this asserts the two + * things that were broken for all fourteen: the chain reaches the signal path, + * and an envelope on it gets scheduled. + */ + describe("every registry effect, in the signal path", () => { + class RecordingParam { + scheduled = false; + value = 0; + setValueAtTime(): void { + this.scheduled = true; + } + linearRampToValueAtTime(): void { + this.scheduled = true; + } + setValueCurveAtTime(): void { + this.scheduled = true; + } + cancelScheduledValues(): void {} + cancelAndHoldAtTime(): void {} + } + class RecordingNode extends Node { + override frequency = new RecordingParam() as unknown as { value: number }; + override Q = new RecordingParam() as unknown as { value: number }; + override gain = new RecordingParam() as unknown as { value: number }; + override delayTime = new RecordingParam() as unknown as { value: number }; + port = { postMessage: () => {} }; + } + class RichCtx extends Ctx { + currentTime = 0; + workletsReady = false; + made: RecordingNode[] = []; + audioWorklet = { + addModule: async () => { + this.workletsReady = true; + }, + }; + private make(): Node { + const node = new RecordingNode("node"); + this.made.push(node); + return node as unknown as Node; + } + override createGain() { + return this.make(); + } + override createBiquadFilter() { + return this.make(); + } + override createIIRFilter() { + return this.make(); + } + override createDelay() { + return this.make(); + } + override createOscillator() { + return this.make(); + } + override createWaveShaper() { + return this.make(); + } + override createConvolver() { + return this.make(); + } + } + class WorkletNode extends RecordingNode { + constructor(ctx: { workletsReady?: boolean }, name: string) { + super("worklet"); + if (!ctx.workletsReady) { + throw new Error(`AudioWorkletNode: processor "${name}" is not registered`); + } + } + } + const settle = async () => { + await Promise.resolve(); + await Promise.resolve(); + await new Promise((r) => setTimeout(r, 0)); + }; + + for (const def of HF_AUDIO_FX) { + const automatable = def.params.find((p) => p.kind === "number" && p.automatable); + + it(`routes ${def.id} and plays its envelope`, async () => { + const original = (globalThis as { AudioWorkletNode?: unknown }).AudioWorkletNode; + (globalThis as { AudioWorkletNode?: unknown }).AudioWorkletNode = WorkletNode; + try { + const ctxRich = new RichCtx(); + const src = new Node(); + const dst = new Node(); + const node = document.createElement("audio"); + node.setAttribute( + "data-fx-chain", + JSON.stringify({ + version: 1, + nodes: [{ type: def.id, id: "n1", params: defaultAudioFxParams(def.id) }], + }), + ); + if (automatable && automatable.kind === "number") { + node.setAttribute( + "data-automation", + JSON.stringify({ + version: 1, + lanes: [ + { + target: `fx.n1.${automatable.key}`, + points: [ + { t: 0, v: automatable.default }, + { t: 4, v: automatable.max }, + ], + }, + ], + }), + ); + } + document.body.append(node); + attachElementFxChain( + ctxRich as unknown as BaseAudioContext, + node, + src as never, + dst as never, + { scheduledAt: 0, elapsed: 0, rate: 1 }, + ); + await settle(); + + // Not dry: the effect is between the source and its gain. + expect(src.connections.at(-1), `${def.id} was left out of the path`).not.toBe(dst); + if (automatable) { + const scheduled = ctxRich.made.some((n) => + [n.frequency, n.Q, n.gain, n.delayTime].some( + (p) => (p as unknown as RecordingParam).scheduled, + ), + ); + // A worklet exposes no AudioParams, so a lane on one has nowhere to + // write — that is by design and the scheduler skips it. + if (!def.web.startsWith("worklet-")) { + expect(scheduled, `${def.id}'s envelope was never scheduled`).toBe(true); + } + } + } finally { + (globalThis as { AudioWorkletNode?: unknown }).AudioWorkletNode = original; + } + }); + } + }); }); diff --git a/packages/core/src/runtime/audioFx.ts b/packages/core/src/runtime/audioFx.ts index bf37fabfe0..8bd4c4e3ad 100644 --- a/packages/core/src/runtime/audioFx.ts +++ b/packages/core/src/runtime/audioFx.ts @@ -19,6 +19,7 @@ import { } from "../audioAutomation.js"; import { cancelParamLane, + clearParamLane, scheduleChainAutomation, type AutomationTiming, } from "../audio/audioFxAutomation.js"; @@ -91,41 +92,15 @@ export function attachElementFxChain( ): { dispose(): void } | null { const { chain } = readChain(el); - // An AudioWorkletNode cannot be constructed before its processor is - // registered — it throws, and the whole chain is lost. So when the chain - // needs worklets and the module has not landed yet, play dry and swap the - // graph in once registration resolves. - if (chainNeedsWorklets(chain) && !audioFxWorkletsReady(ctx)) { - source.connect(destination); - let cancelled = false; - let pending: FxChainHandle | null = null; - void ensureAudioFxWorklets(ctx) - .then(() => { - if (cancelled) return; - try { - const late = buildFxChain(ctx, chain); - source.disconnect(destination); - source.connect(late.input); - late.output.connect(destination); - pending = late; - } catch { - // Still unbuildable; the dry connection already stands. - } - }) - .catch(() => undefined); - return { - dispose: () => { - cancelled = true; - pending?.dispose(); - }, - }; - } - // Null means the source runs straight into its gain: an empty chain, or one // that could not be realised. Mutable because a structural edit swaps the // whole graph rather than re-parameterising it. let handle: FxChainHandle | null = null; let automated: FxParamTarget[] = []; + let disposed = false; + /** Bumped per attach, so a worklet wait that resolves late cannot revive a + * chain the element has since moved on from. */ + let workletGeneration = 0; /** Take the current graph out of the path, leaving the source connected dry. */ const detach = (): void => { @@ -154,6 +129,28 @@ export function attachElementFxChain( source.connect(destination); return; } + // An AudioWorkletNode cannot be constructed before its processor is + // registered — it throws, and the whole chain is lost. So a chain holding a + // limiter, compressor, gate or bitcrush plays dry until the module lands and + // then takes the ordinary rebuild path. + // + // That wait used to return early from the whole function, which left the + // track with no automation scheduled and no observer on the attribute: adding + // a compressor to a carved bed killed the carve's envelopes and froze every + // later edit until the composition reloaded. Rebuilding through the same path + // an edit uses is what keeps those two working. + if (chainNeedsWorklets(next) && !audioFxWorkletsReady(ctx)) { + source.connect(destination); + const generation = ++workletGeneration; + void ensureAudioFxWorklets(ctx) + .then(() => { + // Stale if the element was disposed or the chain changed while waiting. + if (disposed || generation !== workletGeneration) return; + rebuild(readChain(el).chain); + }) + .catch(() => undefined); + return; + } try { const built = buildFxChain(ctx, next); source.connect(built.input); @@ -223,6 +220,13 @@ export function attachElementFxChain( ) { observer = new MutationObserver(() => { const next = readChain(el); + // Clear the booked envelopes before touching the graph. `update` writes each + // knob straight onto its AudioParam, and a write landing inside a running + // curve is refused with NotSupportedError unless the param is cancelled + // first — so an edit made while one was playing threw instead of applying, + // and the console filled with uncaught errors. The reschedule below puts the + // envelope back from the current playhead. + if (automated.length > 0) clearParamLane(automated); // `update` reports false when the change is structural rather than a new // set of values, which is the signal to swap the graph. if (!handle || !handle.update(next.chain)) rebuild(next.chain); @@ -236,6 +240,7 @@ export function attachElementFxChain( return { dispose: () => { + disposed = true; observer?.disconnect(); if (automated.length > 0) { cancelParamLane(automated, typeof ctx.currentTime === "number" ? ctx.currentTime : 0); diff --git a/packages/producer/src/services/render/stages/audioStage.ts b/packages/producer/src/services/render/stages/audioStage.ts index 1ed74658bf..bac3d3f552 100644 --- a/packages/producer/src/services/render/stages/audioStage.ts +++ b/packages/producer/src/services/render/stages/audioStage.ts @@ -18,6 +18,7 @@ import { join } from "node:path"; import { processCompositionAudio, type AudioProcessingFailure } from "@hyperframes/engine"; import type { CompositionMetadata } from "../shared.js"; +import type { ProducerLogger } from "../../../logger.js"; export interface AudioStageInput { projectDir: string; @@ -30,6 +31,8 @@ export interface AudioStageInput { audios: CompositionMetadata["audios"]; abortSignal: AbortSignal | undefined; assertNotAborted: () => void; + /** Where a per-track failure's detail goes. Optional so tests need not pass one. */ + log?: ProducerLogger; } export interface AudioStageResult { @@ -50,7 +53,7 @@ export interface AudioStageResult { } export async function runAudioStage(input: AudioStageInput): Promise { - const { projectDir, workDir, compiledDir, duration, audios, abortSignal, assertNotAborted } = + const { projectDir, workDir, compiledDir, duration, audios, abortSignal, assertNotAborted, log } = input; const stage3Start = Date.now(); @@ -97,7 +100,23 @@ export async function runAudioStage(input: AudioStageInput): Promise { + it("is stable for the same file", () => { + const a = buildWaveformCacheKey("assets/music-bed.m4a", { size: 4187869, mtimeMs: 1000 }); + const b = buildWaveformCacheKey("assets/music-bed.m4a", { size: 4187869, mtimeMs: 1000 }); + expect(a).toBe(b); + }); + + it("changes when the file behind the path is replaced", () => { + // The case this exists for: an asset rebuilt in place — same name, new + // content. Keyed on the path alone the cache served the old peaks forever, + // so a bed whose ducking had just been removed still drew as ducked. + const before = buildWaveformCacheKey("assets/music-bed.m4a", { size: 4187869, mtimeMs: 1000 }); + const after = buildWaveformCacheKey("assets/music-bed.m4a", { size: 3900000, mtimeMs: 2000 }); + expect(after).not.toBe(before); + }); + + it("separates two files of the same size edited at different times, and vice versa", () => { + const base = { size: 100, mtimeMs: 1000 }; + expect(buildWaveformCacheKey("a.m4a", base)).not.toBe( + buildWaveformCacheKey("a.m4a", { ...base, mtimeMs: 1001 }), + ); + expect(buildWaveformCacheKey("a.m4a", base)).not.toBe( + buildWaveformCacheKey("a.m4a", { ...base, size: 101 }), + ); + }); + + it("keeps distinct assets apart and stays a plain filename", () => { + const fp = { size: 10, mtimeMs: 5 }; + expect(buildWaveformCacheKey("a/b.m4a", fp)).not.toBe(buildWaveformCacheKey("a/c.m4a", fp)); + expect(buildWaveformCacheKey("a/b.m4a", fp)).not.toMatch(/[/\\]/); + expect(buildWaveformCacheKey("a/b.m4a", fp)).toMatch(/\.json$/); + }); +}); diff --git a/packages/studio-server/src/helpers/waveform.ts b/packages/studio-server/src/helpers/waveform.ts index e2f972db15..79e3f1ca47 100644 --- a/packages/studio-server/src/helpers/waveform.ts +++ b/packages/studio-server/src/helpers/waveform.ts @@ -7,8 +7,28 @@ const SAMPLE_RATE = 4000; const PEAK_COUNT = 4000; const WAVEFORM_CACHE_VERSION = "v2"; -export function buildWaveformCacheKey(assetPath: string): string { - return `${WAVEFORM_CACHE_VERSION}_${assetPath.replace(/[/\\]/g, "_")}.json`; +/** + * Cache filename for one asset's peaks, keyed on its content as well as its name. + * + * The path alone is not an identity. An asset rebuilt in place — a bed + * re-encoded without its ducking, a plate swapped for the right one — keeps its + * name and gets new samples, and a path-keyed entry then served the old peaks + * for the rest of the project's life: the timeline drew a duck that was no + * longer in the file, which reads as the render having done it. Size and mtime + * are what a rebuild always changes, and both are already on the stat the route + * takes to check the file exists. + * + * Without a fingerprint it falls back to the old path-only key, so a caller that + * cannot stat still gets caching rather than an error. + */ +export function buildWaveformCacheKey( + assetPath: string, + fingerprint?: { size: number; mtimeMs: number }, +): string { + const name = assetPath.replace(/[/\\]/g, "_"); + if (!fingerprint) return `${WAVEFORM_CACHE_VERSION}_${name}.json`; + const stamp = `${fingerprint.size}-${Math.round(fingerprint.mtimeMs)}`; + return `${WAVEFORM_CACHE_VERSION}_${name}_${stamp}.json`; } function computePeaks(floats: Float32Array, count: number): number[] { diff --git a/packages/studio-server/src/routes/waveform.ts b/packages/studio-server/src/routes/waveform.ts index 8677ef5bba..f40ef79536 100644 --- a/packages/studio-server/src/routes/waveform.ts +++ b/packages/studio-server/src/routes/waveform.ts @@ -1,4 +1,4 @@ -import { existsSync, readFileSync, writeFileSync, mkdirSync } from "node:fs"; +import { existsSync, readFileSync, writeFileSync, mkdirSync, statSync } from "node:fs"; import { join } from "node:path"; import type { Hono } from "hono"; import type { StudioApiAdapter } from "../types.js"; @@ -13,10 +13,13 @@ export function registerWaveformRoutes(api: Hono, adapter: StudioApiAdapter): vo c.req.path.replace(`/projects/${project.id}/waveform/`, "").split("?")[0] ?? "", ); const audioPath = join(project.dir, assetPath); - if (!existsSync(audioPath)) return c.json({ error: "file not found" }, 404); + const stats = statSync(audioPath, { throwIfNoEntry: false }); + if (!stats) return c.json({ error: "file not found" }, 404); const cacheDir = join(project.dir, ".waveform-cache"); - const cachePath = join(cacheDir, buildWaveformCacheKey(assetPath)); + // Keyed on the file's size and mtime as well as its name, so re-encoding an + // asset in place invalidates its peaks instead of drawing the old ones. + const cachePath = join(cacheDir, buildWaveformCacheKey(assetPath, stats)); if (existsSync(cachePath)) { try { diff --git a/packages/studio/src/components/StudioRightPanel.tsx b/packages/studio/src/components/StudioRightPanel.tsx index 8ee3b9c88d..31ac169dd0 100644 --- a/packages/studio/src/components/StudioRightPanel.tsx +++ b/packages/studio/src/components/StudioRightPanel.tsx @@ -327,6 +327,21 @@ export function StudioRightPanel({ }, [projectId, refreshFileTree, showToast], ); + + /** + * A dial being dragged writes to the preview and stops there. + * + * Every one of these panels previews on each pointermove and commits on + * release. Persisting the moves too put a fragment of the drag in the undo + * stack — and since those writes race, history could not coalesce them + * reliably, so undo took back a sliver of the gesture rather than the gesture. + * The release's own commit is what reaches the file and the undo stack. + */ + const setAttributeWhileDragging = useCallback( + (attr: string, value: string | null) => + handleDomAttributeLiveCommit(attr, value, undefined, { previewOnly: true }), + [handleDomAttributeLiveCommit], + ); const handleHideAllSelected = () => { const { elements } = usePlayerStore.getState(); const keys = timelineKeysForSelections(domEditGroupSelections, elements, activeCompPath); @@ -361,7 +376,7 @@ export function StudioRightPanel({ onSetStyle={handleDomStyleCommit} onSetAttribute={handleDomAttributeCommit} onSetAttributes={handleDomAttributesCommit} - onSetAttributeLive={handleDomAttributeLiveCommit} + onSetAttributeLive={setAttributeWhileDragging} onSetAttributeQuiet={handleDomAttributeQuietCommit} onApplyColorGradingScope={handleApplyColorGradingScope} onSetHtmlAttribute={handleDomHtmlAttributeCommit} diff --git a/packages/studio/src/components/TimelineToolbar.test.tsx b/packages/studio/src/components/TimelineToolbar.test.tsx index 0050dcfc95..ded3f32cbf 100644 --- a/packages/studio/src/components/TimelineToolbar.test.tsx +++ b/packages/studio/src/components/TimelineToolbar.test.tsx @@ -99,3 +99,49 @@ describe("TimelineToolbar — motion path endpoints", () => { act(() => root.unmount()); }); }); + +describe("TimelineToolbar — keyframes on audio tracks", () => { + const clip = (tag: string) => ({ + id: "bgm", + key: "bgm", + tag, + start: 0, + duration: 10, + track: 1, + }); + + /** A session whose selection would otherwise offer the keyframe toggle. */ + function sessionFor(tag: string) { + usePlayerStore.setState({ elements: [clip(tag)], selectedElementId: "bgm", currentTime: 1 }); + const element = document.createElement(tag); + element.id = "bgm"; + return { + domEditSelection: makeSelection("Element", element), + selectedGsapAnimations: [], + handleGsapAddAnimation: vi.fn(), + handleGsapConvertToKeyframes: vi.fn(), + handleGsapRemoveKeyframe: vi.fn(), + } satisfies NonNullable["domEditSession"]>; + } + + it("offers no keyframe toggle for an audio clip", () => { + // An audio clip has no box on the canvas, so there is nothing to move or fade — + // and pressing this seeded a tween from the position properties, which put a + // position lane on a track that has no position. Audio is automated instead. + const { host, root } = renderToolbar(sessionFor("audio")); + const button = host.querySelector( + 'button[aria-label="Add keyframe at playhead"]', + ); + expect(button?.disabled).toBe(true); + act(() => root.unmount()); + }); + + it("still offers it for a visual clip", () => { + const { host, root } = renderToolbar(sessionFor("div")); + const button = host.querySelector( + 'button[aria-label="Add keyframe at playhead"]', + ); + expect(button?.disabled).toBe(false); + act(() => root.unmount()); + }); +}); diff --git a/packages/studio/src/components/TimelineToolbar.tsx b/packages/studio/src/components/TimelineToolbar.tsx index 5391488000..16ea481eda 100644 --- a/packages/studio/src/components/TimelineToolbar.tsx +++ b/packages/studio/src/components/TimelineToolbar.tsx @@ -86,8 +86,22 @@ function resolveKeyframeToggleState( }; } +/** + * Can this element be keyframed at all? + * + * An audio clip cannot. It has no box on the canvas, so there is nothing to move, + * scale or fade — and "add a keyframe" on one seeds a tween from the position + * properties, which produced a position lane on a track that has no position. Audio + * is automated instead: volume and effect parameters, on their own lanes. + */ +function isKeyframeable(element: TimelineElement | undefined): boolean { + return element?.tag !== "audio"; +} + function useKeyframeToggle(session?: DomEditSessionSlice) { const currentTime = usePlayerStore((s) => s.currentTime); + const selectedElementId = usePlayerStore((s) => s.selectedElementId); + const elements = usePlayerStore((s) => s.elements); const sessionRef = useRef(session); sessionRef.current = session; @@ -95,6 +109,9 @@ function useKeyframeToggle(session?: DomEditSessionSlice) { sessionRef as React.RefObject, ); + const selected = elements.find((element) => (element.key ?? element.id) === selectedElementId); + if (!isKeyframeable(selected)) return { ...NO_KEYFRAME_TOGGLE, onToggle: undefined }; + const toggleState = resolveKeyframeToggleState(session, currentTime); return { diff --git a/packages/studio/src/components/editor/PropertyPanel.tsx b/packages/studio/src/components/editor/PropertyPanel.tsx index 7308aca237..9b14d83a6d 100644 --- a/packages/studio/src/components/editor/PropertyPanel.tsx +++ b/packages/studio/src/components/editor/PropertyPanel.tsx @@ -1,5 +1,5 @@ import { scopedElementKey } from "../../hooks/gsapKeyframeCacheHelpers"; -import { memo, useEffect, useMemo, useRef, useState } from "react"; +import { memo, useMemo, useRef, useState } from "react"; import { Move } from "../../icons/SystemIcons"; import { InspectorHeaderActions } from "./InspectorHeaderActions"; import { useStudioShellContext } from "../../contexts/StudioContext"; @@ -29,7 +29,8 @@ import { KeyframeNavigation } from "./KeyframeNavigation"; import { STUDIO_FLAT_INSPECTOR_ENABLED } from "./manualEditingAvailability"; import { PropertyPanelFlat } from "./PropertyPanelFlat"; import { createGsapLivePreview } from "./gsapLivePreview"; -import { usePlayerStore, liveTime } from "../../player"; +import { usePlayerStore } from "../../player"; +import { useLivePlayheadTime } from "../../hooks/useLivePlayheadTime"; import { TimingSection } from "./propertyPanelTimingSection"; import { type PropertyPanelProps } from "./propertyPanelHelpers"; import { GestureRecordPanelButton } from "./GestureRecordControl"; @@ -114,31 +115,14 @@ export const PropertyPanel = memo(function PropertyPanel(props: PropertyPanelPro const { showToast } = useStudioShellContext(); const [clipboardCopied, setClipboardCopied] = useState(false); const clipboardTimerRef = useRef>(undefined); - const storeTime = usePlayerStore((s) => s.currentTime); - const isPlaying = usePlayerStore((s) => s.isPlaying); const timelineElements = usePlayerStore((s) => s.elements); const selectedElementId = usePlayerStore((s) => s.selectedElementId); const selectedElementHidden = isSelectedElementHidden(timelineElements, selectedElementId); const visibilityToggleLabel = selectedElementHidden ? "Show element" : "Hide element"; - const liveTimeRef = useRef(storeTime); - const [, forceRender] = useState(0); - useEffect(() => { - if (!isPlaying) return; - let timerId: ReturnType | 0 = 0; - const unsub = liveTime.subscribe((t) => { - liveTimeRef.current = t; - if (!timerId) - timerId = setTimeout(() => { - timerId = 0; - forceRender((v) => v + 1); - }, 33); - }); - return () => { - unsub(); - if (timerId) clearTimeout(timerId); - }; - }, [isPlaying]); - const currentTime = isPlaying ? liveTimeRef.current : storeTime; + // Live during playback, the store's when paused — see the hook. Shared with the + // audio FX panel, which follows the playhead for the same reason: a value the + // timeline drives has to be shown moving, not frozen at what the attribute says. + const currentTime = useLivePlayheadTime(); const cacheElementKey = element?.id ?? element?.selector ?? ""; const cacheEntry = usePlayerStore((s) => s.keyframeCache.get(cacheElementKey)); diff --git a/packages/studio/src/components/editor/PropertyPanelFlat.tsx b/packages/studio/src/components/editor/PropertyPanelFlat.tsx index cf86039fd2..1c7c20de68 100644 --- a/packages/studio/src/components/editor/PropertyPanelFlat.tsx +++ b/packages/studio/src/components/editor/PropertyPanelFlat.tsx @@ -6,6 +6,7 @@ import { slugifyDesignInput } from "../../utils/designInputTracking"; import { isTextEditableSelection } from "./domEditing"; import type { PropertyPanelFlatProps } from "./propertyPanelFlatProps"; import { formatPxMetricValue } from "./propertyPanelHelpers"; +import { audioFxSummary } from "./audioFxSummary"; import { PropertyPanelFlatHeader } from "./PropertyPanelFlatHeader"; import { PropertyPanelFlatFooter } from "./PropertyPanelFlatFooter"; import { FlatGroupHeader } from "./propertyPanelFlatPrimitives"; @@ -13,11 +14,9 @@ import { FlatTextSection } from "./propertyPanelFlatTextSection"; import { FlatStyleSection } from "./propertyPanelFlatStyleSections"; import { FlatLayoutSection } from "./propertyPanelFlatLayoutSection"; import { FlatMotionSection } from "./propertyPanelFlatMotionSection"; -import { parseAudioFxChain } from "@hyperframes/core/audio-fx"; import { AudioFxGroup } from "./propertyPanelAudioFxGroup.js"; import { useVolumeAutomation } from "./useVolumeAutomation"; import { FlatMediaSection } from "./propertyPanelFlatMediaSection"; -import type { DomEditSelection } from "./domEditing"; import { deriveElementTiming } from "./propertyPanelFlatTimingDerivation"; import { createGsapLivePreview } from "./gsapLivePreview"; import { formatTextFieldPreview } from "./propertyPanelSections"; @@ -536,21 +535,3 @@ export function PropertyPanelFlat({ ); } - -/** Chain length at a glance, so the collapsed group says whether anything is on. */ -function audioFxSummary(element: DomEditSelection): string { - const raw = element.dataAttributes?.["fx-chain"]; - const carve = element.dataAttributes?.["fx-carve"]; - let count = 0; - if (raw) { - try { - count = parseAudioFxChain(raw).nodes.filter((n) => n.enabled !== false).length; - } catch { - return "unreadable"; - } - } - const parts: string[] = []; - if (count > 0) parts.push(`${count} effect${count === 1 ? "" : "s"}`); - if (carve) parts.push("carve"); - return parts.length > 0 ? parts.join(" + ") : "none"; -} diff --git a/packages/studio/src/components/editor/audioFxSummary.test.ts b/packages/studio/src/components/editor/audioFxSummary.test.ts new file mode 100644 index 0000000000..5ba7a87a5a --- /dev/null +++ b/packages/studio/src/components/editor/audioFxSummary.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, it } from "vitest"; +import { audioFxSummary } from "./audioFxSummary"; +import type { DomEditSelection } from "./domEditingTypes"; + +const el = (dataAttributes: Record): DomEditSelection => + ({ dataAttributes }) as unknown as DomEditSelection; + +const chain = (nodes: unknown[]) => JSON.stringify({ version: 1, nodes }); + +describe("audioFxSummary", () => { + it("counts a carve as one module, not as the filters behind it", () => { + // Six bands and a level stage reading "7 effects" is the misreading the + // grouping exists to prevent. + const summary = audioFxSummary( + el({ + "fx-chain": chain([ + { type: "peaking", id: "n1", fromCarve: true, params: { frequency: 400 } }, + { type: "peaking", id: "n2", fromCarve: true, params: { frequency: 1600 } }, + { type: "gain", id: "n3", fromCarve: true, params: { gain: -6 } }, + ]), + "fx-carve": JSON.stringify({ source: "vo", strength: 0.25 }), + }), + ); + expect(summary).toBe("carve"); + }); + + it("counts hand-built effects alongside the module", () => { + expect( + audioFxSummary( + el({ + "fx-chain": chain([ + { type: "peaking", id: "n1", fromCarve: true, params: { frequency: 400 } }, + { type: "lowpass", id: "n2", params: { frequency: 8000 } }, + { type: "delay", id: "n3", params: { time: 200 } }, + ]), + }), + ), + ).toBe("2 effects + carve"); + }); + + it("says how many when there is no carve", () => { + expect(audioFxSummary(el({ "fx-chain": chain([{ type: "lowpass", id: "n1" }]) }))).toBe( + "1 effect", + ); + }); + + it("names a carve that is on but has not compiled to filters yet", () => { + // Switching it on with no voice chosen leaves the control in this section with + // nothing behind it; the summary should still say the section holds one. + expect(audioFxSummary(el({ "fx-carve": JSON.stringify({ source: "", strength: 0.25 }) }))).toBe( + "carve", + ); + }); + + it("ignores bypassed effects, as it always did", () => { + expect( + audioFxSummary( + el({ + "fx-chain": chain([ + { type: "lowpass", id: "n1", enabled: false }, + { type: "delay", id: "n2" }, + ]), + }), + ), + ).toBe("1 effect"); + }); + + it("says none for a track with neither", () => { + expect(audioFxSummary(el({}))).toBe("none"); + }); + + it("says so when the chain cannot be read", () => { + expect(audioFxSummary(el({ "fx-chain": "{not json" }))).toBe("unreadable"); + }); +}); diff --git a/packages/studio/src/components/editor/audioFxSummary.ts b/packages/studio/src/components/editor/audioFxSummary.ts new file mode 100644 index 0000000000..441217a8ab --- /dev/null +++ b/packages/studio/src/components/editor/audioFxSummary.ts @@ -0,0 +1,36 @@ +/** + * What the collapsed Audio FX group says it holds. + * + * It has to describe the rack the author would see on opening it, which counts a + * carve as one module rather than as the filters it compiles to. Six peaking + * bands and a level stage reading "7 effects" invited exactly the misreading the + * grouping exists to prevent — that they are seven things to manage. + */ + +import { parseAudioFxChain } from "@hyperframes/core/audio-fx"; +import type { DomEditSelection } from "./domEditingTypes"; + +export function audioFxSummary(element: DomEditSelection): string { + const raw = element.dataAttributes?.["fx-chain"]; + const carveAttr = element.dataAttributes?.["fx-carve"]; + let handBuilt = 0; + let carveNodes = 0; + if (raw) { + try { + for (const node of parseAudioFxChain(raw).nodes) { + if (node.enabled === false) continue; + if (node.fromCarve) carveNodes += 1; + else handBuilt += 1; + } + } catch { + return "unreadable"; + } + } + const parts: string[] = []; + if (handBuilt > 0) parts.push(`${handBuilt} effect${handBuilt === 1 ? "" : "s"}`); + // One name for the module however many filters are behind it. Named when the + // carve is switched on at all, because the control is in this section whether or + // not it has compiled to anything yet. + if (carveNodes > 0 || carveAttr) parts.push("carve"); + return parts.length > 0 ? parts.join(" + ") : "none"; +} diff --git a/packages/studio/src/components/editor/propertyPanelAudioFxGroup.test.tsx b/packages/studio/src/components/editor/propertyPanelAudioFxGroup.test.tsx index de4153689b..9c508668d7 100644 --- a/packages/studio/src/components/editor/propertyPanelAudioFxGroup.test.tsx +++ b/packages/studio/src/components/editor/propertyPanelAudioFxGroup.test.tsx @@ -4,6 +4,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { createRoot } from "react-dom/client"; import { AudioFxGroup } from "./propertyPanelAudioFxGroup.js"; import type { DomEditSelection } from "./domEditingTypes"; +import { liveTime, usePlayerStore } from "../../player"; (globalThis as unknown as { IS_REACT_ACT_ENVIRONMENT: boolean }).IS_REACT_ACT_ENVIRONMENT = true; @@ -19,23 +20,36 @@ afterEach(() => { }); /** - * A selected `