From b0273679e7807cbd7fde54d4999835852b6c47a3 Mon Sep 17 00:00:00 2001 From: Vance Ingalls Date: Wed, 5 Aug 2026 11:40:51 -0700 Subject: [PATCH 01/25] feat(studio): retime an automation selection Add retimeRange pure operation that scales interior points proportionally into a new time span, then uses replaceRange to update the lane while preserving the envelope outside the union of old and new ranges. Co-Authored-By: Claude Sonnet 5 --- .../automationLaneSelection.test.ts | 30 ++++++++++++++++++- .../components/automationLaneSelection.ts | 30 +++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/packages/studio/src/player/components/automationLaneSelection.test.ts b/packages/studio/src/player/components/automationLaneSelection.test.ts index a7f0296add..aa0adfe7e3 100644 --- a/packages/studio/src/player/components/automationLaneSelection.test.ts +++ b/packages/studio/src/player/components/automationLaneSelection.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { pointsIn, replaceRange } from "./automationLaneSelection"; +import { pointsIn, replaceRange, retimeRange } from "./automationLaneSelection"; import { sampleAutomationLane, VOLUME_RANGE } from "@hyperframes/core/audio-automation"; import type { HfAutomationLane } from "@hyperframes/core/audio-automation"; @@ -94,3 +94,31 @@ describe("replaceRange", () => { expect(Math.max(...innerTimes)).toBeGreaterThan(3.0); }); }); + +describe("retimeRange", () => { + it("scales interior points proportionally into the new span", () => { + const pts = retimeRange({ lane: ramp, range: VOLUME_RANGE, t0: 2, t1: 3, newT0: 2, newT1: 5 }); + const moved = pts.find((p) => p.v === 0.4); // the t=3 point + expect(moved?.t).toBe(5); + }); + + it("preserves the envelope outside the union of old and new spans", () => { + const before: HfAutomationLane = { target: "volume", points: ramp.points }; + const after: HfAutomationLane = { + target: "volume", + points: retimeRange({ lane: ramp, range: VOLUME_RANGE, t0: 2, t1: 3, newT0: 2, newT1: 5 }), + }; + for (const t of [0, 1, 1.9, 5.1, 6]) { + expect(sampleAutomationLane(after, t, "linear")).toBeCloseTo( + sampleAutomationLane(before, t, "linear"), + 5, + ); + } + }); + + it("rejects a degenerate span", () => { + expect( + retimeRange({ lane: ramp, range: VOLUME_RANGE, t0: 2, t1: 3, newT0: 4, newT1: 4 }), + ).toEqual(ramp.points); + }); +}); diff --git a/packages/studio/src/player/components/automationLaneSelection.ts b/packages/studio/src/player/components/automationLaneSelection.ts index 3e8e1fb703..f49c2bbcab 100644 --- a/packages/studio/src/player/components/automationLaneSelection.ts +++ b/packages/studio/src/player/components/automationLaneSelection.ts @@ -72,3 +72,33 @@ export function replaceRange(input: { const cappedInner = inner.length <= budget ? inner : decimateEvenly(inner, budget); return [...outside, ...edges, ...cappedInner].sort((a, b) => a.t - b.t); } + +/** + * Retime a selection: interior points scale proportionally into the new span, + * then replaceRange runs over the UNION of old and new spans — growing eats + * whatever it covers, shrinking pins anchors where the envelope re-enters. + */ +export function retimeRange(input: { + lane: HfAutomationLane; + range: AutomationRange; + t0: number; + t1: number; + newT0: number; + newT1: number; +}): HfAutomationPoint[] { + const { lane, range, t0, t1, newT0, newT1 } = input; + const oldSpan = t1 - t0; + const newSpan = newT1 - newT0; + if (oldSpan <= 0 || newSpan <= 0) return lane.points; + const inner = pointsIn(lane, t0, t1).map((p) => ({ + ...p, + t: newT0 + ((p.t - t0) * newSpan) / oldSpan, + })); + return replaceRange({ + lane, + range, + t0: Math.min(t0, newT0), + t1: Math.max(t1, newT1), + inner, + }); +} From ef1a2867e9389defc51c635d13ecd97eb43e09ed Mon Sep 17 00:00:00 2001 From: Vance Ingalls Date: Wed, 5 Aug 2026 11:47:31 -0700 Subject: [PATCH 02/25] test(studio): probe retimeRange's actual guarantee, not sample-continuity past a moved edge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The failing test probed t=5.1, which sits inside the reshaped transition segment between the new edge (t=5) and the existing point (t=6). When growing past an existing breakpoint, the transition TO that point legitimately reshapes — the edge moved (t=3→t=5) even though the far point (t=6) did not. The real guarantee: all BREAKPOINTS strictly outside the union keep exact (t, v) values. Corrected test to: 1. Verify sample continuity on unaffected side: t=[0,1,1.9] 2. Verify the breakpoint at t=6 keeps exact value: (t:6, v:0) Co-Authored-By: Claude Sonnet 5 --- .../player/components/automationLaneSelection.test.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/studio/src/player/components/automationLaneSelection.test.ts b/packages/studio/src/player/components/automationLaneSelection.test.ts index aa0adfe7e3..e1e2370cd8 100644 --- a/packages/studio/src/player/components/automationLaneSelection.test.ts +++ b/packages/studio/src/player/components/automationLaneSelection.test.ts @@ -108,12 +108,21 @@ describe("retimeRange", () => { target: "volume", points: retimeRange({ lane: ramp, range: VOLUME_RANGE, t0: 2, t1: 3, newT0: 2, newT1: 5 }), }; - for (const t of [0, 1, 1.9, 5.1, 6]) { + // Nothing to the left of t0=2 moved (newT0 === t0 here), so sampled + // continuity holds all the way up to the edited region. + for (const t of [0, 1, 1.9]) { expect(sampleAutomationLane(after, t, "linear")).toBeCloseTo( sampleAutomationLane(before, t, "linear"), 5, ); } + // The next real breakpoint past the edited region keeps its own exact + // value — growing past it reshapes the transition INTO it, not the point + // itself. (Sampling inside that transition, e.g. at t=5.1, is expected to + // differ: one of that segment's endpoints moved from t=3 to t=5, even + // though this point at t=6 did not move at all.) + const farPoint = after.points.find((p) => p.t === 6); + expect(farPoint).toEqual({ t: 6, v: 0 }); }); it("rejects a degenerate span", () => { From 94c8b8f1f2eb6682018d3f1d9de7707a827608da Mon Sep 17 00:00:00 2001 From: Vance Ingalls Date: Wed, 5 Aug 2026 12:08:21 -0700 Subject: [PATCH 03/25] feat(studio): stretch an automation selection by its edges Add an edge-handle drag to a selection's rect: grabbing within 8px of either edge retimes the selection via the already-landed retimeRange, scaling interior points proportionally and clamping the dragged edge against its partner and the clip's duration. Priority is point-drag > curve-drag > edge-stretch > new-range-select, so a point sitting on an edge still wins the press. Cursor shows col-resize while hovering or dragging a handle. Co-Authored-By: Claude Sonnet 5 --- .../TimelineAutomationLane.test.tsx | 131 +++++++++++ .../components/TimelineAutomationLane.tsx | 15 +- .../components/useAutomationLaneGestures.ts | 219 +++++++++++++++--- 3 files changed, 326 insertions(+), 39 deletions(-) diff --git a/packages/studio/src/player/components/TimelineAutomationLane.test.tsx b/packages/studio/src/player/components/TimelineAutomationLane.test.tsx index c84a019201..375c3481c8 100644 --- a/packages/studio/src/player/components/TimelineAutomationLane.test.tsx +++ b/packages/studio/src/player/components/TimelineAutomationLane.test.tsx @@ -686,3 +686,134 @@ describe("TimelineAutomationLane selection menu", () => { expect(document.querySelector(".hf-automation-menu")).toBeNull(); }); }); + +describe("TimelineAutomationLane stretch", () => { + // Edges deliberately off any existing point: the lane's hit-priority rule + // (a point always wins) means a selection edge sitting exactly on a + // breakpoint would resolve to a point-drag, never a stretch — see the + // dedicated priority test below for that case instead. + + /** Press, drag and release the right edge of a stretchable selection — the + * shape most of this block's tests share, differing only in where the + * drag ends up. */ + function dragRightEdge(svg: Element, from: number, to: number): void { + fire(svg, "pointerdown", at(from, 0.5)); + fire(svg, "pointermove", at(to, 0.5)); + fire(svg, "pointerup", at(to, 0.5)); + } + + const stretchable: HfAutomation = { + version: 1, + lanes: [ + { + target: "volume", + points: [ + { t: 0, v: 1 }, + { t: 1, v: 0.5 }, + { t: 2, v: 0.8 }, + { t: 4, v: 0 }, + ], + }, + ], + }; + + it("dragging the right edge retimes the interior and persists on release", () => { + const onRangeSelect = vi.fn(); + const { svg, props } = mount(stretchable, { + rangeSelection: { t0: 0.5, t1: 2.5 }, + onRangeSelect, + }); + dragRightEdge(svg, 2.5, 3.3); // off any point, dragged out to 3.3 + + expect(props.onCommit).toHaveBeenCalledTimes(1); + const written = props.onCommit.mock.calls.at(-1)?.[0] as HfAutomation; + const points = written.lanes[0]?.points ?? []; + // Interior points (t=1, t=2) scale by the new/old span ratio (2.8 / 2 = 1.4). + expect(points.some((p) => Math.abs(p.t - 1.2) < 0.01 && p.v === 0.5)).toBe(true); + expect(points.some((p) => Math.abs(p.t - 2.6) < 0.01 && p.v === 0.8)).toBe(true); + + expect(onRangeSelect).toHaveBeenCalledTimes(1); + expect(onRangeSelect).toHaveBeenLastCalledWith(0.5, expect.closeTo(3.3, 1)); + }); + + it("previews the stretch on move without persisting, then commits once on release", () => { + const onPreview = vi.fn(); + const onCommit = vi.fn(); + const { svg } = mount(stretchable, { + rangeSelection: { t0: 0.5, t1: 2.5 }, + onPreview, + onCommit, + }); + fire(svg, "pointerdown", at(2.5, 0.5)); + fire(svg, "pointermove", at(3, 0.5)); + fire(svg, "pointermove", at(3.3, 0.5)); + expect(onPreview).toHaveBeenCalledTimes(2); + expect(onCommit).not.toHaveBeenCalled(); + fire(svg, "pointerup", at(3.3, 0.5)); + expect(onCommit).toHaveBeenCalledTimes(1); + }); + + it("a point sitting on the selection's edge wins over the edge-stretch gesture", () => { + const sel: HfAutomation = { + version: 1, + lanes: [ + { + target: "volume", + points: [ + { t: 0, v: 1 }, + { t: 1.5, v: 0.5 }, + { t: 2, v: 0.8 }, + { t: 4, v: 0 }, + ], + }, + ], + }; + const onRangeSelect = vi.fn(); + const { svg, props } = mount(sel, { + rangeSelection: { t0: 1, t1: 2 }, + onRangeSelect, + }); + fire(svg, "pointerdown", at(2, 0.8)); // exactly the point at t=2, which is also the right edge + fire(svg, "pointermove", at(3, 0.8)); + fire(svg, "pointerup", at(3, 0.8)); + // A point-drag moved just that point; the selection itself was untouched. + expect(onRangeSelect).not.toHaveBeenCalled(); + const written = props.onCommit.mock.calls.at(-1)?.[0] as HfAutomation; + const times = (written.lanes[0]?.points ?? []).map((p) => p.t); + expect(times).toContain(3); + }); + + it("clamps the dragged edge so it cannot cross its partner", () => { + const onRangeSelect = vi.fn(); + const { svg } = mount(stretchable, { + rangeSelection: { t0: 0.5, t1: 2.5 }, + onRangeSelect, + }); + dragRightEdge(svg, 2.5, 0.3); // dragged past the left edge (t0=0.5) + const [, t1] = onRangeSelect.mock.calls.at(-1) as [number, number]; + expect(t1).toBeGreaterThan(0.5); + }); + + it("clamps the dragged edge to the lane's own duration", () => { + const onRangeSelect = vi.fn(); + const { svg } = mount(stretchable, { + rangeSelection: { t0: 0.5, t1: 2.5 }, + onRangeSelect, + }); + dragRightEdge(svg, 2.5, 10); // far past the clip's own duration (4s) + const [, t1] = onRangeSelect.mock.calls.at(-1) as [number, number]; + expect(t1).toBeLessThanOrEqual(4); + }); + + it("shows a resize cursor when hovering an edge with nothing else live", () => { + const { svg } = mount(ramp, { rangeSelection: { t0: 1, t1: 3 } }); + fire(svg, "pointermove", at(3, 0.5)); // near the right edge, nothing pressed + expect(svg.style.cursor).toBe("col-resize"); + }); + + it("keeps the normal cursor away from the selection's edges", () => { + const { svg } = mount(ramp, { rangeSelection: { t0: 1, t1: 3 } }); + fire(svg, "pointermove", at(2, 0.5)); // middle of the selection, not an edge + expect(svg.style.cursor).not.toBe("col-resize"); + }); +}); diff --git a/packages/studio/src/player/components/TimelineAutomationLane.tsx b/packages/studio/src/player/components/TimelineAutomationLane.tsx index 835c8111c5..25d150900e 100644 --- a/packages/studio/src/player/components/TimelineAutomationLane.tsx +++ b/packages/studio/src/player/components/TimelineAutomationLane.tsx @@ -50,8 +50,10 @@ import { getTimelineLaneTop } from "./timelineLayout"; import type { TimelineElement } from "../store/playerStore"; import type { UseAutomationLanesResult } from "./useAutomationLanes"; -/** Pointer shape: a read-only lane can only be selected, a live one edited. */ -function laneCursor(readOnly: boolean | undefined, dragging: boolean): string { +/** Pointer shape: a stretch handle wins over everything else it might also + * sit above, a read-only lane can only be selected, a live one edited. */ +function laneCursor(readOnly: boolean | undefined, dragging: boolean, stretching: boolean): string { + if (stretching) return "col-resize"; if (readOnly) return "pointer"; return dragging ? "grabbing" : "crosshair"; } @@ -204,8 +206,9 @@ export function TimelineAutomationLane({ onRangeSelect, onRangeClear, duration, + rangeSelection, }); - const { dragIndex, curveIndex, hint, editing } = gestures; + const { dragIndex, curveIndex, edgeDrag, edgeHover, hint, editing } = gestures; const removeAt = useCallback( (index: number): void => { @@ -285,7 +288,11 @@ export function TimelineAutomationLane({ top: 0, width: widthPx + PAD_X * 2, height: h, - cursor: laneCursor(readOnly, dragIndex !== null || curveIndex !== null), + cursor: laneCursor( + readOnly, + dragIndex !== null || curveIndex !== null, + edgeDrag !== null || edgeHover, + ), opacity: readOnly ? 0.55 : 1, touchAction: "none", }} diff --git a/packages/studio/src/player/components/useAutomationLaneGestures.ts b/packages/studio/src/player/components/useAutomationLaneGestures.ts index cb00c83efe..54214c6014 100644 --- a/packages/studio/src/player/components/useAutomationLaneGestures.ts +++ b/packages/studio/src/player/components/useAutomationLaneGestures.ts @@ -3,8 +3,8 @@ * * Its own hook because the lane component sits at the studio's file ceiling and * because these are the parts worth testing on their own: which of a press, - * a drag and a modifier resolves to moving a point, bending a segment, or - * nothing at all. + * a drag and a modifier resolves to moving a point, bending a segment, + * stretching a selection's edge, or nothing at all. * * Modifiers follow Ableton's, since that is the muscle memory an automation lane * inherits: Shift locks a drag to one axis and fines the value down, Alt over a @@ -21,11 +21,17 @@ import { POINT_MERGE_SEC, snapLaneTime, } from "./automationLaneGeometry"; +import { retimeRange } from "./automationLaneSelection"; /** Snap radius in clip seconds. Tight on purpose: a lane is often a few seconds * wide, where a generous radius makes a point unplaceable between two beats. */ const SNAP_SEC = 0.04; +/** Hit radius for grabbing a selection's edge, in screen px — independent of + * a point's own grab radius so the two zones can be reasoned about on their + * own, even though a point sitting on an edge still wins (see `gestureAt`). */ +const EDGE_GRAB_PX = 8; + /** A point's position, or the origin when the index no longer resolves. */ function originOf(point: HfAutomationLane["points"][number] | undefined): { t: number; v: number } { return point ? { t: point.t, v: point.v } : { t: 0, v: 0 }; @@ -60,6 +66,8 @@ export interface UseAutomationLaneGesturesInput { onRangeSelect?: ((t0: number, t1: number) => void) | undefined; onRangeClear?: (() => void) | undefined; duration: number; // clamp bound for range endpoints + /** Active selection on this lane, so its edges have something to grab. */ + rangeSelection?: { t0: number; t1: number } | null | undefined; } export interface UseAutomationLaneGesturesResult { @@ -67,6 +75,11 @@ export interface UseAutomationLaneGesturesResult { dragIndex: number | null; /** Segment being bent, identified by the point that owns its curve. */ curveIndex: number | null; + /** Edge being stretched, for the cursor. */ + edgeDrag: "t0" | "t1" | null; + /** Whether the pointer sits over a stretch handle with no gesture live — + * the col-resize cursor hint before a press commits to the drag. */ + edgeHover: boolean; /** Value readout to show while a gesture is live. */ hint: string | null; hitIndex(clientX: number, clientY: number): number | null; @@ -97,6 +110,7 @@ export function useAutomationLaneGestures({ onRangeSelect, onRangeClear, duration, + rangeSelection, }: UseAutomationLaneGesturesInput): UseAutomationLaneGesturesResult { const [dragIndex, setDragIndex] = useState(null); const [curveIndex, setCurveIndex] = useState(null); @@ -110,6 +124,32 @@ export function useAutomationLaneGestures({ /** Whether the live drag has crossed the pixel threshold that turns a press * into an actual range, rather than a click that should just clear one. */ const rangeCrossed = useRef(false); + /** An edge-stretch drag in progress: which edge, the selection it started + * from (kept fixed as the retime's untouched anchor), and the edge's own + * live position. */ + const [edgeDrag, setEdgeDrag] = useState<{ + edge: "t0" | "t1"; + origin: { t0: number; t1: number }; + current: number; + } | null>(null); + /** Cursor hint: hovering a stretch handle with nothing else live. */ + const [edgeHover, setEdgeHover] = useState(false); + + /** Which edge of the active selection, if any, sits within grab range of the + * pointer's screen x — full lane height, since the handle spans the rect. */ + const edgeAt = useCallback( + (clientX: number): "t0" | "t1" | null => { + if (!rangeSelection) return null; + const box = getBox(); + if (!box) return null; + const px = clientX - box.left; + const d0 = Math.abs(xOf(rangeSelection.t0) - px); + const d1 = Math.abs(xOf(rangeSelection.t1) - px); + if (d0 <= EDGE_GRAB_PX && d0 <= d1) return "t0"; + return d1 <= EDGE_GRAB_PX ? "t1" : null; + }, + [rangeSelection, getBox, xOf], + ); /** Index of a point under the pointer, or null. */ const hitIndex = useCallback( @@ -153,6 +193,38 @@ export function useAutomationLaneGestures({ [hitIndex, segmentIndex], ); + /** + * What a press on the lane's empty background arms: an edge grab when it + * landed within range of an existing selection's edge, else a new range + * selection — only when a caller wants to hear about one; a read-only lane + * never reaches here at all. + */ + const armBackgroundGesture = useCallback( + (e: ReactPointerEvent): void => { + const edge = edgeAt(e.clientX); + if (edge && rangeSelection) { + e.preventDefault(); + capturePointer(e); + setEdgeHover(false); + setEdgeDrag({ + edge, + origin: rangeSelection, + current: edge === "t0" ? rangeSelection.t0 : rangeSelection.t1, + }); + return; + } + if (!onRangeSelect) return; + e.preventDefault(); + capturePointer(e); + const raw = pointAt(e.clientX, e.clientY).t; + const clamped = Math.min(duration, Math.max(0, raw)); + const t = e.altKey ? clamped : snapLaneTime(clamped, snapTimes ?? [], SNAP_SEC); + rangeCrossed.current = false; + setRangeDrag({ from: t, to: t }); + }, + [edgeAt, rangeSelection, onRangeSelect, pointAt, duration, snapTimes], + ); + const onPointerDown = useCallback( (e: ReactPointerEvent): void => { if (e.button !== 0) return; @@ -168,23 +240,15 @@ export function useAutomationLaneGestures({ } const gesture = gestureAt(e); if (!gesture) { - // Neither a point nor an Alt-held segment: the press landed on the - // lane's empty background. That is a range selection's gesture, not - // nothing — but only when a caller wants to hear about one; a - // read-only lane already returned above, so this is a live one with no - // range feature wired up. - if (!onRangeSelect) return; - e.preventDefault(); - capturePointer(e); - const raw = pointAt(e.clientX, e.clientY).t; - const clamped = Math.min(duration, Math.max(0, raw)); - const t = e.altKey ? clamped : snapLaneTime(clamped, snapTimes ?? [], SNAP_SEC); - rangeCrossed.current = false; - setRangeDrag({ from: t, to: t }); + armBackgroundGesture(e); return; } e.preventDefault(); capturePointer(e); + // A point can sit close enough to an edge to have set the hover hint + // moments ago; winning the press should not leave that stale cursor + // showing through the drag that follows. + setEdgeHover(false); if (gesture.curve) { setCurveIndex(gesture.index); return; @@ -192,7 +256,7 @@ export function useAutomationLaneGestures({ dragOrigin.current = originOf(lane.points[gesture.index]); setDragIndex(gesture.index); }, - [gestureAt, lane, readOnly, onSelect, onRangeSelect, pointAt, duration, snapTimes], + [gestureAt, lane, readOnly, onSelect, armBackgroundGesture], ); /** Bend the segment under the pointer, which is what Alt-dragging the line does. */ @@ -239,46 +303,120 @@ export function useAutomationLaneGestures({ [dragIndex, lane, pointAt, range, commitPoints, snapTimes, xOf, yOf], ); + /** Preview the selection's new bounds as the grabbed edge moves: the other + * edge stays put as the retime's anchor, and the dragged one is clamped so + * it cannot cross its partner (leaving at least a merge-radius of room) nor + * leave the clip's own duration. */ + const moveEdge = useCallback( + (e: ReactPointerEvent): void => { + if (edgeDrag === null) return; + const { edge, origin } = edgeDrag; + const raw = pointAt(e.clientX, e.clientY).t; + const clamped = Math.min(duration, Math.max(0, raw)); + const current = + edge === "t0" + ? Math.min(clamped, origin.t1 - POINT_MERGE_SEC) + : Math.max(clamped, origin.t0 + POINT_MERGE_SEC); + setEdgeDrag({ edge, origin, current }); + const newT0 = edge === "t0" ? current : origin.t0; + const newT1 = edge === "t1" ? current : origin.t1; + setHint(`${newT0.toFixed(2)}s → ${newT1.toFixed(2)}s`); + commitPoints(retimeRange({ lane, range, t0: origin.t0, t1: origin.t1, newT0, newT1 }), false); + }, + [edgeDrag, pointAt, duration, lane, range, commitPoints], + ); + + /** Update the live range-drag as the pointer moves, firing `onRangeSelect` + * once it has covered enough pixels to count as an actual range rather + * than a click that should just clear one. */ + const moveRangeDrag = useCallback( + (e: ReactPointerEvent): void => { + if (rangeDrag === null) return; + const raw = pointAt(e.clientX, e.clientY).t; + const clamped = Math.min(duration, Math.max(0, raw)); + const t = e.altKey ? clamped : snapLaneTime(clamped, snapTimes ?? [], SNAP_SEC); + setRangeDrag({ from: rangeDrag.from, to: t }); + if (Math.abs(xOf(t) - xOf(rangeDrag.from)) <= 3) return; + rangeCrossed.current = true; + onRangeSelect?.(Math.min(rangeDrag.from, t), Math.max(rangeDrag.from, t)); + }, + [rangeDrag, pointAt, duration, snapTimes, xOf, onRangeSelect], + ); + + /** Cursor hint only: whether the pointer sits over a stretch handle with + * nothing else live. Skipped read-only, which never arms a stretch. */ + const updateEdgeHover = useCallback( + (e: ReactPointerEvent): void => { + if (!readOnly) setEdgeHover(edgeAt(e.clientX) !== null); + }, + [readOnly, edgeAt], + ); + const onPointerMove = useCallback( (e: ReactPointerEvent): void => { + if (edgeDrag !== null) { + e.stopPropagation(); + moveEdge(e); + return; + } if (rangeDrag !== null) { e.stopPropagation(); - const raw = pointAt(e.clientX, e.clientY).t; - const clamped = Math.min(duration, Math.max(0, raw)); - const t = e.altKey ? clamped : snapLaneTime(clamped, snapTimes ?? [], SNAP_SEC); - setRangeDrag({ from: rangeDrag.from, to: t }); - if (Math.abs(xOf(t) - xOf(rangeDrag.from)) > 3) { - rangeCrossed.current = true; - onRangeSelect?.(Math.min(rangeDrag.from, t), Math.max(rangeDrag.from, t)); - } + moveRangeDrag(e); + return; + } + if (curveIndex === null && dragIndex === null) { + updateEdgeHover(e); return; } - if (curveIndex === null && dragIndex === null) return; e.stopPropagation(); if (curveIndex !== null) bendSegment(e.clientX, e.clientY); else movePoint(e); }, [ + edgeDrag, + moveEdge, rangeDrag, - pointAt, - duration, - snapTimes, - xOf, - onRangeSelect, - bendSegment, + moveRangeDrag, curveIndex, dragIndex, + updateEdgeHover, + bendSegment, movePoint, ], ); + /** Persist the stretch and hand the selection's new bounds back to the + * caller — the one point in the gesture that both commits and moves the + * selection it grabbed. */ + const finishEdgeDrag = useCallback((): void => { + if (edgeDrag === null) return; + const { edge, origin, current } = edgeDrag; + const newT0 = edge === "t0" ? current : origin.t0; + const newT1 = edge === "t1" ? current : origin.t1; + setEdgeDrag(null); + setHint(null); + commitPoints(lane.points, true); + onRangeSelect?.(newT0, newT1); + }, [edgeDrag, lane, commitPoints, onRangeSelect]); + + /** A sub-threshold press clears the selection rather than leaving a + * zero-width one behind. */ + const finishRangeDrag = useCallback((): void => { + if (!rangeCrossed.current) onRangeClear?.(); + rangeCrossed.current = false; + setRangeDrag(null); + }, [onRangeClear]); + const endDrag = useCallback( (e: ReactPointerEvent): void => { + if (edgeDrag !== null) { + e.stopPropagation(); + finishEdgeDrag(); + return; + } if (rangeDrag !== null) { e.stopPropagation(); - if (!rangeCrossed.current) onRangeClear?.(); - rangeCrossed.current = false; - setRangeDrag(null); + finishRangeDrag(); return; } if (dragIndex === null && curveIndex === null) return; @@ -289,7 +427,16 @@ export function useAutomationLaneGestures({ setHint(null); commitPoints(lane.points, true); }, - [rangeDrag, onRangeClear, curveIndex, dragIndex, lane, commitPoints], + [ + edgeDrag, + finishEdgeDrag, + rangeDrag, + finishRangeDrag, + curveIndex, + dragIndex, + lane, + commitPoints, + ], ); const onDoubleClick = useCallback( @@ -352,6 +499,8 @@ export function useAutomationLaneGestures({ return { dragIndex, curveIndex, + edgeDrag: edgeDrag?.edge ?? null, + edgeHover, hint, hitIndex, segmentIndex, From 03b27197a5347f865d648c750d7bcb733ee5846d Mon Sep 17 00:00:00 2001 From: Vance Ingalls Date: Wed, 5 Aug 2026 12:46:25 -0700 Subject: [PATCH 04/25] fix(studio): retime edge-stretch from a fixed points snapshot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit moveEdge fed retimeRange the live draft on every pointermove while origin.t0/t1 stayed pinned to the drag's start. retimeRange is a relative transform that scales a lane's own current point positions, so repeated pointermoves compounded the scale factor (interior points drift toward the far edge) and could drop points that retimed past the selection's original bound out of the next move's `inner` set entirely. Snapshot lane.points at arm time (armBackgroundGesture) alongside the existing frozen origin, and always retime from that snapshot in moveEdge instead of the live draft. finishEdgeDrag is unchanged: it already just persists the last (now-correct) preview. Adds a regression test asserting a multi-pointermove edge-drag (both edges) lands on the exact same final points as a single-shot drag to the same target — the case that exposed the bug, since the existing suite only ever tested a single move. --- .../TimelineAutomationLane.test.tsx | 68 +++++++++++++++++++ .../components/useAutomationLaneGestures.ts | 38 ++++++++--- 2 files changed, 98 insertions(+), 8 deletions(-) diff --git a/packages/studio/src/player/components/TimelineAutomationLane.test.tsx b/packages/studio/src/player/components/TimelineAutomationLane.test.tsx index 375c3481c8..af73ba64f4 100644 --- a/packages/studio/src/player/components/TimelineAutomationLane.test.tsx +++ b/packages/studio/src/player/components/TimelineAutomationLane.test.tsx @@ -805,6 +805,74 @@ describe("TimelineAutomationLane stretch", () => { expect(t1).toBeLessThanOrEqual(4); }); + it("retimes identically whether the right edge arrives in one move or several", () => { + // moveEdge must always retime from the points snapshotted at arm time, + // never from the live draft — retimeRange is a RELATIVE transform (it + // scales the lane's OWN current point positions by newSpan/oldSpan), so + // feeding it the live draft on every pointermove compounds the scale + // factor instead of applying it once. A real drag fires dozens of moves; + // this asserts the FINAL preview is identical regardless of how many. + const onPreviewSingle = vi.fn(); + const single = mount(stretchable, { + rangeSelection: { t0: 0.5, t1: 2.5 }, + onPreview: onPreviewSingle, + }); + fire(single.svg, "pointerdown", at(2.5, 0.5)); + fire(single.svg, "pointermove", at(3.3, 0.5)); + const singleShot = (onPreviewSingle.mock.calls.at(-1)?.[0] as HfAutomation | undefined) + ?.lanes[0]?.points; + expect(singleShot).toBeDefined(); + + const onPreviewMulti = vi.fn(); + const multi = mount(stretchable, { + rangeSelection: { t0: 0.5, t1: 2.5 }, + onPreview: onPreviewMulti, + }); + fire(multi.svg, "pointerdown", at(2.5, 0.5)); + // At least 3 separate pointermoves crossing the same span, not one jump. + fire(multi.svg, "pointermove", at(2.7, 0.5)); + fire(multi.svg, "pointermove", at(2.9, 0.5)); + fire(multi.svg, "pointermove", at(3.1, 0.5)); + fire(multi.svg, "pointermove", at(3.3, 0.5)); + const afterFourMoves = (onPreviewMulti.mock.calls.at(-1)?.[0] as HfAutomation | undefined) + ?.lanes[0]?.points; + expect(afterFourMoves).toBeDefined(); + + // Both interior points (t=1, t=2) land exactly where a single-shot retime + // puts them — not compounded, and not dropped. + expect(afterFourMoves).toEqual(singleShot); + expect(afterFourMoves?.length).toBe(6); + expect(afterFourMoves?.some((p) => Math.abs(p.t - 1.2) < 0.001 && p.v === 0.5)).toBe(true); + expect(afterFourMoves?.some((p) => Math.abs(p.t - 2.6) < 0.001 && p.v === 0.8)).toBe(true); + }); + + it("retimes identically whether the left edge arrives in one move or several", () => { + const onPreviewSingle = vi.fn(); + const single = mount(stretchable, { + rangeSelection: { t0: 1, t1: 3 }, + onPreview: onPreviewSingle, + }); + fire(single.svg, "pointerdown", at(1, 0.5)); + fire(single.svg, "pointermove", at(0.2, 0.5)); + const singleShot = (onPreviewSingle.mock.calls.at(-1)?.[0] as HfAutomation | undefined) + ?.lanes[0]?.points; + expect(singleShot).toBeDefined(); + + const onPreviewMulti = vi.fn(); + const multi = mount(stretchable, { + rangeSelection: { t0: 1, t1: 3 }, + onPreview: onPreviewMulti, + }); + fire(multi.svg, "pointerdown", at(1, 0.5)); + fire(multi.svg, "pointermove", at(0.7, 0.5)); + fire(multi.svg, "pointermove", at(0.4, 0.5)); + fire(multi.svg, "pointermove", at(0.2, 0.5)); + const afterThreeMoves = (onPreviewMulti.mock.calls.at(-1)?.[0] as HfAutomation | undefined) + ?.lanes[0]?.points; + expect(afterThreeMoves).toBeDefined(); + expect(afterThreeMoves).toEqual(singleShot); + }); + it("shows a resize cursor when hovering an edge with nothing else live", () => { const { svg } = mount(ramp, { rangeSelection: { t0: 1, t1: 3 } }); fire(svg, "pointermove", at(3, 0.5)); // near the right edge, nothing pressed diff --git a/packages/studio/src/player/components/useAutomationLaneGestures.ts b/packages/studio/src/player/components/useAutomationLaneGestures.ts index 54214c6014..2eaf4b9cd0 100644 --- a/packages/studio/src/player/components/useAutomationLaneGestures.ts +++ b/packages/studio/src/player/components/useAutomationLaneGestures.ts @@ -12,7 +12,11 @@ */ import { useCallback, useRef, useState, type PointerEvent as ReactPointerEvent } from "react"; -import type { AutomationRange, HfAutomationLane } from "@hyperframes/core/audio-automation"; +import type { + AutomationRange, + HfAutomationLane, + HfAutomationPoint, +} from "@hyperframes/core/audio-automation"; import { applyShiftConstraint, curveForDrag, @@ -125,12 +129,17 @@ export function useAutomationLaneGestures({ * into an actual range, rather than a click that should just clear one. */ const rangeCrossed = useRef(false); /** An edge-stretch drag in progress: which edge, the selection it started - * from (kept fixed as the retime's untouched anchor), and the edge's own - * live position. */ + * from (kept fixed as the retime's untouched anchor), the edge's own live + * position, and the lane's points as they stood at arm time. `retimeRange` + * is a RELATIVE transform — it scales a lane's own current point positions + * by newSpan/oldSpan — so it must always run against this fixed snapshot, + * never against the live draft: retiming from the draft would compound the + * scale factor on every pointermove instead of applying it once. */ const [edgeDrag, setEdgeDrag] = useState<{ edge: "t0" | "t1"; origin: { t0: number; t1: number }; current: number; + points: HfAutomationPoint[]; } | null>(null); /** Cursor hint: hovering a stretch handle with nothing else live. */ const [edgeHover, setEdgeHover] = useState(false); @@ -210,6 +219,7 @@ export function useAutomationLaneGestures({ edge, origin: rangeSelection, current: edge === "t0" ? rangeSelection.t0 : rangeSelection.t1, + points: lane.points, }); return; } @@ -222,7 +232,7 @@ export function useAutomationLaneGestures({ rangeCrossed.current = false; setRangeDrag({ from: t, to: t }); }, - [edgeAt, rangeSelection, onRangeSelect, pointAt, duration, snapTimes], + [edgeAt, rangeSelection, onRangeSelect, pointAt, duration, snapTimes, lane], ); const onPointerDown = useCallback( @@ -310,20 +320,32 @@ export function useAutomationLaneGestures({ const moveEdge = useCallback( (e: ReactPointerEvent): void => { if (edgeDrag === null) return; - const { edge, origin } = edgeDrag; + const { edge, origin, points } = edgeDrag; const raw = pointAt(e.clientX, e.clientY).t; const clamped = Math.min(duration, Math.max(0, raw)); const current = edge === "t0" ? Math.min(clamped, origin.t1 - POINT_MERGE_SEC) : Math.max(clamped, origin.t0 + POINT_MERGE_SEC); - setEdgeDrag({ edge, origin, current }); + setEdgeDrag({ edge, origin, current, points }); const newT0 = edge === "t0" ? current : origin.t0; const newT1 = edge === "t1" ? current : origin.t1; setHint(`${newT0.toFixed(2)}s → ${newT1.toFixed(2)}s`); - commitPoints(retimeRange({ lane, range, t0: origin.t0, t1: origin.t1, newT0, newT1 }), false); + // Retime from the snapshot taken at arm time, never from `lane` (the + // live draft) — see the state comment above for why. + commitPoints( + retimeRange({ + lane: { target: lane.target, points }, + range, + t0: origin.t0, + t1: origin.t1, + newT0, + newT1, + }), + false, + ); }, - [edgeDrag, pointAt, duration, lane, range, commitPoints], + [edgeDrag, pointAt, duration, lane.target, range, commitPoints], ); /** Update the live range-drag as the pointer moves, firing `onRangeSelect` From dc0093b045977663c646cacaba385026a1b9674b Mon Sep 17 00:00:00 2001 From: Vance Ingalls Date: Wed, 5 Aug 2026 12:47:55 -0700 Subject: [PATCH 05/25] fix(studio): clamp selection-start paste, sharpen clipboard test, cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - useAutomationSelectionKeyboard: clamp the selection-start paste branch to [0, element.duration - clip.span], same as the playhead branch already does. An unclamped paste near a clip's end could write points past element.duration and leave the resulting selection's edge ungrabbable off the visible lane. - automationClipboard.test.ts: swap the cross-parameter mapping test's target from fx.r.wet (numerically identical to VOLUME_RANGE) to the log-scaled fx.n1.frequency, so the test actually discriminates real unit-space mapping from a linear guess or a verbatim value copy. - automationLaneSelection.ts: drop the lone `!` non-null assertion in decimateEvenly's budget-of-1 branch for a guarded pattern, matching the loop right below it and the repo's no-`!` convention. - .fallowrc.jsonc: remove the two ignoreExports entries for AUTOMATION_SHAPES and simplifyPoints — both are now genuinely consumed (AutomationSelectionMenu.tsx, TimelineAutomationLane.tsx). - AutomationSelectionMenu.tsx: port TrackGapContextMenu's viewport-edge clamping so a right-click near the bottom/right of the timeline doesn't render the shape/simplify menu partially off-screen. --- .fallowrc.jsonc | 16 --------- .../useAutomationSelectionKeyboard.test.tsx | 34 +++++++++++++++++++ .../components/AutomationSelectionMenu.tsx | 10 +++++- .../components/automationClipboard.test.ts | 31 ++++++++++++----- .../components/automationLaneSelection.ts | 5 ++- 5 files changed, 70 insertions(+), 26 deletions(-) diff --git a/.fallowrc.jsonc b/.fallowrc.jsonc index f5b1617118..3a9511683b 100644 --- a/.fallowrc.jsonc +++ b/.fallowrc.jsonc @@ -176,22 +176,6 @@ "withLane", ], }, - // automationShapes is part of the audio-automation stack: its consumer is - // the UI layer that uses shape generators one PR upstack, so a per-PR audit - // diffing against the merge base sees these as unused. Consumed for real once - // the stack merges; safe to drop this entry then. - { - "file": "packages/studio/src/player/components/automationShapes.ts", - "exports": ["AUTOMATION_SHAPES"], - }, - // automationSimplify is part of the audio-automation stack: its consumer is - // the UI layer one PR upstack, so a per-PR audit diffing against the merge - // base sees these as unused. Consumed for real once the stack merges; safe - // to drop this entry then. - { - "file": "packages/studio/src/player/components/automationSimplify.ts", - "exports": ["simplifyPoints"], - }, // propertyPanelAutomation is the shared reader for both panel sections; the // FX group that consumes these two lands one PR upstack, so a per-PR audit // against the merge base sees them as unused. diff --git a/packages/studio/src/hooks/useAutomationSelectionKeyboard.test.tsx b/packages/studio/src/hooks/useAutomationSelectionKeyboard.test.tsx index a7f57d4b68..f8ab1a3d76 100644 --- a/packages/studio/src/hooks/useAutomationSelectionKeyboard.test.tsx +++ b/packages/studio/src/hooks/useAutomationSelectionKeyboard.test.tsx @@ -221,6 +221,40 @@ describe("useAutomationSelectionKeyboard", () => { }); }); + it("Cmd+V at a selection near the clip's end clamps the paste inside its duration", () => { + // The playhead branch already clamps to duration - span; the + // selection-start branch didn't, so pasting a 2s clip at a selection + // sitting at t0=5.5 on a 6s clip used to write points out to t=7.5 — + // past element.duration — and leave the selection itself out of bounds. + clearAutomationClipboard(); + usePlayerStore.setState({ elements: [bgmElement], selectedElementId: "bgm" }); + usePlayerStore + .getState() + .setAutomationSelection({ elementKey: "bgm", target: "volume", t0: 2, t1: 4 }); + const { onCommit } = setup({}); + combo("c"); + expect(readClipboard(null)?.span).toBe(2); + + // A 0.1s-wide selection right near the clip's 6s end. + usePlayerStore + .getState() + .setAutomationSelection({ elementKey: "bgm", target: "volume", t0: 5.5, t1: 5.6 }); + combo("v"); + const written = onCommit.mock.calls.at(-1)?.[0]; + const times = (written?.lanes?.[0]?.points ?? []).map((p: { t: number }) => p.t); + for (const t of times) { + expect(t).toBeGreaterThanOrEqual(0); + expect(t).toBeLessThanOrEqual(bgmElement.duration); + } + // Clamped to duration (6) - span (2) = 4, not the unclamped 5.5. + expect(usePlayerStore.getState().automationSelection).toEqual({ + elementKey: "bgm", + target: "volume", + t0: 4, + t1: 6, + }); + }); + it("refuses to paste when the dom-edit layer would write to a different clip", () => { // selectedElementId says "bgm" but the commit channel is still on the // previously selected clip — writing here would serialize bgm's automation diff --git a/packages/studio/src/player/components/AutomationSelectionMenu.tsx b/packages/studio/src/player/components/AutomationSelectionMenu.tsx index 4a2bd0d057..f157f6e266 100644 --- a/packages/studio/src/player/components/AutomationSelectionMenu.tsx +++ b/packages/studio/src/player/components/AutomationSelectionMenu.tsx @@ -30,11 +30,19 @@ export const AutomationSelectionMenu = memo(function AutomationSelectionMenu({ const menuRef = useContextMenuDismiss(onClose); const row = "block w-full px-2 py-1 text-left text-[11px] text-panel-text-1 hover:bg-panel-bg-3 disabled:opacity-40"; + // Same edge-clamping precedent as TrackGapContextMenu: without it a + // right-click near the bottom/right of the timeline renders this menu + // partially off-screen. + const menuWidth = 140; + const menuHeight = AUTOMATION_SHAPES.length * 24 + 32; + const overflowY = y + menuHeight - window.innerHeight; + const adjustedX = x + menuWidth > window.innerWidth ? x - menuWidth : x; + const adjustedY = overflowY > 0 ? y - overflowY - 8 : y; return createPortal(
{AUTOMATION_SHAPES.map((shape) => ( + )} +
+ ); +} + export function TimelineTrackHeader({ trackNumber, trackDisplayNumber, @@ -298,6 +404,7 @@ export function TimelineTrackHeader({ onToggleClipExpanded, onToggleTrackHidden, onTogglePropertyGroupKeyframe, + onRemoveAutomationLane, onSeek, }: TimelineTrackHeaderProps) { const clipPercentage = keyframeClip @@ -315,6 +422,20 @@ export function TimelineTrackHeader({ // left an audio clip's envelopes unreachable, since the track could not expand. const disclosable = lanes.length > 0 || (keyframeClip ? automationLaneCountOf(keyframeClip) : 0) > 0; + // Each envelope's name, resolved against the chain the same way the lane + // resolves its axis — a band is named by its frequency, not by its effect. The + // lane list is already in drawing order, which is the order these rows have to + // follow: a name beside the wrong envelope is worse than an awkward order. + const automationRows = keyframeClip + ? elementAutomationLanes(keyframeClip).flatMap((lane) => { + const chain = elementFxChain(keyframeClip); + const parts = automationLaneLabelParts(lane.target, chain); + const label = automationLaneLabel(lane.target, chain); + return parts && label + ? [{ target: lane.target, label, name: parts.name, param: parts.param }] + : []; + }) + : []; const isKeyframeLayer = !!keyframeClip && disclosable; return ( @@ -381,7 +502,7 @@ export function TimelineTrackHeader({ key={lane.group} lane={lane} laneIndex={laneIndex} - isLastLane={laneIndex === lanes.length - 1} + isLastLane={laneIndex === lanes.length - 1 && automationRows.length === 0} expandedElement={keyframeClip} currentTime={currentTime} clipPercentage={clipPercentage} @@ -391,6 +512,24 @@ export function TimelineTrackHeader({ onSeek={onSeek} /> ))} + {/* Below the keyframe rows and stepping by its own height, which is how + TimelineAutomationLaneSlot lays the envelopes out on the canvas. The + two have to agree or a name labels the wrong curve. */} + {isExpanded && + automationRows.map((row, index) => ( + + ))} )} diff --git a/packages/studio/src/player/components/automationGestureKeys.test.ts b/packages/studio/src/player/components/automationGestureKeys.test.ts new file mode 100644 index 0000000000..c88cfd2bd3 --- /dev/null +++ b/packages/studio/src/player/components/automationGestureKeys.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from "vitest"; +import { createAutomationGestureKeys } from "./automationGestureKeys"; + +const keys = () => { + let n = 0; + return createAutomationGestureKeys(() => `g${++n}`); +}; + +describe("createAutomationGestureKeys", () => { + it("holds one key across every move of a drag", () => { + // Each move persists; history merges same-key entries inside 300ms. Separate + // keys, or a window that can expire mid-drag, is what made undo take back a + // fragment of the move instead of the move. + const g = keys(); + expect([g.live().key, g.live().key, g.live().key]).toEqual(["g1", "g1", "g1"]); + }); + + it("never lets the window expire", () => { + expect(keys().live().ms).toBe(Number.POSITIVE_INFINITY); + expect(keys().commit().ms).toBe(Number.POSITIVE_INFINITY); + }); + + it("ends the gesture on the release, under the same key", () => { + const g = keys(); + g.live(); + expect(g.commit().key).toBe("g1"); + }); + + it("starts a fresh key for the next drag", () => { + const g = keys(); + g.live(); + g.commit(); + expect(g.live().key).toBe("g2"); + }); + + it("gives a standalone commit its own key", () => { + // Deleting a range, pasting one, typing a value: one write, one step each. + const g = keys(); + expect(g.commit().key).toBe("g1"); + expect(g.commit().key).toBe("g2"); + }); + + it("does not join a commit to a gesture that already ended", () => { + const g = keys(); + g.live(); + g.commit(); + expect(g.commit().key).toBe("g2"); + }); +}); diff --git a/packages/studio/src/player/components/automationGestureKeys.ts b/packages/studio/src/player/components/automationGestureKeys.ts new file mode 100644 index 0000000000..387ff52096 --- /dev/null +++ b/packages/studio/src/player/components/automationGestureKeys.ts @@ -0,0 +1,49 @@ +/** + * Undo grouping for a lane gesture. + * + * Dragging a point persists on every move and once more on release. History + * coalesces entries that share a key and land within 300ms of each other, and + * these writes did neither: the moves and the release used different keys, and a + * drag slower than the window split into several entries. Undo then took back + * whatever fragment happened to be last, leaving the point near where the drag + * had dropped it — which reads as undo not working at all. + * + * So a gesture mints one key and holds it: every move and the release that ends + * it record under that key with no window to expire, and the next gesture gets a + * fresh one so two drags never collapse into a single step. + */ + +export interface CoalesceHint { + key: string; + /** No expiry: a gesture is one step however long the pointer is held. */ + ms: number; +} + +export interface AutomationGestureKeys { + /** A continuous write. Opens a gesture if one is not already open. */ + live(): CoalesceHint; + /** The persisting write that ends a gesture — or a standalone edit. */ + commit(): CoalesceHint; +} + +export function createAutomationGestureKeys( + mint: () => string = () => `automation-gesture:${++counter}`, +): AutomationGestureKeys { + let open: string | null = null; + const hint = (key: string): CoalesceHint => ({ key, ms: Number.POSITIVE_INFINITY }); + return { + live: () => { + open ??= mint(); + return hint(open); + }, + commit: () => { + // A commit with no drag before it — a Delete, a paste, a typed value — is + // its own step, so it mints rather than joining anything. + const key = open ?? mint(); + open = null; + return hint(key); + }, + }; +} + +let counter = 0; diff --git a/packages/studio/src/player/components/automationLaneData.test.ts b/packages/studio/src/player/components/automationLaneData.test.ts index 948c926167..69b486b871 100644 --- a/packages/studio/src/player/components/automationLaneData.test.ts +++ b/packages/studio/src/player/components/automationLaneData.test.ts @@ -1,6 +1,11 @@ // @vitest-environment happy-dom import { describe, expect, it } from "vitest"; -import { elementAutomation, elementFxChain } from "./automationLaneData"; +import { + automationLaneLabel, + automationLaneLabelParts, + elementAutomation, + elementFxChain, +} from "./automationLaneData"; import type { TimelineElement } from "../store/timelineElement"; const el = (over: Partial = {}): TimelineElement => ({ @@ -13,6 +18,9 @@ const el = (over: Partial = {}): TimelineElement => ({ ...over, }); +/** The chain as the lane code sees it: parsed, not the attribute text. */ +const parseChain = (chain: unknown) => elementFxChain(el({ fxChain: JSON.stringify(chain) }))!; + const CHAIN = JSON.stringify({ version: 1, nodes: [{ type: "lowpass", id: "n1", params: { frequency: 400, q: 0.9, poles: "2" } }], @@ -63,4 +71,91 @@ describe("automationLaneData", () => { expect(elementAutomation(el({ automation: "{nope" })).lanes).toEqual([]); expect(elementFxChain(el({ fxChain: "{nope" }))).toBeNull(); }); + + describe("lane order", () => { + // A stack of EQ bands reads as a spectrum, so it has to be laid out like one: + // the top of the stack is the top of the audible range. Attribute order is + // whatever the carve happened to mint, which is the opposite — bands come out + // ascending. + const bandChain = JSON.stringify({ + version: 1, + nodes: [ + { type: "peaking", id: "n1", params: { frequency: 400, gain: -6, q: 1.4 } }, + { type: "peaking", id: "n2", params: { frequency: 1600, gain: -9, q: 1.4 } }, + { type: "peaking", id: "n3", params: { frequency: 1000, gain: -3, q: 1.4 } }, + { type: "gain", id: "n4", params: { gain: -6 } }, + ], + }); + const bandLanes = JSON.stringify({ + version: 1, + lanes: [ + { target: "volume", points: [{ t: 0, v: 1 }] }, + { target: "fx.n1.gain", points: [{ t: 0, v: -6 }] }, + { target: "fx.n2.gain", points: [{ t: 0, v: -9 }] }, + { target: "fx.n3.gain", points: [{ t: 0, v: -3 }] }, + { target: "fx.n4.gain", points: [{ t: 0, v: -6 }] }, + ], + }); + + it("puts the highest frequency at the top", () => { + const lanes = elementAutomation(el({ automation: bandLanes, fxChain: bandChain })).lanes; + expect(lanes.map((l) => l.target)).toEqual([ + "fx.n2.gain", // 1.6 kHz + "fx.n3.gain", // 1 kHz + "fx.n1.gain", // 400 Hz + // Neither of these is a band, so they sit under the spectrum in the order + // they were written. + "volume", + "fx.n4.gain", + ]); + }); + + it("keeps the same object identity, so a drag survives the sort", () => { + const a = elementAutomation(el({ automation: bandLanes, fxChain: bandChain })); + const b = elementAutomation(el({ automation: bandLanes, fxChain: bandChain })); + expect(a).toBe(b); + }); + }); + + describe("automationLaneLabel", () => { + const chain = parseChain({ + version: 1, + nodes: [ + { type: "peaking", id: "n1", params: { frequency: 1600, gain: -9, q: 1.4 } }, + { type: "peaking", id: "n2", params: { frequency: 400, gain: -6, q: 1.4 } }, + { type: "gain", id: "n3", params: { gain: -6 } }, + ], + }); + + it("names the effect and the band it sits at", () => { + // Three lanes all reading "Peaking EQ · Gain" say nothing about which band + // each one is; a bare frequency does not say whether it is a bell or a + // shelf. Both, then the parameter. + expect(automationLaneLabel("fx.n1.gain", chain)).toBe("Peaking EQ 1.6 kHz · Gain"); + expect(automationLaneLabel("fx.n2.gain", chain)).toBe("Peaking EQ 400 Hz · Gain"); + }); + + it("falls back to the effect's name when it has no frequency", () => { + expect(automationLaneLabel("fx.n3.gain", chain)).toBe("Gain · Gain"); + expect(automationLaneLabel("volume", chain)).toBe("Volume"); + }); + + it("has nothing to say about a target that does not resolve", () => { + expect(automationLaneLabel("fx.gone.gain", chain)).toBeNull(); + expect(automationLaneLabelParts("fx.gone.gain", chain)).toBeNull(); + }); + + it("splits the name from the parameter, which the column stacks", () => { + expect(automationLaneLabelParts("fx.n1.gain", chain)).toEqual({ + name: "Peaking EQ 1.6 kHz", + param: "Gain", + }); + expect(automationLaneLabelParts("fx.n3.gain", chain)).toEqual({ + name: "Gain", + param: "Gain", + }); + // Volume is one word with no effect behind it, so there is no second line. + expect(automationLaneLabelParts("volume", chain)).toEqual({ name: "Volume", param: "" }); + }); + }); }); diff --git a/packages/studio/src/player/components/automationLaneData.ts b/packages/studio/src/player/components/automationLaneData.ts index 375f891dcd..c1c8c850c7 100644 --- a/packages/studio/src/player/components/automationLaneData.ts +++ b/packages/studio/src/player/components/automationLaneData.ts @@ -15,7 +15,9 @@ import { parseAutomation, + parseAutomationTarget, resolveAutomation, + resolveAutomationRange, type HfAutomation, type HfAutomationLane, } from "@hyperframes/core/audio-automation"; @@ -79,7 +81,8 @@ export function elementAutomation(element: TimelineElement): HfAutomation { // through a separator no attribute can contain. return cached(automationCache, `${raw}\u0000${element.fxChain ?? ""}`, () => { try { - return resolveAutomation(parseAutomation(raw), chain ?? undefined); + const resolved = resolveAutomation(parseAutomation(raw), chain ?? undefined); + return { ...resolved, lanes: orderLanes(resolved.lanes, chain) }; } catch { // Unreadable automation draws no lanes rather than breaking the row. return EMPTY; @@ -91,3 +94,83 @@ export function elementAutomation(element: TimelineElement): HfAutomation { export function elementAutomationLanes(element: TimelineElement): HfAutomationLane[] { return elementAutomation(element).lanes; } + +/** The frequency the lane's effect sits at, when it has one. */ +function laneFrequency(target: string, chain: HfAudioFxChain | null): number | null { + const parsed = parseAutomationTarget(target); + if (!parsed || parsed.kind !== "fx") return null; + const node = chain?.nodes.find((n) => n.id === parsed.nodeId); + const freq = node?.params?.["frequency"]; + return typeof freq === "number" ? freq : null; +} + +/** + * Lane order: the audible spectrum, top down, then everything else. + * + * A stack of EQ bands is read as a spectrum, so it has to be laid out like one — + * high at the top, the way every analyser and every EQ curve is drawn. Attribute + * order is whatever minted the nodes, which for a carve is ascending: exactly + * upside down. Lanes with no frequency to place them — a level stage, the track's + * own volume — keep their written order and sit under the bands, like a fader + * below the EQ section of a channel strip. + * + * Sorted here, in the one function both the canvas lanes and the label column + * read, because a label whose row disagrees with the envelope it names is worse + * than either order. + */ +function orderLanes(lanes: HfAutomationLane[], chain: HfAudioFxChain | null): HfAutomationLane[] { + const withFreq: { lane: HfAutomationLane; freq: number }[] = []; + const rest: HfAutomationLane[] = []; + for (const lane of lanes) { + const freq = laneFrequency(lane.target, chain); + if (freq === null) rest.push(lane); + else withFreq.push({ lane, freq }); + } + withFreq.sort((a, b) => b.freq - a.freq); + return [...withFreq.map((e) => e.lane), ...rest]; +} + +/** A frequency as an author reads it: 400 Hz, 1.6 kHz, 10 kHz. */ +export function formatHz(freq: number): string { + if (freq < 1000) return `${Math.round(freq)} Hz`; + const k = freq / 1000; + return `${k >= 10 ? Math.round(k) : Number(k.toFixed(1))} kHz`; +} + +/** + * What a lane is called in the timeline, as its two lines. + * + * `name` is the effect and, when it has one, the frequency it sits at: "Peaking + * EQ 1.6 kHz". The frequency is what tells two bands apart — three lanes all + * reading "Peaking EQ" say nothing about which is which — and the effect still + * has to be named, since a chain mixes filter types and a bare frequency does not + * say whether it is a bell or a shelf. + * + * `param` is which knob the envelope drives, on its own line: a band can carry a + * gain lane and a Q lane, and stacking the two lines is what keeps a name legible + * in a column this narrow instead of truncating mid-word. + * + * Null when the target does not resolve against the chain — the same condition + * that stops the lane being drawn at all. + */ +export function automationLaneLabelParts( + target: string, + chain: HfAudioFxChain | null, +): { name: string; param: string } | null { + const range = resolveAutomationRange(target, chain ?? undefined); + if (!range) return null; + // The registry's label is " · ", or just "" for volume. + const parts = range.label.split(" · "); + const param = parts.at(-1) ?? range.label; + const effect = parts.length > 1 ? parts.slice(0, -1).join(" · ") : null; + const freq = laneFrequency(target, chain); + const name = [effect, freq === null ? null : formatHz(freq)].filter(Boolean).join(" "); + return { name: name || param, param: name ? param : "" }; +} + +/** The whole label on one line, for a tooltip or an accessible name. */ +export function automationLaneLabel(target: string, chain: HfAudioFxChain | null): string | null { + const parts = automationLaneLabelParts(target, chain); + if (!parts) return null; + return parts.param ? `${parts.name} · ${parts.param}` : parts.name; +} diff --git a/packages/studio/src/player/components/automationLaneGeometry.test.ts b/packages/studio/src/player/components/automationLaneGeometry.test.ts index 2b087bd9b0..105cb3b6f3 100644 --- a/packages/studio/src/player/components/automationLaneGeometry.test.ts +++ b/packages/studio/src/player/components/automationLaneGeometry.test.ts @@ -79,29 +79,97 @@ describe("curveForDrag", () => { const a = { t: 0, v: 1 }; const b = { t: 4, v: 0 }; + /** The segment the drag describes, as the model would store it. */ + const bentLane = (bend: { viaX: number; viaY: number } | null) => ({ + target: "volume", + points: [{ ...a, ...(bend ?? {}) }, b], + }); + it("puts the curved segment through the point that was dragged", () => { - // The whole contract: whatever curve comes back, sampling the segment at the - // dragged time has to give the dragged value back — otherwise the line runs - // away from the pointer. + // The whole contract: sampling the segment at the dragged time gives the dragged + // value back, so the line never runs away from the pointer. Anywhere in the + // segment, at any depth. for (const [t, v] of [ - [1, 0.9], - [2, 0.8], - [3, 0.15], + [0.4, 0.75], + [1.6, 0.65], + [2, 0.6], + [2, 0.1], + [2.8, 0.4], + [3.6, 0.5], ] as const) { - const curve = curveForDrag({ range: VOLUME_RANGE, a, b, t, v }); - expect(curve).not.toBeNull(); - const lane = { target: "volume", points: [{ ...a, curve: curve ?? 0 }, b] }; - expect(sampleAutomationLane(lane, t, "linear")).toBeCloseTo(v, 2); + const bend = curveForDrag({ range: VOLUME_RANGE, a, b, t, v }); + expect(bend).not.toBeNull(); + expect(sampleAutomationLane(bentLane(bend), t, "linear")).toBeCloseTo(v, 2); } }); - it("stays inside the range the model will accept", () => { - // Anything outside ±1 is clamped on parse, so a drag past the limit has to - // saturate rather than round-trip to something else. - const curve = curveForDrag({ range: VOLUME_RANGE, a, b, t: 0.05, v: 0.02 }); - expect(curve).not.toBeNull(); - expect(Math.abs(curve ?? 0)).toBeLessThanOrEqual(1); - expect(applyCurve(0.5, curve ?? 0)).toBeGreaterThan(0); + it("follows the pointer into the corner of a segment, as deep as it is dragged", () => { + // The extreme: 10% along, pulled almost to the floor of a falling ramp. The old + // exponent saturated a third of the segment away from the pointer, and a later + // slope cap stopped following it too. Reached exactly now, and the shape it draws + // stays one smooth arc — checked by sampling it, not by trusting it. + const bend = curveForDrag({ range: VOLUME_RANGE, a, b, t: 0.4, v: 0.05 }); + expect(bend).not.toBeNull(); + const drawn = bentLane(bend); + expect(sampleAutomationLane(drawn, 0.4, "linear")).toBeCloseTo(0.05, 2); + + let previous: number | null = null; + let worst = 1; + for (let i = 1; i < 60; i += 1) { + const t = (4 * i) / 60; + const h = 0.01; + const slope = + (sampleAutomationLane(drawn, t + h, "linear") - + sampleAutomationLane(drawn, t - h, "linear")) / + (2 * h); + if (previous !== null && Math.abs(previous) > 1e-6) { + const ratio = Math.abs(slope) > Math.abs(previous) ? slope / previous : previous / slope; + worst = Math.max(worst, Math.abs(ratio)); + } + previous = slope; + } + // Gradual: a crease would show up here as a step change in slope. + expect(worst).toBeLessThan(3); + }); + + it("biases the bend toward whichever point the pointer is nearer", () => { + // The behaviour a single exponent could not give: grabbing the line near the + // right-hand point has to bulge it on the RIGHT. Measured as where the curve + // deviates furthest from the straight line it replaced. + const apexOf = (bend: { viaX: number; viaY: number } | null): number => { + const lane = bentLane(bend); + let best = 0; + let at = 0; + for (let i = 1; i < 40; i++) { + const t = (4 * i) / 40; + const straight = 1 - t / 4; + const gap = Math.abs(sampleAutomationLane(lane, t, "linear") - straight); + if (gap > best) { + best = gap; + at = t; + } + } + return at; + }; + const nearA = curveForDrag({ range: VOLUME_RANGE, a, b, t: 0.6, v: 0.55 }); + const nearB = curveForDrag({ range: VOLUME_RANGE, a, b, t: 3.4, v: 0.45 }); + expect(apexOf(nearA)).toBeLessThan(1.6); + expect(apexOf(nearB)).toBeGreaterThan(2.4); + // And the two sit on opposite sides of centre, which is the whole complaint: + // every bend used to land on the same side whatever the pointer did. + expect(apexOf(nearA)).toBeLessThan(apexOf(nearB)); + }); + + it("stays inside the normalised segment the model will accept", () => { + // Both coordinates are clamped clear of the ends on parse, so a drag right up + // against a breakpoint has to describe an interior point, not the breakpoint. + const bend = curveForDrag({ range: VOLUME_RANGE, a, b, t: 0.05, v: 0.02 }); + expect(bend).not.toBeNull(); + expect(bend?.viaX).toBeGreaterThan(0); + expect(bend?.viaX).toBeLessThan(1); + expect(bend?.viaY).toBeGreaterThan(0); + expect(bend?.viaY).toBeLessThan(1); + expect(applyCurve(0.5, 0)).toBe(0.5); }); it("declines a segment with no room to bend", () => { diff --git a/packages/studio/src/player/components/automationLaneGeometry.ts b/packages/studio/src/player/components/automationLaneGeometry.ts index a0da602c82..ef9966f2b9 100644 --- a/packages/studio/src/player/components/automationLaneGeometry.ts +++ b/packages/studio/src/player/components/automationLaneGeometry.ts @@ -11,6 +11,7 @@ import { fxAutomationTarget, resolveAutomationRange, sampleAutomationLane, + steadyViaPoint, VOLUME_RANGE, VOLUME_TARGET, type AutomationRange, @@ -21,6 +22,17 @@ import { getAudioFxDef, type HfAudioFxChain } from "@hyperframes/core/audio-fx"; /** Points nearer than this in clip seconds are the same point, not two. */ export const POINT_MERGE_SEC = 0.02; + +/** + * Closest two breakpoints may sit in clip seconds while still being two points. + * + * A drag clamps to this short of its neighbour rather than onto it. Landing on the + * exact same time is not a step, it is a deletion: the lane's own normalisation + * collapses points that share a `t`, keeping the later one — so dragging a point + * fully into its neighbour used to consume that neighbour. A millisecond is under a + * pixel at any zoom the lane offers, so the two still read as touching. + */ +export const MIN_POINT_GAP_SEC = 0.001; /** Hit radius for grabbing a point, in px. */ export const GRAB_PX = 7; /** Samples used to draw a segment the eye should see as curved. */ @@ -98,16 +110,21 @@ export function formatValue(range: AutomationRange, value: number): string { } /** - * The `curve` that bends a segment through a dragged point. + * The via point that bends a segment through a dragged pointer: the pointer's own + * position in the segment's normalised space, which is all the model needs. * - * `applyCurve` raises normalised progress to `2^(2*curve)`, so a point the - * pointer holds at progress `x` and unit height `f` fixes the exponent: - * `x^e = f`, hence `e = ln f / ln x` and `curve = log2(e) / 2`. Solving rather - * than accumulating a delta means the segment passes through the pointer - * instead of drifting away from it over a long drag. + * There is nothing to solve any more, and that is the point. This used to fit an + * exponent — `x^e = f`, so `curve = log2(ln f / ln x) / 2` — and an exponent is + * one knob for two questions. It spent it on the wrong one: every upward bend it + * could draw deviated most inside the first fifth of the segment, so grabbing the + * line near its right-hand breakpoint still bulged it on the left. And reaching a + * pointer near either end needed an exponent the model refuses — `e` runs past 15 + * at `x = 0.9`, clamped to 4 — so the line stopped following the pointer + * altogether, missing it by up to a third of the segment's height. Naming the + * point the curve passes through says both things at once, exactly, anywhere. * - * Null when the segment cannot express the shape: a flat segment has no room to - * bend, and progress or height at the very ends divides by zero. + * Null when the segment cannot take a bend: a flat segment draws the same line + * whatever the shape, and a pointer at the very ends is the ends. */ export function curveForDrag(input: { range: AutomationRange; @@ -115,7 +132,7 @@ export function curveForDrag(input: { b: { t: number; v: number }; t: number; v: number; -}): number | null { +}): { viaX: number; viaY: number } | null { const { range, a, b, t, v } = input; const span = b.t - a.t; if (span <= 0) return null; @@ -126,7 +143,11 @@ export function curveForDrag(input: { if (Math.abs(ub - ua) < 0.001) return null; const f = (toUnit(range, v) - ua) / (ub - ua); if (f <= 0.001 || f >= 0.999) return null; - return Math.max(-1, Math.min(1, Math.log2(Math.log(f) / Math.log(x)) / 2)); + // Reported as the model will honour it, not as the pointer asked. A bend is held + // to a steady curve, so a pointer dragged past that stops being followed — and + // the write, the preview and the readout all have to say the same thing about + // where the line actually went. + return steadyViaPoint(x, f); } /** @@ -137,6 +158,32 @@ export function curveForDrag(input: { * Which axis "won" is decided in pixels, not in seconds and dB — those are * different units and comparing them would make the lock depend on the zoom. */ +/** Which way a gesture is going, in pixels — the only comparable unit. */ +export function dominantDragAxis(input: { + origin: { t: number; v: number }; + raw: { t: number; v: number }; + xOf(t: number): number; + yOf(v: number): number; +}): "time" | "value" { + const { origin, raw, xOf, yOf } = input; + return Math.abs(xOf(raw.t) - xOf(origin.t)) > Math.abs(yOf(raw.v) - yOf(origin.v)) + ? "time" + : "value"; +} + +/** + * The pointer, constrained to one axis. + * + * The axis is handed in rather than worked out here, because it has to be decided + * once for the gesture and held. Recomputed per event it followed whichever way + * the last move happened to lean, so a hand drifting sideways during a vertical + * drag flipped the lock and the point moved in both — which is indistinguishable + * from no lock at all. + * + * Locking to time holds the value exactly. Locking to value holds the time and + * moves the value at a quarter speed: the same gesture is the fine adjustment, + * because a fader spanning 60px of lane has no other way to be set precisely. + */ export function applyShiftConstraint(input: { range: AutomationRange; origin: { t: number; v: number }; @@ -144,11 +191,12 @@ export function applyShiftConstraint(input: { /** Same projections the lane draws with, so the comparison is on screen. */ xOf(t: number): number; yOf(v: number): number; + /** Decided on the gesture's first travel; worked out here when absent. */ + axis?: "time" | "value"; }): { t: number; v: number } { - const { range, origin, raw, xOf, yOf } = input; - if (Math.abs(xOf(raw.t) - xOf(origin.t)) > Math.abs(yOf(raw.v) - yOf(origin.v))) { - return { t: raw.t, v: origin.v }; - } + const { range, origin, raw } = input; + const axis = input.axis ?? dominantDragAxis(input); + if (axis === "time") return { t: raw.t, v: origin.v }; const from = toUnit(range, origin.v); return { t: origin.t, v: fromUnit(range, from + (toUnit(range, raw.v) - from) * 0.25) }; } @@ -202,7 +250,9 @@ export function envelopePath(input: { const a = lane.points[i]; const b = lane.points[i + 1]; if (!a || !b) continue; - if (!a.curve && range.scale === "linear") { + // A via point bends the segment with no `curve` of its own, so the + // straight-line shortcut has to rule out both. + if (!a.curve && a.viaX === undefined && range.scale === "linear") { pts.push(`L ${xOf(b.t)} ${yOf(b.v)}`); continue; } diff --git a/packages/studio/src/player/components/automationLaneHeight.ts b/packages/studio/src/player/components/automationLaneHeight.ts index 3ad43c4282..b657ee5f76 100644 --- a/packages/studio/src/player/components/automationLaneHeight.ts +++ b/packages/studio/src/player/components/automationLaneHeight.ts @@ -6,6 +6,10 @@ * component's constant living somewhere it is not used. * * Taller than a keyframe lane because it carries a value axis rather than a row - * of diamonds: a fader envelope drawn 28px high cannot be aimed. + * of diamonds: a fader envelope drawn 28px high cannot be aimed. 48 was still + * too tight in use — the drawing area is the height minus 6px of padding either + * side, so 48 left 36px for the whole 0..1 fader and a breakpoint's 11px grab + * disc covered a third of the axis. 72 leaves 60px, which is what makes a value + * aimable and two points at similar values separately grabbable. */ -export const AUTOMATION_LANE_H = 48; +export const AUTOMATION_LANE_H = 72; diff --git a/packages/studio/src/player/components/automationLaneSelection.test.ts b/packages/studio/src/player/components/automationLaneSelection.test.ts index b894984f74..a7f0296add 100644 --- a/packages/studio/src/player/components/automationLaneSelection.test.ts +++ b/packages/studio/src/player/components/automationLaneSelection.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { pointsIn, replaceRange, retimeRange } from "./automationLaneSelection"; +import { pointsIn, replaceRange } from "./automationLaneSelection"; import { sampleAutomationLane, VOLUME_RANGE } from "@hyperframes/core/audio-automation"; import type { HfAutomationLane } from "@hyperframes/core/audio-automation"; @@ -94,90 +94,3 @@ describe("replaceRange", () => { expect(Math.max(...innerTimes)).toBeGreaterThan(3.0); }); }); - -describe("retimeRange", () => { - it("scales interior points proportionally into the new span", () => { - const pts = retimeRange({ lane: ramp, range: VOLUME_RANGE, t0: 2, t1: 3, newT0: 2, newT1: 5 }); - const moved = pts.find((p) => p.v === 0.4); // the t=3 point - expect(moved?.t).toBe(5); - }); - - it("preserves the envelope outside the union of old and new spans", () => { - const before: HfAutomationLane = { target: "volume", points: ramp.points }; - const after: HfAutomationLane = { - target: "volume", - points: retimeRange({ lane: ramp, range: VOLUME_RANGE, t0: 2, t1: 3, newT0: 2, newT1: 5 }), - }; - // Nothing to the left of t0=2 moved (newT0 === t0 here), so sampled - // continuity holds all the way up to the edited region. - for (const t of [0, 1, 1.9]) { - expect(sampleAutomationLane(after, t, "linear")).toBeCloseTo( - sampleAutomationLane(before, t, "linear"), - 5, - ); - } - // The next real breakpoint past the edited region keeps its own exact value. - const farPoint = after.points.find((p) => p.t === 6); - expect(farPoint).toEqual({ t: 6, v: 0 }); - }); - - it("preserves the envelope on BOTH sides when no breakpoint sits on the moved edge", () => { - // The right side is where the invariant is worth asserting — `newT0 === t0` - // makes the left side of the test above trivially true, and an earlier - // right-side probe at t=5.1 was DELETED as inherent when it was reporting - // the real behaviour below. With the selection's edges off any breakpoint, - // the guarantee holds exactly, in both directions. - const before: HfAutomationLane = { target: "volume", points: ramp.points }; - const after: HfAutomationLane = { - target: "volume", - points: retimeRange({ - lane: ramp, - range: VOLUME_RANGE, - t0: 2.2, - t1: 2.9, - newT0: 2.2, - newT1: 4, - }), - }; - for (const t of [0, 1, 2, 4.5, 5, 5.5, 6]) { - expect(sampleAutomationLane(after, t, "linear")).toBeCloseTo( - sampleAutomationLane(before, t, "linear"), - 5, - ); - } - }); - - it("moves a breakpoint sitting exactly on the dragged edge, reshaping the segment past it", () => { - // The design decision this pins, because it is not free either way. - // `pointsIn` is endpoint-inclusive, so a breakpoint ON the edge is interior - // and travels with the stretch. It has to: every range operation leaves a - // breakpoint exactly on the edge it created, so treating that point as an - // anchor instead would make the commonest stretch — grabbing the edge to - // drag that very point outward — delete it and flatten the span. - // - // The cost is that the retimed point lands ON the union's boundary, where a - // preservation anchor would also go, and `anchor()` stands down within a - // merge radius. Two different values cannot occupy one time; the segment - // leaving the union reshapes, which is the "envelope outside the selection - // never moves" invariant bending exactly here and nowhere else. - const pts = retimeRange({ lane: ramp, range: VOLUME_RANGE, t0: 2, t1: 3, newT0: 2, newT1: 5 }); - expect(pts).toEqual([ - { t: 0, v: 1 }, - { t: 2, v: 0.6 }, - { t: 5, v: 0.4 }, // the t=3 point, retimed onto the new edge - { t: 6, v: 0 }, - ]); - // The old 3→6 segment sloped -0.133/s; the new 5→6 slopes -0.4/s, so the - // envelope past the union genuinely moves. Asserted, not tolerated: if this - // number changes, the decision above changed with it. - const after: HfAutomationLane = { target: "volume", points: pts }; - expect(sampleAutomationLane(after, 5.5, "linear")).toBeCloseTo(0.2, 5); - expect(sampleAutomationLane(ramp, 5.5, "linear")).toBeCloseTo(0.0667, 4); - }); - - it("rejects a degenerate span", () => { - expect( - retimeRange({ lane: ramp, range: VOLUME_RANGE, t0: 2, t1: 3, newT0: 4, newT1: 4 }), - ).toEqual(ramp.points); - }); -}); diff --git a/packages/studio/src/player/components/automationLaneSelection.ts b/packages/studio/src/player/components/automationLaneSelection.ts index 66e384e6f1..1b6adae6a9 100644 --- a/packages/studio/src/player/components/automationLaneSelection.ts +++ b/packages/studio/src/player/components/automationLaneSelection.ts @@ -2,7 +2,7 @@ * Range operations over one automation lane. * * `replaceRange` is the only mutator every range feature (delete, shapes, - * paste, stretch) composes, and it carries the invariant that makes them safe: + * paste) composes, and it carries the invariant that makes them safe: * the envelope OUTSIDE the selection never moves. It samples the lane at both * edges first and pins anchor points there, so cutting the middle out of a * ramp cannot reshape the rest of the clip. @@ -77,41 +77,20 @@ export function replaceRange(input: { } /** - * Retime a selection: interior points scale proportionally into the new span, - * then replaceRange runs over the UNION of old and new spans — growing eats - * whatever it covers, shrinking pins anchors where the envelope re-enters. + * Whether a breakpoint falls inside the selection box, edges included. * - * Interior is `pointsIn`, so a breakpoint sitting exactly ON an edge travels - * with the stretch. Deliberate: every range operation leaves a breakpoint on the - * edge it created, so treating that point as a fixed anchor would make the - * commonest stretch of all — grabbing the edge to drag that point outward — - * delete it instead. The price is that such a point lands on the union's own - * boundary, where `anchor` then stands down (one time cannot hold two values), - * so the segment leaving the union reshapes. That is the ONE place - * `replaceRange`'s outside-never-moves invariant bends, and it is pinned by name - * in automationLaneSelection.test.ts. + * The one rule three places need: what Delete removes, what the lane rings, and + * what a group drag moves. They have to agree — a point drawn as caught but left + * behind by the drag is worse than either answer. + * + * Both axes, which is what makes a selection a box: a point at the right time but + * the wrong value is not in it. Values compare in the parameter's own units, and + * that is correct on a logarithmic axis too — the mapping to screen is monotonic, + * so a box drawn around some pixels holds exactly the values it looks like it does. */ -export function retimeRange(input: { - lane: HfAutomationLane; - range: AutomationRange; - t0: number; - t1: number; - newT0: number; - newT1: number; -}): HfAutomationPoint[] { - const { lane, range, t0, t1, newT0, newT1 } = input; - const oldSpan = t1 - t0; - const newSpan = newT1 - newT0; - if (oldSpan <= 0 || newSpan <= 0) return lane.points; - const inner = pointsIn(lane, t0, t1).map((p) => ({ - ...p, - t: newT0 + ((p.t - t0) * newSpan) / oldSpan, - })); - return replaceRange({ - lane, - range, - t0: Math.min(t0, newT0), - t1: Math.max(t1, newT1), - inner, - }); +export function pointInSelection( + point: { t: number; v: number }, + box: { t0: number; t1: number; v0: number; v1: number }, +): boolean { + return point.t >= box.t0 && point.t <= box.t1 && point.v >= box.v0 && point.v <= box.v1; } diff --git a/packages/studio/src/player/components/useAutomationEdgeStretch.test.ts b/packages/studio/src/player/components/useAutomationEdgeStretch.test.ts deleted file mode 100644 index 691066d22b..0000000000 --- a/packages/studio/src/player/components/useAutomationEdgeStretch.test.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { clampEdge } from "./useAutomationEdgeStretch"; - -describe("clampEdge", () => { - it("keeps the dragged edge inside the clip", () => { - expect(clampEdge("t0", -5, { t0: 1, t1: 3 }, 4)).toBe(0); - expect(clampEdge("t1", 99, { t0: 1, t1: 3 }, 4)).toBe(4); - }); - - it("keeps the dragged edge clear of its partner", () => { - expect(clampEdge("t0", 3.5, { t0: 1, t1: 3 }, 4)).toBeCloseTo(2.98, 5); - expect(clampEdge("t1", 0.5, { t0: 1, t1: 3 }, 4)).toBeCloseTo(1.02, 5); - }); - - it("never yields a negative time, even when the partner bound is itself below zero", () => { - // A selection thinner than the minimum width has no legal t0 at all. Bounding - // against the partner AFTER the 0-floor returned that illegal value, and - // core's cleanPoint collapses a negative time onto a duplicate t=0 on the - // next serialize round-trip — silent envelope corruption, not a visible bug. - expect(clampEdge("t0", -1, { t0: 0.004, t1: 0.01 }, 4)).toBe(0); - expect(clampEdge("t0", 0.008, { t0: 0.004, t1: 0.01 }, 4)).toBe(0); - }); - - it("never yields a time past the clip when the partner bound is past it", () => { - expect(clampEdge("t1", 99, { t0: 3.995, t1: 4 }, 4)).toBe(4); - }); -}); diff --git a/packages/studio/src/player/components/useAutomationEdgeStretch.ts b/packages/studio/src/player/components/useAutomationEdgeStretch.ts deleted file mode 100644 index faae242e46..0000000000 --- a/packages/studio/src/player/components/useAutomationEdgeStretch.ts +++ /dev/null @@ -1,248 +0,0 @@ -/** - * Stretching an automation selection by one of its edges. - * - * Its own module because edge-stretch is a fifth mutually-exclusive gesture on a - * lane that already sits near the studio's file ceiling, and because it has to - * follow the same three-part contract the other gestures do — a movement - * threshold before anything is written, a live preview the user can see, and a - * revert when the browser abandons the gesture. It was first written without - * them, and every one of its bugs came from that: a bare click near an edge - * pushed an undo entry that changed nothing, the highlight stayed frozen at the - * pre-drag bounds so the handle was invisible while it moved, and a - * `pointercancel` persisted whatever partial retime it had reached. - * - * `retimeRange` is a RELATIVE transform — it scales a lane's own current point - * positions by newSpan/oldSpan — so it must always run against the snapshot - * taken when the drag armed, never the live draft, or the scale factor compounds - * on every pointermove. - */ - -import { useCallback, useRef, useState, type PointerEvent as ReactPointerEvent } from "react"; -import type { - AutomationRange, - HfAutomationLane, - HfAutomationPoint, -} from "@hyperframes/core/audio-automation"; -import { capturePointer } from "./automationLanePointer"; -import { retimeRange } from "./automationLaneSelection"; - -/** Hit radius for grabbing a selection's edge, in screen px — independent of a - * point's own grab radius so the two zones can be reasoned about on their own. */ -const EDGE_GRAB_PX = 8; - -/** Screen px a press has to travel before it counts as a stretch rather than a - * click. The sibling range-drag uses the same 3px, and for the same reason: - * below it, a press is the "clear the selection" escape, and writing anything - * would put a no-op entry in undo. */ -const EDGE_DRAG_PX = 3; - -/** - * Narrowest selection a stretch can leave behind, in clip seconds. Its own - * constant rather than a borrowed `POINT_MERGE_SEC`: that one is about when two - * breakpoints are the same breakpoint, which is a different question from how - * thin a time selection may get and still be grabbable. - */ -const MIN_SELECTION_SEC = 0.02; - -export interface UseAutomationEdgeStretchInput { - getBox(): DOMRect | null; - lane: HfAutomationLane; - range: AutomationRange; - /** Pointer position as a clip-local time and a parameter value. */ - pointAt(clientX: number, clientY: number): { t: number; v: number }; - xOf(t: number): number; - commitPoints(points: HfAutomationLane["points"], persist: boolean): void; - /** Clamp bound for the dragged edge. */ - duration: number; - readOnly?: boolean | undefined; - /** Active selection on this lane, so its edges have something to grab. */ - rangeSelection?: { t0: number; t1: number } | null | undefined; - onRangeSelect?: ((t0: number, t1: number) => void) | undefined; - onRangeClear?: (() => void) | undefined; - /** Value readout owned by the lane's gesture hook. */ - onHint(text: string | null): void; -} - -export interface UseAutomationEdgeStretchResult { - /** Edge being stretched, for the cursor. Null when no stretch is live. */ - edge: "t0" | "t1" | null; - /** Pointer sits over a handle with no gesture live — the col-resize hint. */ - hover: boolean; - /** Take the press as a stretch, or decline it so another gesture can have it. */ - arm(e: ReactPointerEvent): boolean; - move(e: ReactPointerEvent): void; - /** Pointer released: persist the stretch, or clear the selection when the - * press never travelled far enough to be one. */ - finish(): void; - /** The browser abandoned the gesture: put the envelope and the selection back - * the way they were, with nothing persisted. */ - cancel(): void; - updateHover(e: ReactPointerEvent): void; -} - -/** The dragged edge, clamped inside the clip AND clear of its partner. The - * 0/duration bound is applied LAST so a selection thinner than - * `MIN_SELECTION_SEC` can never push its own t0 negative — core's `cleanPoint` - * collapses negative times onto a duplicate t=0 on the next serialize. - * Exported for that ordering's own test: reaching it through the pointer needs - * a selection so thin that no drag can clear the movement threshold. */ -export function clampEdge( - edge: "t0" | "t1", - raw: number, - origin: { t0: number; t1: number }, - duration: number, -): number { - if (edge === "t0") { - return Math.max(0, Math.min(raw, origin.t1 - MIN_SELECTION_SEC)); - } - return Math.min(duration, Math.max(raw, origin.t0 + MIN_SELECTION_SEC)); -} - -export function useAutomationEdgeStretch({ - getBox, - lane, - range, - pointAt, - xOf, - commitPoints, - duration, - readOnly, - rangeSelection, - onRangeSelect, - onRangeClear, - onHint, -}: UseAutomationEdgeStretchInput): UseAutomationEdgeStretchResult { - /** The live stretch: which edge, the selection it started from (fixed, as the - * retime's untouched anchor), the edge's own live position, and the lane's - * points as they stood at arm time. */ - const [drag, setDrag] = useState<{ - edge: "t0" | "t1"; - origin: { t0: number; t1: number }; - current: number; - points: HfAutomationPoint[]; - } | null>(null); - /** Whether the drag has travelled far enough to write anything at all. */ - const crossed = useRef(false); - const [hover, setHover] = useState(false); - - /** - * Which edge of the active selection is within grab range of the pointer's - * screen x, over the full lane height — the handle spans the rect. - * - * A selection narrower than two halos has both edges under one press. The - * midpoint split says which one wins — the same rule a nearest-distance - * comparison expresses, written so that is legible rather than inferred, since - * on a narrow selection it is the ONLY thing deciding the gesture. Pressing a - * handle without moving clears the selection (see `finish`), which is what - * gets a user out of a halo too small to aim inside. - */ - const edgeAt = useCallback( - (clientX: number): "t0" | "t1" | null => { - if (!rangeSelection) return null; - const box = getBox(); - if (!box) return null; - const px = clientX - box.left; - const x0 = xOf(rangeSelection.t0); - const x1 = xOf(rangeSelection.t1); - const near0 = Math.abs(x0 - px) <= EDGE_GRAB_PX; - const near1 = Math.abs(x1 - px) <= EDGE_GRAB_PX; - if (near0 && near1) return px <= (x0 + x1) / 2 ? "t0" : "t1"; - if (near0) return "t0"; - return near1 ? "t1" : null; - }, - [rangeSelection, getBox, xOf], - ); - - const arm = useCallback( - (e: ReactPointerEvent): boolean => { - if (readOnly || !rangeSelection) return false; - const edge = edgeAt(e.clientX); - if (!edge) return false; - e.preventDefault(); - capturePointer(e); - setHover(false); - crossed.current = false; - setDrag({ - edge, - origin: rangeSelection, - current: edge === "t0" ? rangeSelection.t0 : rangeSelection.t1, - points: lane.points, - }); - return true; - }, - [readOnly, rangeSelection, edgeAt, lane], - ); - - /** Preview the stretch: the partner edge stays put as the retime's anchor, and - * the selection itself follows the pointer so the handle being dragged is - * visible — the same live `onRangeSelect` the marquee drag fires. */ - const move = useCallback( - (e: ReactPointerEvent): void => { - if (drag === null) return; - const { edge, origin, points } = drag; - const current = clampEdge(edge, pointAt(e.clientX, e.clientY).t, origin, duration); - setDrag({ edge, origin, current, points }); - if (!crossed.current) { - const from = edge === "t0" ? origin.t0 : origin.t1; - if (Math.abs(xOf(current) - xOf(from)) <= EDGE_DRAG_PX) return; - crossed.current = true; - } - const newT0 = edge === "t0" ? current : origin.t0; - const newT1 = edge === "t1" ? current : origin.t1; - onHint(`${newT0.toFixed(2)}s → ${newT1.toFixed(2)}s`); - onRangeSelect?.(newT0, newT1); - commitPoints( - retimeRange({ - lane: { target: lane.target, points }, - range, - t0: origin.t0, - t1: origin.t1, - newT0, - newT1, - }), - false, - ); - }, - [drag, pointAt, duration, xOf, onHint, onRangeSelect, commitPoints, lane.target, range], - ); - - const finish = useCallback((): void => { - if (drag === null) return; - const { edge, origin, current } = drag; - setDrag(null); - onHint(null); - // A press that never travelled is the "clear the selection" click, exactly - // as it is anywhere else on the background. Persisting here instead pushed - // an undo entry that changed nothing AND made that escape unreachable - // within a halo of either edge. - if (!crossed.current) { - onRangeClear?.(); - return; - } - crossed.current = false; - commitPoints(lane.points, true); - onRangeSelect?.(edge === "t0" ? current : origin.t0, edge === "t1" ? current : origin.t1); - }, [drag, onHint, onRangeClear, commitPoints, lane, onRangeSelect]); - - const cancel = useCallback((): void => { - if (drag === null) return; - const { origin, points } = drag; - setDrag(null); - onHint(null); - if (!crossed.current) return; - crossed.current = false; - // A live write is a preview, so putting the snapshot back through the same - // channel is the whole revert — there is nothing persisted to undo. - commitPoints(points, false); - onRangeSelect?.(origin.t0, origin.t1); - }, [drag, onHint, commitPoints, onRangeSelect]); - - const updateHover = useCallback( - (e: ReactPointerEvent): void => { - if (!readOnly) setHover(edgeAt(e.clientX) !== null); - }, - [readOnly, edgeAt], - ); - - return { edge: drag?.edge ?? null, hover, arm, move, finish, cancel, updateHover }; -} diff --git a/packages/studio/src/player/components/useAutomationLaneGestures.ts b/packages/studio/src/player/components/useAutomationLaneGestures.ts index 73f1a23553..3cbe4937c5 100644 --- a/packages/studio/src/player/components/useAutomationLaneGestures.ts +++ b/packages/studio/src/player/components/useAutomationLaneGestures.ts @@ -4,7 +4,7 @@ * Its own hook because the lane component sits at the studio's file ceiling and * because these are the parts worth testing on their own: which of a press, * a drag and a modifier resolves to moving a point, bending a segment, - * stretching a selection's edge, or nothing at all. + * drawing a selection box, or nothing at all. * * Modifiers follow Ableton's, since that is the muscle memory an automation lane * inherits: Shift locks a drag to one axis and fines the value down, Alt over a @@ -15,14 +15,19 @@ import { useCallback, useRef, useState, type PointerEvent as ReactPointerEvent } import type { AutomationRange, HfAutomationLane } from "@hyperframes/core/audio-automation"; import { applyShiftConstraint, + dominantDragAxis, curveForDrag, formatValue, GRAB_PX, + MIN_POINT_GAP_SEC, POINT_MERGE_SEC, snapLaneTime, } from "./automationLaneGeometry"; +import { pointInSelection } from "./automationLaneSelection"; import { capturePointer } from "./automationLanePointer"; -import { useAutomationEdgeStretch } from "./useAutomationEdgeStretch"; + +/** How far a press may travel and still count as a click rather than a drag. */ +const CLICK_SLOP_PX = 3; /** Snap radius in clip seconds. Tight on purpose: a lane is often a few seconds * wide, where a generous radius makes a point unplaceable between two beats. */ @@ -49,12 +54,12 @@ export interface UseAutomationLaneGesturesInput { snapTimes?: readonly number[] | undefined; readOnly?: boolean | undefined; onSelect?: (() => void) | undefined; - /** Live range-select callbacks; absent = background drags do nothing (read-only lanes). */ - onRangeSelect?: ((t0: number, t1: number) => void) | undefined; + /** Live box-select callbacks; absent = background drags do nothing (read-only lanes). */ + onRangeSelect?: ((t0: number, t1: number, v0: number, v1: number) => void) | undefined; onRangeClear?: (() => void) | undefined; - duration: number; // clamp bound for range endpoints - /** Active selection on this lane, so its edges have something to grab. */ - rangeSelection?: { t0: number; t1: number } | null | undefined; + duration: number; // clamp bound for box endpoints + /** Active selection box on this lane, so a press inside it can drag its points. */ + rangeSelection?: { t0: number; t1: number; v0: number; v1: number } | null | undefined; } export interface UseAutomationLaneGesturesResult { @@ -62,11 +67,6 @@ export interface UseAutomationLaneGesturesResult { dragIndex: number | null; /** Segment being bent, identified by the point that owns its curve. */ curveIndex: number | null; - /** Edge being stretched, for the cursor. */ - edgeDrag: "t0" | "t1" | null; - /** Whether the pointer sits over a stretch handle with no gesture live — - * the col-resize cursor hint before a press commits to the drag. */ - edgeHover: boolean; /** Value readout to show while a gesture is live. */ hint: string | null; hitIndex(clientX: number, clientY: number): number | null; @@ -74,9 +74,6 @@ export interface UseAutomationLaneGesturesResult { onPointerDown(e: ReactPointerEvent): void; onPointerMove(e: ReactPointerEvent): void; endDrag(e: ReactPointerEvent): void; - /** The browser took the gesture away (`pointercancel`): a stretch reverts - * rather than persisting whatever partial retime it had reached. */ - cancelDrag(e: ReactPointerEvent): void; /** Adds a point, opens the value field on one, or straightens a segment. */ onDoubleClick(e: ReactPointerEvent): void; /** The point whose value is being typed, and the text so far. */ @@ -107,28 +104,65 @@ export function useAutomationLaneGestures({ const [hint, setHint] = useState(null); /** Where a point drag began, so Shift can lock an axis and fine the value. */ const dragOrigin = useRef<{ t: number; v: number } | null>(null); + /** + * The set a group drag moves, snapshotted on the press. + * + * Snapshotted rather than recomputed per move for two reasons: every point is + * moving, so "which ones are selected" has to mean what it meant when the + * gesture started, and the deltas have to accumulate from the original + * positions or a rounded move would drift on every pointermove. + */ + const groupDrag = useRef<{ + points: HfAutomationLane["points"]; + indices: number[]; + anchor: { t: number; v: number }; + selection: { t0: number; t1: number; v0: number; v1: number }; + } | null>(null); /** Point whose value is being typed, and the text so far. */ const [editing, setEditing] = useState<{ index: number; text: string } | null>(null); - /** A background drag in progress: its start and live end, in clip seconds. */ - const [rangeDrag, setRangeDrag] = useState<{ from: number; to: number } | null>(null); + /** A background drag in progress: the two corners of the box it is drawing, + * each a clip-local time and a parameter value. */ + const [rangeDrag, setRangeDrag] = useState<{ + from: { t: number; v: number }; + to: { t: number; v: number }; + } | null>(null); /** Whether the live drag has crossed the pixel threshold that turns a press - * into an actual range, rather than a click that should just clear one. */ + * into an actual box, rather than a click that should just clear one. */ const rangeCrossed = useRef(false); + /** Where a press on a point landed, and whether it has travelled since. A press + * that goes nowhere is a click, which is a different gesture from a drag even + * though both start the same way — see the Shift branch in `endDrag`. */ + const pressAt = useRef<{ x: number; y: number } | null>(null); + const pressTravelled = useRef(false); + /** + * The axis Shift locked, decided on the gesture's first travel and held until + * the drag ends or Shift is let go. Deciding per event followed whichever way + * the last move leaned, so a drifting hand unlocked the axis mid-drag. + */ + const shiftAxis = useRef<"time" | "value" | null>(null); + + // The value field's own handlers, above the gestures that close it: a press on the + // lane applies whatever was typed, so onPointerDown lists commitEdit as a + // dependency and cannot be declared before it. + const setEditingText = useCallback((text: string): void => { + setEditing((current) => (current ? { index: current.index, text } : null)); + }, []); + + const cancelEdit = useCallback((): void => setEditing(null), []); - const stretch = useAutomationEdgeStretch({ - getBox, - lane, - range, - pointAt, - xOf, - commitPoints, - duration, - readOnly, - rangeSelection, - onRangeSelect, - onRangeClear, - onHint: setHint, - }); + /** Apply a typed value, or drop the edit when it is not a number. */ + const commitEdit = useCallback((): void => { + const active = editing; + setEditing(null); + if (!active) return; + const typed = Number(active.text); + if (!Number.isFinite(typed)) return; + const clamped = Math.min(range.max, Math.max(range.min, typed)); + commitPoints( + lane.points.map((p, i) => (i === active.index ? { ...p, v: clamped } : p)), + true, + ); + }, [editing, lane, range, commitPoints]); /** Index of a point under the pointer, or null. */ const hitIndex = useCallback( @@ -173,22 +207,40 @@ export function useAutomationLaneGestures({ ); /** - * What a press on the lane's empty background arms: a new range selection, - * and only when a caller wants to hear about one — a read-only lane never - * reaches here at all. + * A corner of the selection box under the pointer. + * + * Time is clamped to the clip and snaps to the grid, because the box's span is + * also what a shape insert or a paste acts over and those want beat-aligned + * edges. The value is taken as it lies: there is no grid on a parameter axis, + * and rounding a dB bound would move which points the box catches. + */ + const boxCornerAt = useCallback( + (e: ReactPointerEvent): { t: number; v: number } => { + const raw = pointAt(e.clientX, e.clientY); + const clamped = Math.min(duration, Math.max(0, raw.t)); + return { + t: e.altKey ? clamped : snapLaneTime(clamped, snapTimes ?? [], SNAP_SEC), + v: raw.v, + }; + }, + [pointAt, duration, snapTimes], + ); + + /** + * What a press on the lane's empty background arms: a new selection box, and + * only when a caller wants to hear about one — a read-only lane never reaches + * here at all. */ const armRangeDrag = useCallback( (e: ReactPointerEvent): void => { if (!onRangeSelect) return; e.preventDefault(); capturePointer(e); - const raw = pointAt(e.clientX, e.clientY).t; - const clamped = Math.min(duration, Math.max(0, raw)); - const t = e.altKey ? clamped : snapLaneTime(clamped, snapTimes ?? [], SNAP_SEC); + const corner = boxCornerAt(e); rangeCrossed.current = false; - setRangeDrag({ from: t, to: t }); + setRangeDrag({ from: corner, to: corner }); }, - [onRangeSelect, pointAt, duration, snapTimes], + [onRangeSelect, boxCornerAt], ); const onPointerDown = useCallback( @@ -198,18 +250,22 @@ export function useAutomationLaneGestures({ // timeline's own gesture (scrub / marquee / clip drag), which then eats the // rest of the sequence — including the second half of a double-click. e.stopPropagation(); + // A press anywhere on the lane closes the value field, applying what was + // typed — the same thing Enter and a blur do. It cannot rely on the blur: the + // gesture branches below call preventDefault to keep the timeline from + // scrubbing, and that suppresses the focus change the blur would come from, + // so the field sat open until Enter however far away the next click landed. + if (editing) commitEdit(); if (readOnly) { // The lane sits below the clip bar, so the timeline's selection handler - // never sees this press; selecting here is the only way in. + // never sees this press; selecting here is the only way in. The press then + // goes on to arm a range drag rather than being spent on the selection: a + // press that only selected made the first drag over any lane do nothing + // visible, so a range took two gestures and looked broken on the first. onSelect?.(); + armRangeDrag(e); return; } - // An active selection's edge outranks a point sitting on it. Every range - // operation — stretch, delete, shape insert — leaves a breakpoint exactly - // on the edge it just created, so a point-first rule meant the second - // stretch of the same edge resolved to a point-drag and the feature was - // not repeatable. Clear the selection to reach that point again. - if (stretch.arm(e)) return; const gesture = gestureAt(e); if (!gesture) { armRangeDrag(e); @@ -222,9 +278,39 @@ export function useAutomationLaneGestures({ return; } dragOrigin.current = originOf(lane.points[gesture.index]); + // Pressing one of a selected set drags the whole set. Pressing a point + // outside the selection is an ordinary single-point drag, selection or no. + const pressed = lane.points[gesture.index]; + const indices = + rangeSelection && pressed && pointInSelection(pressed, rangeSelection) + ? lane.points.flatMap((p, i) => (pointInSelection(p, rangeSelection) ? [i] : [])) + : []; + groupDrag.current = + indices.length > 1 && pressed + ? { + points: lane.points.map((p) => ({ ...p })), + indices, + anchor: { t: pressed.t, v: pressed.v }, + selection: rangeSelection ? { ...rangeSelection } : { t0: 0, t1: 0, v0: 0, v1: 0 }, + } + : null; + pressAt.current = { x: e.clientX, y: e.clientY }; + pressTravelled.current = false; setDragIndex(gesture.index); }, - [gestureAt, lane, readOnly, onSelect, armRangeDrag, stretch], + [ + gestureAt, + lane, + readOnly, + onSelect, + armRangeDrag, + editing, + commitEdit, + // The press decides whether it starts a group drag, so it has to see the + // selection as it is now — captured stale, a point pressed straight after + // selecting a range read the previous selection, or none. + rangeSelection, + ], ); /** Bend the segment under the pointer, which is what Alt-dragging the line does. */ @@ -235,85 +321,189 @@ export function useAutomationLaneGestures({ const b = lane.points[curveIndex + 1]; if (!a || !b) return; const { t, v } = pointAt(clientX, clientY); - const curve = curveForDrag({ range, a, b, t, v }); - if (curve === null) return; - setHint(`curve ${curve.toFixed(2)}`); + const bend = curveForDrag({ range, a, b, t, v }); + if (bend === null) return; + // Read out where the bend now sits along the segment, which is what the + // pointer is choosing: a percentage means more here than a curve exponent + // the author never types. + setHint(`bend ${Math.round(bend.viaX * 100)}%`); commitPoints( - lane.points.map((p, i) => (i === curveIndex ? { ...p, curve } : p)), + // `curve` is dropped rather than carried: the via point supersedes it, and + // leaving a stale exponent behind would make the segment's shape depend on + // which of the two the reader happened to honour. + lane.points.map((p, i) => + i === curveIndex ? { ...p, curve: undefined, viaX: bend.viaX, viaY: bend.viaY } : p, + ), false, ); }, [curveIndex, lane, pointAt, range, commitPoints], ); + /** + * Move a selected set by one delta, taken from the point under the pointer. + * + * The whole group has to stop when its first member reaches a boundary, not + * each point on its own: clamping individually squashes the shape flat against + * the edge, and the gesture is meant to preserve it. Deltas are in the + * parameter's own units, so on a logarithmic axis a group moves by Hz rather + * than by octaves — the same as dragging one point does. + */ + const moveGroup = useCallback( + (e: ReactPointerEvent, group: NonNullable): void => { + const raw = pointAt(e.clientX, e.clientY); + const moving = group.indices.map((i) => group.points[i]!); + let dt = raw.t - group.anchor.t; + let dv = raw.v - group.anchor.v; + if (e.shiftKey) { + // The axis lock, as a single-point drag has it: keep whichever the pointer + // has travelled further along and drop the other. + const alongTime = Math.abs(dt * (xOf(1) - xOf(0))); + const alongValue = Math.abs( + (dv * (yOf(range.min) - yOf(range.max))) / (range.max - range.min), + ); + if (alongTime >= alongValue) dv = 0; + else dt = 0; + } else if (!e.altKey) { + // Snap the point under the pointer, and move the set by that same amount. + const others = group.points.filter((_, i) => !group.indices.includes(i)).map((p) => p.t); + dt = + snapLaneTime(group.anchor.t + dt, [...(snapTimes ?? []), ...others], SNAP_SEC) - + group.anchor.t; + } + const times = moving.map((p) => p.t); + const values = moving.map((p) => p.v); + // No member may cross a point that is staying put. Only stationary + // neighbours constrain: two selected points travel together, so the gap + // between them never changes. Per member rather than per end of the group, + // because a box can select a non-contiguous set — the peaks of an envelope + // and not the dip between them. + const gapTo = (step: 1 | -1): number[] => + group.indices.flatMap((i) => { + const neighbour = group.points[i + step]; + if (!neighbour || group.indices.includes(i + step)) return []; + // Short of the neighbour, not onto it — the lane collapses points that + // share a time, and a group drag must not consume what it runs into. + return [Math.max(0, Math.abs(neighbour.t - group.points[i]!.t) - MIN_POINT_GAP_SEC)]; + }); + dt = Math.min( + Math.max(dt, -Math.min(Math.min(...times), ...gapTo(-1))), + Math.min(duration - Math.max(...times), ...gapTo(1)), + ); + dv = Math.min(Math.max(dv, range.min - Math.min(...values)), range.max - Math.max(...values)); + + // No re-sort: clamped to the neighbours, the lane's order cannot change + // under a drag, so the point under the pointer keeps its index. + const next = group.points.map((p, i) => + group.indices.includes(i) ? { ...p, t: p.t + dt, v: p.v + dv } : p, + ); + // The box travels with the points — both axes, or a vertical nudge would + // slide its own points out of the box that caught them and a second nudge + // would move fewer of them. + onRangeSelect?.( + group.selection.t0 + dt, + group.selection.t1 + dt, + group.selection.v0 + dv, + group.selection.v1 + dv, + ); + setHint(`${group.indices.length} points ${dt >= 0 ? "+" : ""}${dt.toFixed(2)}s`); + commitPoints(next, false); + }, + [commitPoints, duration, onRangeSelect, pointAt, range, snapTimes, xOf, yOf], + ); + /** Move the point being dragged, honouring the modifiers held with it. */ const movePoint = useCallback( (e: ReactPointerEvent): void => { if (dragIndex === null) return; + const group = groupDrag.current; + if (group) { + moveGroup(e, group); + return; + } const raw = pointAt(e.clientX, e.clientY); const origin = dragOrigin.current; - let { t, v } = - e.shiftKey && origin ? applyShiftConstraint({ range, origin, raw, xOf, yOf }) : raw; + if (!e.shiftKey) shiftAxis.current = null; + let { t, v } = raw; + if (e.shiftKey && origin) { + shiftAxis.current ??= dominantDragAxis({ origin, raw, xOf, yOf }); + ({ t, v } = applyShiftConstraint({ + range, + origin, + raw, + xOf, + yOf, + axis: shiftAxis.current, + })); + } // Shift is a deliberate free-hand move as much as Alt is, so neither snaps. if (!e.altKey && !e.shiftKey) { const neighbours = lane.points.filter((_, i) => i !== dragIndex).map((p) => p.t); t = snapLaneTime(t, [...(snapTimes ?? []), ...neighbours], SNAP_SEC); } + // A breakpoint cannot cross another in time, and cannot land exactly on one + // either: the lane collapses points that share a `t`, so arriving on top of a + // neighbour deletes it. It stops a hair short instead, which reads as touching + // and keeps both points. Applied after the snap, which can itself put the + // point on a beat past a neighbour. + const floor = (lane.points[dragIndex - 1]?.t ?? -Infinity) + MIN_POINT_GAP_SEC; + const ceiling = (lane.points[dragIndex + 1]?.t ?? Infinity) - MIN_POINT_GAP_SEC; + const held = lane.points[dragIndex]?.t ?? t; + t = + ceiling >= floor + ? Math.min(ceiling, Math.max(floor, Math.min(duration, Math.max(0, t)))) + : // Neighbours closer together than the gap leave nowhere to go, so the + // point stays where it is rather than being flung to one side. + held; const next = lane.points.map((p, i) => (i === dragIndex ? { ...p, t, v } : p)); - // Re-sort so dragging a point past a neighbour behaves, and keep the - // dragged one addressable by following where it landed. - const moved = next[dragIndex]; - next.sort((a, b) => a.t - b.t); - if (moved) setDragIndex(next.indexOf(moved)); setHint(`${formatValue(range, v)} @ ${t.toFixed(2)}s`); commitPoints(next, false); }, - [dragIndex, lane, pointAt, range, commitPoints, snapTimes, xOf, yOf], + [dragIndex, duration, lane, pointAt, range, commitPoints, snapTimes, xOf, yOf, moveGroup], ); - /** Update the live range-drag as the pointer moves, firing `onRangeSelect` - * once it has covered enough pixels to count as an actual range rather - * than a click that should just clear one. */ + /** Update the live box-drag as the pointer moves, firing `onRangeSelect` once + * it has covered enough pixels along EITHER axis to count as an actual box + * rather than a click that should just clear one. */ const moveRangeDrag = useCallback( (e: ReactPointerEvent): void => { if (rangeDrag === null) return; - const raw = pointAt(e.clientX, e.clientY).t; - const clamped = Math.min(duration, Math.max(0, raw)); - const t = e.altKey ? clamped : snapLaneTime(clamped, snapTimes ?? [], SNAP_SEC); - setRangeDrag({ from: rangeDrag.from, to: t }); - if (Math.abs(xOf(t) - xOf(rangeDrag.from)) <= 3) return; + const to = boxCornerAt(e); + const { from } = rangeDrag; + setRangeDrag({ from, to }); + const travelled = Math.max( + Math.abs(xOf(to.t) - xOf(from.t)), + Math.abs(yOf(to.v) - yOf(from.v)), + ); + if (travelled <= 3) return; rangeCrossed.current = true; - onRangeSelect?.(Math.min(rangeDrag.from, t), Math.max(rangeDrag.from, t)); + onRangeSelect?.( + Math.min(from.t, to.t), + Math.max(from.t, to.t), + Math.min(from.v, to.v), + Math.max(from.v, to.v), + ); }, - [rangeDrag, pointAt, duration, snapTimes, xOf, onRangeSelect], + [rangeDrag, boxCornerAt, xOf, yOf, onRangeSelect], ); const onPointerMove = useCallback( (e: ReactPointerEvent): void => { - if (stretch.edge !== null) { - e.stopPropagation(); - // A capture lost without a `pointercancel` — the child it was taken on - // unmounted, or the browser handed the gesture elsewhere — leaves no - // gesture-end event at all, and every later hover would keep retiming. - // A move with no button held is the only signal left that it is over. - if (e.buttons === 0) stretch.cancel(); - else stretch.move(e); - return; - } if (rangeDrag !== null) { e.stopPropagation(); moveRangeDrag(e); return; } - if (curveIndex === null && dragIndex === null) { - stretch.updateHover(e); - return; - } + if (curveIndex === null && dragIndex === null) return; e.stopPropagation(); + const from = pressAt.current; + if (from && Math.hypot(e.clientX - from.x, e.clientY - from.y) > CLICK_SLOP_PX) { + pressTravelled.current = true; + } if (curveIndex !== null) bendSegment(e.clientX, e.clientY); else movePoint(e); }, - [stretch, rangeDrag, moveRangeDrag, curveIndex, dragIndex, bendSegment, movePoint], + [rangeDrag, moveRangeDrag, curveIndex, dragIndex, bendSegment, movePoint], ); /** A sub-threshold press clears the selection rather than leaving a @@ -326,11 +516,6 @@ export function useAutomationLaneGestures({ const endDrag = useCallback( (e: ReactPointerEvent): void => { - if (stretch.edge !== null) { - e.stopPropagation(); - stretch.finish(); - return; - } if (rangeDrag !== null) { e.stopPropagation(); finishRangeDrag(); @@ -338,28 +523,28 @@ export function useAutomationLaneGestures({ } if (dragIndex === null && curveIndex === null) return; e.stopPropagation(); + const index = dragIndex; + const shiftClicked = index !== null && e.shiftKey && !pressTravelled.current; setDragIndex(null); setCurveIndex(null); dragOrigin.current = null; + groupDrag.current = null; + shiftAxis.current = null; + pressAt.current = null; setHint(null); - commitPoints(lane.points, true); - }, - [stretch, rangeDrag, finishRangeDrag, curveIndex, dragIndex, lane, commitPoints], - ); - - /** `pointercancel`: the browser abandoned the gesture, so a stretch reverts - * instead of persisting the partial retime `endDrag` would have committed. - * Anything else ends the way a release ends it. */ - const cancelDrag = useCallback( - (e: ReactPointerEvent): void => { - if (stretch.edge !== null) { - e.stopPropagation(); - stretch.cancel(); + // Shift+click removes the point. Decided on RELEASE, not on the press: Shift + // held through a drag is the axis lock, and acting on the press would take + // that gesture away. A press that never travelled was a click. + if (shiftClicked) { + commitPoints( + lane.points.filter((_, i) => i !== index), + true, + ); return; } - endDrag(e); + commitPoints(lane.points, true); }, - [stretch, endDrag], + [rangeDrag, finishRangeDrag, curveIndex, dragIndex, lane, commitPoints], ); const onDoubleClick = useCallback( @@ -399,38 +584,15 @@ export function useAutomationLaneGestures({ [lane, pointAt, commitPoints, readOnly, hitIndex, segmentIndex], ); - const setEditingText = useCallback((text: string): void => { - setEditing((current) => (current ? { index: current.index, text } : null)); - }, []); - - const cancelEdit = useCallback((): void => setEditing(null), []); - - /** Apply a typed value, or drop the edit when it is not a number. */ - const commitEdit = useCallback((): void => { - const active = editing; - setEditing(null); - if (!active) return; - const typed = Number(active.text); - if (!Number.isFinite(typed)) return; - const clamped = Math.min(range.max, Math.max(range.min, typed)); - commitPoints( - lane.points.map((p, i) => (i === active.index ? { ...p, v: clamped } : p)), - true, - ); - }, [editing, lane, range, commitPoints]); - return { dragIndex, curveIndex, - edgeDrag: stretch.edge, - edgeHover: stretch.hover, hint, hitIndex, segmentIndex, onPointerDown, onPointerMove, endDrag, - cancelDrag, onDoubleClick, editing, setEditingText, diff --git a/packages/studio/src/player/components/useAutomationLanes.test.tsx b/packages/studio/src/player/components/useAutomationLanes.test.tsx index 2003035ea2..17e5cd1322 100644 --- a/packages/studio/src/player/components/useAutomationLanes.test.tsx +++ b/packages/studio/src/player/components/useAutomationLanes.test.tsx @@ -78,6 +78,9 @@ describe("useAutomationLanes", () => { }); it("gives one lane per automated parameter, in draw order", () => { + // Draw order is the spectrum: a lane belonging to an effect that sits at a + // frequency is placed by that frequency, high first, and lanes with none — + // the track's own volume — follow in written order. const automation = JSON.stringify({ version: 1, lanes: [ @@ -87,7 +90,7 @@ describe("useAutomationLanes", () => { ], }); const bound = bindOnce(el({ automation, fxChain: CHAIN })); - expect(bound.lanes.map((l) => l.target)).toEqual(["volume", "fx.n2.frequency", "fx.n2.q"]); + expect(bound.lanes.map((l) => l.target)).toEqual(["fx.n2.frequency", "fx.n2.q", "volume"]); }); it("reads an element with neither attribute as an empty volume lane", () => { diff --git a/packages/studio/src/player/components/useAutomationLanes.ts b/packages/studio/src/player/components/useAutomationLanes.ts index 1862d4d33c..66242dd747 100644 --- a/packages/studio/src/player/components/useAutomationLanes.ts +++ b/packages/studio/src/player/components/useAutomationLanes.ts @@ -10,7 +10,7 @@ * read only, which is also what stops a stray drag from editing the wrong track. */ -import { useCallback, useMemo } from "react"; +import { useCallback, useMemo, useRef } from "react"; import { HF_AUDIO_AUTOMATION_ATTR, serializeAutomation, @@ -27,6 +27,7 @@ import { getTimelineElementIdentity } from "../lib/timelineElementHelpers"; import { usePlayerStore, type TimelineElement } from "../store/playerStore"; import type { AutomationSelection } from "../store/automationSelectionSlice"; import { elementAutomation, elementFxChain } from "./automationLaneData"; +import { createAutomationGestureKeys } from "./automationGestureKeys"; export interface AutomationLaneBinding { automation: HfAutomation; @@ -54,12 +55,13 @@ export interface AutomationLaneBinding { * element's key; it lags only in the window a non-gesture caller can hit. */ commitTargetKey: string | null; - /** This element's active time selection, or null if none / it belongs to a + /** This element's active selection box, or null if none / it belongs to a * different element. */ selection: AutomationSelection | null; - /** Live write while dragging a range on the given lane; does not persist — - * the selection is ephemeral store state, not part of the composition. */ - onRangeSelect(target: string, t0: number, t1: number): void; + /** Live write while dragging a selection box on the given lane; does not + * persist — the selection is ephemeral store state, not part of the + * composition. */ + onRangeSelect(target: string, t0: number, t1: number, v0: number, v1: number): void; onRangeClear(): void; } @@ -68,6 +70,9 @@ export interface UseAutomationLanesResult { } export function useAutomationLanes(): UseAutomationLanesResult { + // One per hook instance, held across renders: a gesture spans many commits and + // they all have to record under the same key for undo to take the whole drag. + const gestureKeys = useRef(createAutomationGestureKeys()); // Optional: the player also runs outside Studio, where there is no edit // session. There the lanes render read-only, which is the right fallback. const domEdit = useDomEditActionsContextOptional(); @@ -97,14 +102,32 @@ export function useAutomationLanes(): UseAutomationLanesResult { const write = (next: HfAutomation, persist: boolean): void => { if (!domEdit || !isSelected) return; const value = next.lanes.length > 0 ? serializeAutomation(next) : ""; + // Every write of one gesture under one key, so undo takes back the whole + // drag rather than the last fragment history happened to keep. + const coalesce = persist ? gestureKeys.current.commit() : gestureKeys.current.live(); // Quiet, not the refreshing commit: releasing a dragged point used to // reload the preview, which restarts every playing track — the same chop // the live write during the drag exists to avoid. Quiet still persists // and still resyncs the selection, so the next edit sees this one. - if (persist) void domEdit.handleDomAttributeQuietCommit(HF_AUDIO_AUTOMATION_ATTR, value); + if (persist) { + void domEdit.handleDomAttributeQuietCommit(HF_AUDIO_AUTOMATION_ATTR, value, coalesce); + } // Dragging a point writes live: no preview refresh, so the composition // does not reload and restart playback on every pixel. - else void domEdit.handleDomAttributeLiveCommit(HF_AUDIO_AUTOMATION_ATTR, value || null); + else { + // Preview only, because a gesture writes on every pointermove: the + // preview and the running audio follow the pointer, and the release + // below is the one write that reaches the file and the undo stack. + void domEdit.handleDomAttributeLiveCommit( + HF_AUDIO_AUTOMATION_ATTR, + value || null, + undefined, + { + coalesce, + previewOnly: true, + }, + ); + } }; return { @@ -121,9 +144,15 @@ export function useAutomationLanes(): UseAutomationLanesResult { readOnly: !domEdit || !isSelected, commitTargetKey: domEdit ? commitTargetKey : null, selection: automationSelection?.elementKey === elementKey ? automationSelection : null, - onRangeSelect: (target, t0, t1) => { - if (!domEdit || !isSelected) return; - setAutomationSelection({ elementKey, target, t0, t1 }); + // Not gated on `isSelected`, unlike the writes above. A selection is + // ephemeral store state, and the drag that draws one on a read-only lane is + // the same press that selects the clip — refusing it here meant the first + // drag on a lane silently did nothing and the author had to drag again. + // Nothing can be written through it while the lane is read-only: every + // consumer resolves the binding again and finds `readOnly`. + onRangeSelect: (target, t0, t1, v0, v1) => { + if (!domEdit) return; + setAutomationSelection({ elementKey, target, t0, t1, v0, v1 }); }, onRangeClear: () => clearAutomationSelection(), }; diff --git a/packages/studio/src/player/lib/automationStoreSync.test.ts b/packages/studio/src/player/lib/automationStoreSync.test.ts new file mode 100644 index 0000000000..78148dbb35 --- /dev/null +++ b/packages/studio/src/player/lib/automationStoreSync.test.ts @@ -0,0 +1,83 @@ +// @vitest-environment happy-dom +import { beforeEach, describe, expect, it } from "vitest"; +import { syncStoredAutomationFromPreview } from "./automationStoreSync"; +import { usePlayerStore, type TimelineElement } from "../store/playerStore"; + +const TWO_POINTS = '{"version":1,"lanes":[{"target":"volume","points":[{"t":0,"v":1}]}]}'; +const RESTORED = + '{"version":1,"lanes":[{"target":"volume","points":[{"t":0,"v":1},{"t":4,"v":0}]}]}'; + +const el = (over: Partial = {}): TimelineElement => ({ + id: "bgm", + key: "bgm", + tag: "audio", + start: 0, + duration: 10, + track: 1, + domId: "bgm", + ...over, +}); + +/** A stand-in preview document carrying the attributes an undo would have written. */ +function previewWith(attrs: Record): Document { + const doc = document.implementation.createHTMLDocument("preview"); + const audio = doc.createElement("audio"); + audio.id = "bgm"; + for (const [name, value] of Object.entries(attrs)) audio.setAttribute(name, value); + doc.body.append(audio); + return doc; +} + +beforeEach(() => { + usePlayerStore.getState().reset(); +}); + +describe("syncStoredAutomationFromPreview", () => { + it("reads back an envelope an undo restored on the preview", () => { + // The bug: a soft undo patches the preview document and re-runs the timeline, but + // the store keeps its own copy of the attributes and that copy is what a lane + // draws — so an undone delete stayed invisible until the page was reloaded. + usePlayerStore.setState({ elements: [el({ automation: TWO_POINTS })] }); + syncStoredAutomationFromPreview(previewWith({ "data-automation": RESTORED })); + expect(usePlayerStore.getState().elements[0]?.automation).toBe(RESTORED); + }); + + it("reads back an undone FX chain too, since a lane's targets come from it", () => { + const chain = '{"version":1,"nodes":[{"type":"lowpass","id":"n1","params":{}}]}'; + usePlayerStore.setState({ elements: [el()] }); + syncStoredAutomationFromPreview(previewWith({ "data-fx-chain": chain })); + expect(usePlayerStore.getState().elements[0]?.fxChain).toBe(chain); + }); + + it("clears an attribute the undo removed", () => { + usePlayerStore.setState({ elements: [el({ automation: TWO_POINTS })] }); + syncStoredAutomationFromPreview(previewWith({})); + expect(usePlayerStore.getState().elements[0]?.automation).toBeUndefined(); + }); + + it("finds the node by data-hf-id when the dom id does not match", () => { + // Studio stamps hf-ids; an element discovered under a suffixed dom id still has + // to resolve, or the sync silently skips it. + const doc = document.implementation.createHTMLDocument("preview"); + const audio = doc.createElement("audio"); + audio.setAttribute("data-hf-id", "hf-snao"); + audio.setAttribute("data-automation", RESTORED); + doc.body.append(audio); + usePlayerStore.setState({ elements: [el({ domId: "bgm-2", hfId: "hf-snao" })] }); + syncStoredAutomationFromPreview(doc); + expect(usePlayerStore.getState().elements[0]?.automation).toBe(RESTORED); + }); + + it("keeps the same array when nothing changed, so nothing re-renders", () => { + usePlayerStore.setState({ elements: [el({ automation: RESTORED })] }); + const before = usePlayerStore.getState().elements; + syncStoredAutomationFromPreview(previewWith({ "data-automation": RESTORED })); + expect(usePlayerStore.getState().elements).toBe(before); + }); + + it("does nothing without a preview document", () => { + usePlayerStore.setState({ elements: [el({ automation: TWO_POINTS })] }); + syncStoredAutomationFromPreview(null); + expect(usePlayerStore.getState().elements[0]?.automation).toBe(TWO_POINTS); + }); +}); diff --git a/packages/studio/src/player/lib/automationStoreSync.ts b/packages/studio/src/player/lib/automationStoreSync.ts new file mode 100644 index 0000000000..02894699df --- /dev/null +++ b/packages/studio/src/player/lib/automationStoreSync.ts @@ -0,0 +1,56 @@ +/** + * Keeping the player store's automation attributes true. + * + * The store is what a lane draws from, and it is populated by element discovery — a + * message from the preview runtime, which only arrives on load. Anything that edits + * an envelope afterwards writes to the preview document and the source file, and the + * store would go on holding the value it was born with until a reload. + * + * One reader, called from the two places a change lands: the resync every dom-edit + * attribute commit already runs, and the soft restore an undo or redo applies. It + * reads the preview rather than being told, because those callers know a file + * changed, not which attribute — and because three separate writers shipped without + * remembering to sync, which is what a single sink prevents. + */ + +import { HF_AUDIO_AUTOMATION_ATTR } from "@hyperframes/core/audio-automation"; +import { HF_AUDIO_FX_ATTR } from "@hyperframes/core/audio-fx"; +import { usePlayerStore, type TimelineElement } from "../store/playerStore"; + +/** The preview node an element stands for, by dom id and then by `data-hf-id`. */ +function previewNodeFor(doc: Document, element: TimelineElement): Element | null { + const domId = element.domId ?? element.id; + const byId = domId ? doc.getElementById(domId) : null; + if (byId) return byId; + return element.hfId ? doc.querySelector(`[data-hf-id="${element.hfId}"]`) : null; +} + +/** + * Re-read every element's automation and FX-chain attributes from the preview + * document, for a change that reached the DOM without going through this store. + * + * That is undo and redo. A soft restore patches the reverted attributes onto the + * live preview and re-runs the timeline — deliberately, so the frame does not blank + * — but the store it does not touch is the one the lanes read, so an undone delete + * stayed invisible until a reload. A full restore already clears the store and waits + * for discovery, so it needs nothing from here. + * + * Reads rather than being told: an undo restores whole files, so the attribute it + * reverted is only known by looking. + */ +export function syncStoredAutomationFromPreview(doc: Document | null | undefined): void { + if (!doc) return; + usePlayerStore.setState((state) => { + let changed = false; + const elements = state.elements.map((element) => { + const node = previewNodeFor(doc, element); + if (!node) return element; + const automation = node.getAttribute(HF_AUDIO_AUTOMATION_ATTR) ?? undefined; + const fxChain = node.getAttribute(HF_AUDIO_FX_ATTR) ?? undefined; + if (automation === element.automation && fxChain === element.fxChain) return element; + changed = true; + return { ...element, automation, fxChain }; + }); + return changed ? { elements } : {}; + }); +} diff --git a/packages/studio/src/player/store/automationSelectionSlice.test.ts b/packages/studio/src/player/store/automationSelectionSlice.test.ts index cb1af6c155..7229fcf3a9 100644 --- a/packages/studio/src/player/store/automationSelectionSlice.test.ts +++ b/packages/studio/src/player/store/automationSelectionSlice.test.ts @@ -2,12 +2,28 @@ import { describe, expect, it } from "vitest"; import { usePlayerStore } from "./playerStore"; describe("automationSelectionSlice", () => { - it("stores one ordered selection and clears it", () => { + it("stores one ordered selection box and clears it", () => { const store = usePlayerStore.getState(); - store.setAutomationSelection({ elementKey: "bgm", target: "volume", t0: 2, t1: 1 }); + // Dragged up and to the left: both axes arrive backwards. + store.setAutomationSelection({ + elementKey: "bgm", + target: "volume", + t0: 2, + t1: 1, + v0: 0.8, + v1: 0.3, + }); const sel = usePlayerStore.getState().automationSelection; - // Ordered on write, so every consumer can assume t0 < t1. - expect(sel).toEqual({ elementKey: "bgm", target: "volume", t0: 1, t1: 2 }); + // Ordered on write, so every consumer can assume t0 <= t1 and v0 <= v1 — + // which is what lets a point test be two range checks rather than four. + expect(sel).toEqual({ + elementKey: "bgm", + target: "volume", + t0: 1, + t1: 2, + v0: 0.3, + v1: 0.8, + }); usePlayerStore.getState().clearAutomationSelection(); expect(usePlayerStore.getState().automationSelection).toBeNull(); }); diff --git a/packages/studio/src/player/store/automationSelectionSlice.ts b/packages/studio/src/player/store/automationSelectionSlice.ts index 8af1440237..d59ca83d20 100644 --- a/packages/studio/src/player/store/automationSelectionSlice.ts +++ b/packages/studio/src/player/store/automationSelectionSlice.ts @@ -1,5 +1,5 @@ /** - * The active time selection on one automation lane. + * The active selection box on one automation lane. * * A store slice, not lane-local state, for the same reason keyframe selection * is one: Delete/copy/paste handlers and the shape menu live outside the lane @@ -13,9 +13,19 @@ export interface AutomationSelection { elementKey: string; /** Lane target: "volume" or "fx..". */ target: string; - /** Clip-local seconds; always t0 < t1 (ordered on write). */ + /** Clip-local seconds; always t0 <= t1 (ordered on write). */ t0: number; t1: number; + /** + * The box's value bounds, in the parameter's own units; always v0 <= v1. + * + * What makes the selection a box rather than a time span: a point is caught + * only if it falls inside both axes. Span operations — copy, paste, shape + * insert, simplify — still read t0/t1 alone, because they act on the envelope + * over a stretch of time rather than on a set of breakpoints. + */ + v0: number; + v1: number; } export interface AutomationSelectionSlice { @@ -31,7 +41,13 @@ export function createAutomationSelectionSlice( automationSelection: null, setAutomationSelection: (sel) => set({ - automationSelection: sel.t0 <= sel.t1 ? sel : { ...sel, t0: sel.t1, t1: sel.t0 }, + automationSelection: { + ...sel, + t0: Math.min(sel.t0, sel.t1), + t1: Math.max(sel.t0, sel.t1), + v0: Math.min(sel.v0, sel.v1), + v1: Math.max(sel.v0, sel.v1), + }, }), clearAutomationSelection: () => set({ automationSelection: null }), }; From 1fcf9803dca155fc11f1358925df79e944c1ea20 Mon Sep 17 00:00:00 2001 From: Vance Ingalls Date: Fri, 7 Aug 2026 13:25:01 -0700 Subject: [PATCH 09/25] feat(studio): show every automated knob at the playhead, and carve as one module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An automated parameter has two values: the number sitting in the chain, which is only the seed a lane replaced, and the number the envelope is on right now. The second is the true one, so the panel shows it — on the carve rack's readouts and on every effect's own fader and number field. A rack that showed the seed stood still while the carve was audibly working. Off the clip it keeps sampling rather than falling back to the stored number: a lane holds its first value backwards and its last forwards, so before the clip starts it already knows what it will open on, and the stored seed is a value nothing will ever play. Showing it made the fader jump the moment the clip came under the playhead. The playhead comes off the liveTime channel, throttled to 30 Hz — the RAF loop deliberately keeps frames out of the store, so a panel watching only the store would sit still for a whole take. PropertyPanel had that subscription inline; it is now one shared hook with two callers. Readouts reserve the width their parameter can need rather than what its current value takes, because an updating value one character narrower shunted everything after it sideways 30 times a second. The carve's effects are presented as one module: an author switched on a carve, and the peaking filters plus the level stage are how it is built, not six things to remove one at a time. Opening it lists every member's settings as readouts, since strength is what sets them. No carve control is offered on a track another track already carves against — that track is the voice, not the bed. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/components/StudioRightPanel.tsx | 17 +- .../src/components/TimelineToolbar.test.tsx | 46 ++ .../studio/src/components/TimelineToolbar.tsx | 17 + .../src/components/editor/PropertyPanel.tsx | 30 +- .../components/editor/PropertyPanelFlat.tsx | 21 +- .../components/editor/audioFxSummary.test.ts | 75 +++ .../src/components/editor/audioFxSummary.ts | 36 + .../editor/propertyPanelAudioFxGroup.test.tsx | 631 +++++++++++++++++- .../editor/propertyPanelAudioFxGroup.tsx | 230 ++++++- .../editor/propertyPanelFxControls.tsx | 17 +- .../editor/propertyPanelFxSection.test.tsx | 137 +++- .../editor/propertyPanelFxSection.tsx | 405 +++++++++-- .../useAutomationSelectionKeyboard.test.tsx | 116 +++- .../hooks/useAutomationSelectionKeyboard.ts | 54 +- .../src/hooks/useDomEditAttributeCommits.ts | 80 ++- .../src/hooks/useDomEditCommits.test.tsx | 38 ++ .../studio/src/hooks/useLivePlayheadTime.ts | 49 ++ .../studio/src/hooks/usePreviewPersistence.ts | 7 + 18 files changed, 1817 insertions(+), 189 deletions(-) create mode 100644 packages/studio/src/components/editor/audioFxSummary.test.ts create mode 100644 packages/studio/src/components/editor/audioFxSummary.ts create mode 100644 packages/studio/src/hooks/useLivePlayheadTime.ts diff --git a/packages/studio/src/components/StudioRightPanel.tsx b/packages/studio/src/components/StudioRightPanel.tsx index 8ee3b9c88d..31ac169dd0 100644 --- a/packages/studio/src/components/StudioRightPanel.tsx +++ b/packages/studio/src/components/StudioRightPanel.tsx @@ -327,6 +327,21 @@ export function StudioRightPanel({ }, [projectId, refreshFileTree, showToast], ); + + /** + * A dial being dragged writes to the preview and stops there. + * + * Every one of these panels previews on each pointermove and commits on + * release. Persisting the moves too put a fragment of the drag in the undo + * stack — and since those writes race, history could not coalesce them + * reliably, so undo took back a sliver of the gesture rather than the gesture. + * The release's own commit is what reaches the file and the undo stack. + */ + const setAttributeWhileDragging = useCallback( + (attr: string, value: string | null) => + handleDomAttributeLiveCommit(attr, value, undefined, { previewOnly: true }), + [handleDomAttributeLiveCommit], + ); const handleHideAllSelected = () => { const { elements } = usePlayerStore.getState(); const keys = timelineKeysForSelections(domEditGroupSelections, elements, activeCompPath); @@ -361,7 +376,7 @@ export function StudioRightPanel({ onSetStyle={handleDomStyleCommit} onSetAttribute={handleDomAttributeCommit} onSetAttributes={handleDomAttributesCommit} - onSetAttributeLive={handleDomAttributeLiveCommit} + onSetAttributeLive={setAttributeWhileDragging} onSetAttributeQuiet={handleDomAttributeQuietCommit} onApplyColorGradingScope={handleApplyColorGradingScope} onSetHtmlAttribute={handleDomHtmlAttributeCommit} diff --git a/packages/studio/src/components/TimelineToolbar.test.tsx b/packages/studio/src/components/TimelineToolbar.test.tsx index 0050dcfc95..ded3f32cbf 100644 --- a/packages/studio/src/components/TimelineToolbar.test.tsx +++ b/packages/studio/src/components/TimelineToolbar.test.tsx @@ -99,3 +99,49 @@ describe("TimelineToolbar — motion path endpoints", () => { act(() => root.unmount()); }); }); + +describe("TimelineToolbar — keyframes on audio tracks", () => { + const clip = (tag: string) => ({ + id: "bgm", + key: "bgm", + tag, + start: 0, + duration: 10, + track: 1, + }); + + /** A session whose selection would otherwise offer the keyframe toggle. */ + function sessionFor(tag: string) { + usePlayerStore.setState({ elements: [clip(tag)], selectedElementId: "bgm", currentTime: 1 }); + const element = document.createElement(tag); + element.id = "bgm"; + return { + domEditSelection: makeSelection("Element", element), + selectedGsapAnimations: [], + handleGsapAddAnimation: vi.fn(), + handleGsapConvertToKeyframes: vi.fn(), + handleGsapRemoveKeyframe: vi.fn(), + } satisfies NonNullable["domEditSession"]>; + } + + it("offers no keyframe toggle for an audio clip", () => { + // An audio clip has no box on the canvas, so there is nothing to move or fade — + // and pressing this seeded a tween from the position properties, which put a + // position lane on a track that has no position. Audio is automated instead. + const { host, root } = renderToolbar(sessionFor("audio")); + const button = host.querySelector( + 'button[aria-label="Add keyframe at playhead"]', + ); + expect(button?.disabled).toBe(true); + act(() => root.unmount()); + }); + + it("still offers it for a visual clip", () => { + const { host, root } = renderToolbar(sessionFor("div")); + const button = host.querySelector( + 'button[aria-label="Add keyframe at playhead"]', + ); + expect(button?.disabled).toBe(false); + act(() => root.unmount()); + }); +}); diff --git a/packages/studio/src/components/TimelineToolbar.tsx b/packages/studio/src/components/TimelineToolbar.tsx index 5391488000..16ea481eda 100644 --- a/packages/studio/src/components/TimelineToolbar.tsx +++ b/packages/studio/src/components/TimelineToolbar.tsx @@ -86,8 +86,22 @@ function resolveKeyframeToggleState( }; } +/** + * Can this element be keyframed at all? + * + * An audio clip cannot. It has no box on the canvas, so there is nothing to move, + * scale or fade — and "add a keyframe" on one seeds a tween from the position + * properties, which produced a position lane on a track that has no position. Audio + * is automated instead: volume and effect parameters, on their own lanes. + */ +function isKeyframeable(element: TimelineElement | undefined): boolean { + return element?.tag !== "audio"; +} + function useKeyframeToggle(session?: DomEditSessionSlice) { const currentTime = usePlayerStore((s) => s.currentTime); + const selectedElementId = usePlayerStore((s) => s.selectedElementId); + const elements = usePlayerStore((s) => s.elements); const sessionRef = useRef(session); sessionRef.current = session; @@ -95,6 +109,9 @@ function useKeyframeToggle(session?: DomEditSessionSlice) { sessionRef as React.RefObject, ); + const selected = elements.find((element) => (element.key ?? element.id) === selectedElementId); + if (!isKeyframeable(selected)) return { ...NO_KEYFRAME_TOGGLE, onToggle: undefined }; + const toggleState = resolveKeyframeToggleState(session, currentTime); return { diff --git a/packages/studio/src/components/editor/PropertyPanel.tsx b/packages/studio/src/components/editor/PropertyPanel.tsx index 7308aca237..9b14d83a6d 100644 --- a/packages/studio/src/components/editor/PropertyPanel.tsx +++ b/packages/studio/src/components/editor/PropertyPanel.tsx @@ -1,5 +1,5 @@ import { scopedElementKey } from "../../hooks/gsapKeyframeCacheHelpers"; -import { memo, useEffect, useMemo, useRef, useState } from "react"; +import { memo, useMemo, useRef, useState } from "react"; import { Move } from "../../icons/SystemIcons"; import { InspectorHeaderActions } from "./InspectorHeaderActions"; import { useStudioShellContext } from "../../contexts/StudioContext"; @@ -29,7 +29,8 @@ import { KeyframeNavigation } from "./KeyframeNavigation"; import { STUDIO_FLAT_INSPECTOR_ENABLED } from "./manualEditingAvailability"; import { PropertyPanelFlat } from "./PropertyPanelFlat"; import { createGsapLivePreview } from "./gsapLivePreview"; -import { usePlayerStore, liveTime } from "../../player"; +import { usePlayerStore } from "../../player"; +import { useLivePlayheadTime } from "../../hooks/useLivePlayheadTime"; import { TimingSection } from "./propertyPanelTimingSection"; import { type PropertyPanelProps } from "./propertyPanelHelpers"; import { GestureRecordPanelButton } from "./GestureRecordControl"; @@ -114,31 +115,14 @@ export const PropertyPanel = memo(function PropertyPanel(props: PropertyPanelPro const { showToast } = useStudioShellContext(); const [clipboardCopied, setClipboardCopied] = useState(false); const clipboardTimerRef = useRef>(undefined); - const storeTime = usePlayerStore((s) => s.currentTime); - const isPlaying = usePlayerStore((s) => s.isPlaying); const timelineElements = usePlayerStore((s) => s.elements); const selectedElementId = usePlayerStore((s) => s.selectedElementId); const selectedElementHidden = isSelectedElementHidden(timelineElements, selectedElementId); const visibilityToggleLabel = selectedElementHidden ? "Show element" : "Hide element"; - const liveTimeRef = useRef(storeTime); - const [, forceRender] = useState(0); - useEffect(() => { - if (!isPlaying) return; - let timerId: ReturnType | 0 = 0; - const unsub = liveTime.subscribe((t) => { - liveTimeRef.current = t; - if (!timerId) - timerId = setTimeout(() => { - timerId = 0; - forceRender((v) => v + 1); - }, 33); - }); - return () => { - unsub(); - if (timerId) clearTimeout(timerId); - }; - }, [isPlaying]); - const currentTime = isPlaying ? liveTimeRef.current : storeTime; + // Live during playback, the store's when paused — see the hook. Shared with the + // audio FX panel, which follows the playhead for the same reason: a value the + // timeline drives has to be shown moving, not frozen at what the attribute says. + const currentTime = useLivePlayheadTime(); const cacheElementKey = element?.id ?? element?.selector ?? ""; const cacheEntry = usePlayerStore((s) => s.keyframeCache.get(cacheElementKey)); diff --git a/packages/studio/src/components/editor/PropertyPanelFlat.tsx b/packages/studio/src/components/editor/PropertyPanelFlat.tsx index cf86039fd2..1c7c20de68 100644 --- a/packages/studio/src/components/editor/PropertyPanelFlat.tsx +++ b/packages/studio/src/components/editor/PropertyPanelFlat.tsx @@ -6,6 +6,7 @@ import { slugifyDesignInput } from "../../utils/designInputTracking"; import { isTextEditableSelection } from "./domEditing"; import type { PropertyPanelFlatProps } from "./propertyPanelFlatProps"; import { formatPxMetricValue } from "./propertyPanelHelpers"; +import { audioFxSummary } from "./audioFxSummary"; import { PropertyPanelFlatHeader } from "./PropertyPanelFlatHeader"; import { PropertyPanelFlatFooter } from "./PropertyPanelFlatFooter"; import { FlatGroupHeader } from "./propertyPanelFlatPrimitives"; @@ -13,11 +14,9 @@ import { FlatTextSection } from "./propertyPanelFlatTextSection"; import { FlatStyleSection } from "./propertyPanelFlatStyleSections"; import { FlatLayoutSection } from "./propertyPanelFlatLayoutSection"; import { FlatMotionSection } from "./propertyPanelFlatMotionSection"; -import { parseAudioFxChain } from "@hyperframes/core/audio-fx"; import { AudioFxGroup } from "./propertyPanelAudioFxGroup.js"; import { useVolumeAutomation } from "./useVolumeAutomation"; import { FlatMediaSection } from "./propertyPanelFlatMediaSection"; -import type { DomEditSelection } from "./domEditing"; import { deriveElementTiming } from "./propertyPanelFlatTimingDerivation"; import { createGsapLivePreview } from "./gsapLivePreview"; import { formatTextFieldPreview } from "./propertyPanelSections"; @@ -536,21 +535,3 @@ export function PropertyPanelFlat({ ); } - -/** Chain length at a glance, so the collapsed group says whether anything is on. */ -function audioFxSummary(element: DomEditSelection): string { - const raw = element.dataAttributes?.["fx-chain"]; - const carve = element.dataAttributes?.["fx-carve"]; - let count = 0; - if (raw) { - try { - count = parseAudioFxChain(raw).nodes.filter((n) => n.enabled !== false).length; - } catch { - return "unreadable"; - } - } - const parts: string[] = []; - if (count > 0) parts.push(`${count} effect${count === 1 ? "" : "s"}`); - if (carve) parts.push("carve"); - return parts.length > 0 ? parts.join(" + ") : "none"; -} diff --git a/packages/studio/src/components/editor/audioFxSummary.test.ts b/packages/studio/src/components/editor/audioFxSummary.test.ts new file mode 100644 index 0000000000..5ba7a87a5a --- /dev/null +++ b/packages/studio/src/components/editor/audioFxSummary.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, it } from "vitest"; +import { audioFxSummary } from "./audioFxSummary"; +import type { DomEditSelection } from "./domEditingTypes"; + +const el = (dataAttributes: Record): DomEditSelection => + ({ dataAttributes }) as unknown as DomEditSelection; + +const chain = (nodes: unknown[]) => JSON.stringify({ version: 1, nodes }); + +describe("audioFxSummary", () => { + it("counts a carve as one module, not as the filters behind it", () => { + // Six bands and a level stage reading "7 effects" is the misreading the + // grouping exists to prevent. + const summary = audioFxSummary( + el({ + "fx-chain": chain([ + { type: "peaking", id: "n1", fromCarve: true, params: { frequency: 400 } }, + { type: "peaking", id: "n2", fromCarve: true, params: { frequency: 1600 } }, + { type: "gain", id: "n3", fromCarve: true, params: { gain: -6 } }, + ]), + "fx-carve": JSON.stringify({ source: "vo", strength: 0.25 }), + }), + ); + expect(summary).toBe("carve"); + }); + + it("counts hand-built effects alongside the module", () => { + expect( + audioFxSummary( + el({ + "fx-chain": chain([ + { type: "peaking", id: "n1", fromCarve: true, params: { frequency: 400 } }, + { type: "lowpass", id: "n2", params: { frequency: 8000 } }, + { type: "delay", id: "n3", params: { time: 200 } }, + ]), + }), + ), + ).toBe("2 effects + carve"); + }); + + it("says how many when there is no carve", () => { + expect(audioFxSummary(el({ "fx-chain": chain([{ type: "lowpass", id: "n1" }]) }))).toBe( + "1 effect", + ); + }); + + it("names a carve that is on but has not compiled to filters yet", () => { + // Switching it on with no voice chosen leaves the control in this section with + // nothing behind it; the summary should still say the section holds one. + expect(audioFxSummary(el({ "fx-carve": JSON.stringify({ source: "", strength: 0.25 }) }))).toBe( + "carve", + ); + }); + + it("ignores bypassed effects, as it always did", () => { + expect( + audioFxSummary( + el({ + "fx-chain": chain([ + { type: "lowpass", id: "n1", enabled: false }, + { type: "delay", id: "n2" }, + ]), + }), + ), + ).toBe("1 effect"); + }); + + it("says none for a track with neither", () => { + expect(audioFxSummary(el({}))).toBe("none"); + }); + + it("says so when the chain cannot be read", () => { + expect(audioFxSummary(el({ "fx-chain": "{not json" }))).toBe("unreadable"); + }); +}); diff --git a/packages/studio/src/components/editor/audioFxSummary.ts b/packages/studio/src/components/editor/audioFxSummary.ts new file mode 100644 index 0000000000..441217a8ab --- /dev/null +++ b/packages/studio/src/components/editor/audioFxSummary.ts @@ -0,0 +1,36 @@ +/** + * What the collapsed Audio FX group says it holds. + * + * It has to describe the rack the author would see on opening it, which counts a + * carve as one module rather than as the filters it compiles to. Six peaking + * bands and a level stage reading "7 effects" invited exactly the misreading the + * grouping exists to prevent — that they are seven things to manage. + */ + +import { parseAudioFxChain } from "@hyperframes/core/audio-fx"; +import type { DomEditSelection } from "./domEditingTypes"; + +export function audioFxSummary(element: DomEditSelection): string { + const raw = element.dataAttributes?.["fx-chain"]; + const carveAttr = element.dataAttributes?.["fx-carve"]; + let handBuilt = 0; + let carveNodes = 0; + if (raw) { + try { + for (const node of parseAudioFxChain(raw).nodes) { + if (node.enabled === false) continue; + if (node.fromCarve) carveNodes += 1; + else handBuilt += 1; + } + } catch { + return "unreadable"; + } + } + const parts: string[] = []; + if (handBuilt > 0) parts.push(`${handBuilt} effect${handBuilt === 1 ? "" : "s"}`); + // One name for the module however many filters are behind it. Named when the + // carve is switched on at all, because the control is in this section whether or + // not it has compiled to anything yet. + if (carveNodes > 0 || carveAttr) parts.push("carve"); + return parts.length > 0 ? parts.join(" + ") : "none"; +} diff --git a/packages/studio/src/components/editor/propertyPanelAudioFxGroup.test.tsx b/packages/studio/src/components/editor/propertyPanelAudioFxGroup.test.tsx index de4153689b..f9cad3b146 100644 --- a/packages/studio/src/components/editor/propertyPanelAudioFxGroup.test.tsx +++ b/packages/studio/src/components/editor/propertyPanelAudioFxGroup.test.tsx @@ -4,6 +4,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { createRoot } from "react-dom/client"; import { AudioFxGroup } from "./propertyPanelAudioFxGroup.js"; import type { DomEditSelection } from "./domEditingTypes"; +import { liveTime, usePlayerStore } from "../../player"; (globalThis as unknown as { IS_REACT_ACT_ENVIRONMENT: boolean }).IS_REACT_ACT_ENVIRONMENT = true; @@ -174,13 +175,7 @@ describe("AudioFxGroup carve", () => { ], }); - const carveOn = JSON.stringify({ - source: "vo", - maxCutDb: 6, - bands: 3, - q: 1.4, - intelligibilityBias: 0.7, - }); + const carveOn = JSON.stringify({ source: "vo", strength: 0.5, dynamic: false }); const carveToggle = (host: HTMLElement): HTMLButtonElement => { const block = host.querySelector(".hf-fx-carve")!; @@ -232,7 +227,8 @@ describe("AudioFxGroup carve", () => { act(() => { // React's value tracker swallows a plain assignment, so go through the // prototype setter the way the other panel tests do. - Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, "value")?.set?.call(dial, "0.5"); + // A different value than the carve holds; setting the same one is not a change. + Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, "value")?.set?.call(dial, "0.8"); dial?.dispatchEvent(new Event("input", { bubbles: true })); }); expect(onSetAttributeLive.mock.calls.map((c) => c[0])).toEqual(["data-fx-carve"]); @@ -246,7 +242,373 @@ describe("AudioFxGroup carve", () => { act(() => carveToggle(host).click()); const write = onSetAttributeQuiet.mock.calls.find((c) => c[0] === "data-fx-carve"); expect(write).toBeTruthy(); - expect(JSON.parse(String(write![1])).bands).toBeGreaterThan(0); + expect(JSON.parse(String(write![1])).strength).toBeGreaterThan(0); + }); +}); + +describe("AudioFxGroup dynamic carve", () => { + const carvedChain = JSON.stringify({ + version: 1, + nodes: [{ type: "lowpass", id: "n1", params: { frequency: 400, q: 0.9, poles: "2" } }], + }); + // Strength 0 carves frequencies only — no level ducking — so the spectral + // cases measure just the spectral half. A case that wants the duck raises it. + const settings = (dynamic: boolean, over: Record = {}) => + JSON.stringify({ source: "vo", strength: 0, dynamic, ...over }); + + /** The value written for one attribute, whatever order the writes landed in. */ + const writeFor = (calls: unknown[][], attr: string) => + JSON.parse(String(calls.find((c) => c[0] === attr)![1])); + + /** Choose a voice track the way the select does. */ + const pickSource = (host: HTMLElement, id: string) => { + const select = host.querySelector(".hf-fx-carve select")!; + Object.getOwnPropertyDescriptor(HTMLSelectElement.prototype, "value")?.set?.call(select, id); + select.dispatchEvent(new Event("change", { bubbles: true })); + }; + + const dynamicBox = (host: HTMLElement) => + host.querySelector(".hf-fx-carve-dynamic")!; + + /** A voice with a pause in it, decoded through a stubbed offline context. */ + function stubDecode(): void { + const sampleRate = 48000; + const data = new Float32Array(sampleRate * 4); + for (let i = 0; i < data.length; i++) { + const t = i / sampleRate; + data[i] = t > 1 && t < 3 ? 0.7 * Math.sin(2 * Math.PI * 1000 * t) : 0; + } + vi.stubGlobal( + "fetch", + vi.fn(async () => ({ arrayBuffer: async () => new ArrayBuffer(8) })), + ); + vi.stubGlobal( + "OfflineAudioContext", + class { + decodeAudioData = async () => ({ sampleRate, getChannelData: () => data }); + }, + ); + } + + afterEach(() => vi.unstubAllGlobals()); + + it("records the choice in the carve settings", () => { + const { host, onSetAttributeQuiet } = mount({ + "fx-chain": carvedChain, + "fx-carve": settings(false), + }); + act(() => dynamicBox(host).click()); + const write = onSetAttributeQuiet.mock.calls.find((c) => c[0] === "data-fx-carve"); + expect(JSON.parse(String(write![1])).dynamic).toBe(true); + }); + + it("automates the carve filters' gain from the voice, in the bed's own time", async () => { + stubDecode(); + // Voice starts 10s into the composition, bed at 0: the envelope is measured + // against the voice but read from the start of the bed, so it has to shift. + const { host, onSetAttributeQuiet } = mount({ + "fx-chain": carvedChain, + // No source yet: picking one is what applies the carve. + "fx-carve": settings(true, { source: "" }), + start: "0", + }); + const vo = document.getElementById("vo")!; + vo.setAttribute("data-start", "10"); + vo.setAttribute("src", "voice.wav"); + await act(async () => { + pickSource(host, "vo"); + }); + + // Chain first, then automation: a lane naming a node the chain does not + // carry yet is dropped when it is read back. + const order = onSetAttributeQuiet.mock.calls.map((c) => c[0]); + // The settings land first, then the filters they imply, then the envelopes. + expect(order.indexOf("data-fx-chain")).toBeLessThan(order.indexOf("data-automation")); + + const carved = writeFor(onSetAttributeQuiet.mock.calls, "data-fx-chain").nodes; + const carveNode = carved.find((n: { fromCarve?: boolean }) => n.fromCarve); + expect(carveNode.id).toBeTruthy(); + + const lanes = writeFor(onSetAttributeQuiet.mock.calls, "data-automation").lanes; + const lane = lanes.find((l: { target: string }) => l.target === `fx.${carveNode.id}.gain`) as { + points: { t: number; v: number }[]; + }; + expect(lane).toBeTruthy(); + // Flat at the bed's own start, before the voice exists at all. + expect(lane.points[0]).toMatchObject({ t: 0, v: 0 }); + // The voice's pause is at 0-1s of its own clip, so 10-11s of the bed's. + expect(lane.points.find((p) => p.t > 10.5 && p.t < 11)?.v ?? 0).toBe(0); + // And it cuts once the voice speaks, a second later. Depth is per band and + // relative to that band's own peak in the voice, so the invariant is that the + // envelope gets most of the way to what the analysis put on the node — not a + // fixed number of dB, which changes with the band the analysis chose. + const bandGain = Number(carveNode.params?.gain ?? 0); + // At least half the depth the analysis put on the node; the exact floor + // depends on which band it chose and how the envelope was thinned. + expect(Math.min(...lane.points.map((p) => p.v))).toBeLessThanOrEqual(bandGain * 0.5); + // Ends back at no cut, so the bed is not left dipped for the rest of the clip. + expect(lane.points.at(-1)!.v).toBe(0); + }); + + it("adds a gain stage that ducks the bed under the voice, automated when dynamic", async () => { + // Carving frequencies cannot beat a bed that is simply louder than the + // voice. The level half rides a gain node the carve owns, so the track's own + // volume lane is left alone. + stubDecode(); + const { host, onSetAttributeQuiet } = mount({ + "fx-chain": carvedChain, + "fx-carve": settings(true, { strength: 1, source: "" }), + start: "0", + }); + const vo = document.getElementById("vo")!; + vo.setAttribute("data-start", "0"); + vo.setAttribute("src", "voice.wav"); + // The bed is measured too — "how far over the voice is it" needs both. + document.getElementById("bed")!.setAttribute("src", "bed.m4a"); + await act(async () => { + pickSource(host, "vo"); + }); + + const nodes = writeFor(onSetAttributeQuiet.mock.calls, "data-fx-chain").nodes; + const gain = nodes.find((n: { type: string }) => n.type === "gain"); + expect(gain).toBeTruthy(); + expect(gain.fromCarve).toBe(true); + // Dynamic hands the value to the envelope, so the static one stays at unity. + expect(gain.params.gain).toBe(0); + + const lanes = writeFor(onSetAttributeQuiet.mock.calls, "data-automation").lanes; + const duckLane = lanes.find((l: { target: string }) => l.target === `fx.${gain.id}.gain`); + expect(duckLane).toBeTruthy(); + expect(Math.min(...duckLane.points.map((p: { v: number }) => p.v))).toBeLessThan(0); + // Every carved band gets an envelope reaching that band's own analysed depth. + for (const node of nodes.filter((n: { type: string }) => n.type === "peaking")) { + const lane = lanes.find((l: { target: string }) => l.target === `fx.${node.id}.gain`) as + | { points: { v: number }[] } + | undefined; + expect(lane, `band ${node.id} has no envelope`).toBeTruthy(); + const deepest = Math.min(...lane!.points.map((p) => p.v)); + expect(deepest).toBeLessThanOrEqual(0); + expect(deepest).toBeGreaterThanOrEqual(node.params.gain - 0.2); + expect(deepest).toBeLessThanOrEqual(node.params.gain * 0.5); + } + // The author's own volume lane is not something a carve gets to touch. + expect(lanes.some((l: { target: string }) => l.target === "volume")).toBe(false); + }); + + it("holds one measured value when the carve is not dynamic", async () => { + stubDecode(); + const { host, onSetAttributeQuiet } = mount({ + "fx-chain": carvedChain, + "fx-carve": settings(false, { strength: 1, source: "" }), + start: "0", + }); + const vo = document.getElementById("vo")!; + vo.setAttribute("data-start", "0"); + vo.setAttribute("src", "voice.wav"); + // The bed is measured too — "how far over the voice is it" needs both. + document.getElementById("bed")!.setAttribute("src", "bed.m4a"); + await act(async () => { + pickSource(host, "vo"); + }); + const nodes = writeFor(onSetAttributeQuiet.mock.calls, "data-fx-chain").nodes; + const gain = nodes.find((n: { type: string }) => n.type === "gain"); + expect(gain.params.gain).toBeLessThan(0); + // Nothing to schedule: a static carve is a value, not an envelope. + expect(onSetAttributeQuiet.mock.calls.some((c) => c[0] === "data-automation")).toBe(false); + }); + + it("carves frequencies only when the duck is off", async () => { + stubDecode(); + const { host, onSetAttributeQuiet } = mount({ + "fx-chain": carvedChain, + "fx-carve": settings(true, { strength: 0, source: "" }), + start: "0", + }); + document.getElementById("vo")!.setAttribute("src", "voice.wav"); + document.getElementById("bed")!.setAttribute("src", "bed.m4a"); + await act(async () => { + pickSource(host, "vo"); + }); + const nodes = writeFor(onSetAttributeQuiet.mock.calls, "data-fx-chain").nodes; + expect(nodes.some((n: { type: string }) => n.type === "gain")).toBe(false); + }); + + it("applies as soon as a voice track is picked, with no second step", async () => { + // A carve with a source and no filters is a setting nobody applied. Choosing + // the voice is the whole gesture. + stubDecode(); + const { host, onSetAttributeQuiet } = mount({ + "fx-chain": JSON.stringify({ version: 1, nodes: [] }), + "fx-carve": JSON.stringify({ source: "", strength: 0.25, dynamic: true }), + start: "0", + }); + document.getElementById("vo")!.setAttribute("src", "voice.wav"); + document.getElementById("bed")!.setAttribute("src", "bed.m4a"); + await act(async () => { + pickSource(host, "vo"); + }); + const written = onSetAttributeQuiet.mock.calls.map((c) => c[0]); + expect(written).toEqual(["data-fx-carve", "data-fx-chain", "data-automation"]); + const nodes = JSON.parse( + String(onSetAttributeQuiet.mock.calls.find((c) => c[0] === "data-fx-chain")![1]), + ).nodes; + expect(nodes.every((n: { fromCarve?: boolean }) => n.fromCarve)).toBe(true); + }); + + it("re-applies when dynamic is switched on, not just when strength moves", async () => { + stubDecode(); + const carvedAlready = JSON.stringify({ + version: 1, + nodes: [ + { + type: "peaking", + id: "n1", + fromCarve: true, + params: { frequency: 1000, gain: -6, q: 1.4 }, + }, + ], + }); + const { host, onSetAttributeQuiet } = mount({ + "fx-chain": carvedAlready, + "fx-carve": settings(false, { strength: 0.25 }), + start: "0", + }); + document.getElementById("vo")!.setAttribute("src", "voice.wav"); + document.getElementById("bed")!.setAttribute("src", "bed.m4a"); + await act(async () => { + host.querySelector(".hf-fx-carve-dynamic")!.click(); + }); + // Static and dynamic are different chains, so the switch has to rebuild them. + expect(onSetAttributeQuiet.mock.calls.some((c) => c[0] === "data-fx-chain")).toBe(true); + }); + + it("re-applies an existing carve when strength moves", async () => { + // Strength is the whole control surface, so it has to act on what is already + // applied. Left to the button alone, a carve kept the filters and envelopes + // its old strength produced and the knob silently described nothing. + stubDecode(); + const carvedAlready = JSON.stringify({ + version: 1, + nodes: [ + { + type: "peaking", + id: "n1", + fromCarve: true, + params: { frequency: 1000, gain: -6, q: 1.4 }, + }, + { type: "gain", id: "n2", fromCarve: true, params: { gain: -6 } }, + ], + }); + const { host, onSetAttributeQuiet } = mount({ + "fx-chain": carvedAlready, + "fx-carve": settings(true, { strength: 0.25 }), + start: "0", + }); + document.getElementById("vo")!.setAttribute("src", "voice.wav"); + document.getElementById("bed")!.setAttribute("src", "bed.m4a"); + + const dial = host.querySelector(".hf-fx-carve input[type=range]")!; + await act(async () => { + Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, "value")?.set?.call(dial, "1"); + dial.dispatchEvent(new Event("input", { bubbles: true })); + dial.dispatchEvent(new PointerEvent("pointerup", { bubbles: true })); + }); + + const written = onSetAttributeQuiet.mock.calls.map((c) => c[0]); + expect(written).toContain("data-fx-carve"); + // The settings land first, then the filters they imply, then the envelopes. + expect(written.indexOf("data-fx-carve")).toBeLessThan(written.indexOf("data-fx-chain")); + expect(written.indexOf("data-fx-chain")).toBeLessThan(written.indexOf("data-automation")); + + const chainWrite = onSetAttributeQuiet.mock.calls.find((c) => c[0] === "data-fx-chain"); + const nodes = JSON.parse(String(chainWrite![1])).nodes; + // Full strength: deeper than the 6 dB the quarter-strength carve had. + const deepest = Math.min( + ...nodes + .filter((n: { type: string }) => n.type === "peaking") + .map((n: { params: { gain: number } }) => n.params.gain), + ); + expect(deepest).toBeLessThan(-6); + }); + + it("does nothing but record the setting while no voice track is chosen", async () => { + // There is nothing to listen to, so there is nothing to derive. This is the + // one case that only writes the setting now that the apply button is gone. + stubDecode(); + const { host, onSetAttributeQuiet } = mount({ + "fx-chain": JSON.stringify({ version: 1, nodes: [] }), + "fx-carve": settings(true, { strength: 0.25, source: "" }), + start: "0", + }); + const dial = host.querySelector(".hf-fx-carve input[type=range]")!; + await act(async () => { + Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, "value")?.set?.call(dial, "1"); + dial.dispatchEvent(new Event("input", { bubbles: true })); + dial.dispatchEvent(new PointerEvent("pointerup", { bubbles: true })); + }); + expect(onSetAttributeQuiet.mock.calls.map((c) => c[0])).toEqual(["data-fx-carve"]); + }); + + it("does not re-analyse on every pixel of a drag", async () => { + // Only the release re-applies. Analysing per pointermove would decode both + // tracks on each pixel. + stubDecode(); + const carvedAlready = JSON.stringify({ + version: 1, + nodes: [ + { + type: "peaking", + id: "n1", + fromCarve: true, + params: { frequency: 1000, gain: -6, q: 1.4 }, + }, + ], + }); + const { host, onSetAttributeQuiet, onSetAttributeLive } = mount({ + "fx-chain": carvedAlready, + "fx-carve": settings(true, { strength: 0.25 }), + start: "0", + }); + document.getElementById("vo")!.setAttribute("src", "voice.wav"); + const dial = host.querySelector(".hf-fx-carve input[type=range]")!; + await act(async () => { + for (const v of ["0.4", "0.6", "0.8"]) { + Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, "value")?.set?.call(dial, v); + dial.dispatchEvent(new Event("input", { bubbles: true })); + } + }); + expect(onSetAttributeLive.mock.calls.every((c) => c[0] === "data-fx-carve")).toBe(true); + expect(onSetAttributeQuiet.mock.calls.some((c) => c[0] === "data-fx-chain")).toBe(false); + }); + + it("drops the envelopes when dynamic is switched back off", async () => { + // An automated gain ignores the panel's depth, so leaving the lanes behind + // would keep the filters following a voice with nothing saying they do. + const automation = JSON.stringify({ + version: 1, + lanes: [ + { target: "fx.n2.gain", points: [{ t: 0, v: 0 }] }, + { target: "volume", points: [{ t: 0, v: 1 }] }, + ], + }); + const withCarveNode = JSON.stringify({ + version: 1, + nodes: [ + { type: "peaking", id: "n2", fromCarve: true, params: { frequency: 1000, gain: -6 } }, + ], + }); + const { host, onSetAttributeQuiet } = mount({ + "fx-chain": withCarveNode, + "fx-carve": settings(true), + automation, + }); + await act(async () => { + dynamicBox(host).click(); + }); + const write = onSetAttributeQuiet.mock.calls.find((c) => c[0] === "data-automation"); + expect(write).toBeTruthy(); + const lanes = JSON.parse(String(write![1])).lanes; + expect(lanes.map((l: { target: string }) => l.target)).toEqual(["volume"]); }); }); @@ -298,6 +660,72 @@ describe("AudioFxGroup successive edits", () => { }); describe("AudioFxGroup carve visibility", () => { + /** + * A carve is a relationship: a bed is carved *against* a voice. The voice is the + * other end of it, so offering the same control there is offering to carve a + * track against itself by proxy — and switching it on left a setting that could + * never do anything. + */ + it("does not offer carve on a track another track is carving against", () => { + const bed = document.createElement("audio"); + bed.id = "bed"; + bed.setAttribute("src", "bed.m4a"); + bed.setAttribute("data-fx-carve", JSON.stringify({ source: "vo", strength: 0.25 })); + document.body.append(bed); + const voice = document.createElement("audio"); + voice.id = "vo"; + voice.setAttribute("src", "voice.wav"); + document.body.append(voice); + + const host = document.createElement("div"); + document.body.append(host); + const selection = { + dataAttributes: {}, + id: "vo", + element: voice, + } as unknown as DomEditSelection; + act(() => { + createRoot(host).render( + , + ); + }); + expect(host.querySelector(".hf-fx-carve")).toBeNull(); + }); + + it("still offers it on the bed doing the carving", () => { + const bed = document.createElement("audio"); + bed.id = "bed"; + bed.setAttribute("src", "bed.m4a"); + bed.setAttribute("data-fx-carve", JSON.stringify({ source: "vo", strength: 0.25 })); + document.body.append(bed); + const voice = document.createElement("audio"); + voice.id = "vo"; + voice.setAttribute("src", "voice.wav"); + document.body.append(voice); + + const host = document.createElement("div"); + document.body.append(host); + const selection = { + dataAttributes: { "fx-carve": JSON.stringify({ source: "vo", strength: 0.25 }) }, + id: "bed", + element: bed, + } as unknown as DomEditSelection; + act(() => { + createRoot(host).render( + , + ); + }); + expect(host.querySelector(".hf-fx-carve")).not.toBeNull(); + }); + it("offers carve when the composition holds another audio track", () => { const { host } = mount({ "fx-chain": CHAIN }); expect(host.querySelector(".hf-fx-carve")).toBeTruthy(); @@ -356,3 +784,188 @@ describe("AudioFxGroup deleting an effect", () => { expect(onSetAttributeQuiet.mock.calls.some((c) => c[0] === "data-automation")).toBe(false); }); }); + +describe("AudioFxGroup carve module readouts", () => { + /** + * A carve on a bed running 0–10 s, with a lane that ramps its 400 Hz band from + * no cut to −6 dB across the first five seconds. The stored gain is −1, which is + * deliberately not a value the lane ever passes through: it stands in for the + * seed a lane leaves behind, so a readout showing it can only mean the playhead + * was not consulted. + */ + const carved = { + start: "0", + duration: "10", + "fx-chain": JSON.stringify({ + version: 1, + nodes: [ + { + type: "peaking", + id: "n1", + fromCarve: true, + params: { frequency: 400, gain: -1, q: 1.4 }, + }, + ], + }), + automation: JSON.stringify({ + version: 1, + lanes: [ + { + target: "fx.n1.gain", + points: [ + { t: 0, v: 0 }, + { t: 5, v: -6 }, + ], + }, + ], + }), + "fx-carve": JSON.stringify({ source: "vo", strength: 0.25, dynamic: true }), + }; + + /** Park the playhead somewhere, paused — a scrub is the same question as playback. */ + const seek = (time: number) => { + act(() => { + usePlayerStore.setState({ currentTime: time, isPlaying: false }); + }); + }; + + const gainReadout = (host: HTMLElement): HTMLElement | null => { + for (const span of Array.from(host.querySelectorAll(".hf-fx-carve-member span"))) { + if (span.textContent?.startsWith("Gain")) return span; + } + return null; + }; + + const openModule = (host: HTMLElement) => { + const head = host.querySelector(".hf-fx-carve-module .hf-fx-node-name"); + act(() => head?.click()); + }; + + afterEach(() => { + usePlayerStore.setState({ currentTime: 0, isPlaying: false }); + }); + + it("shows the envelope's value at the playhead, not the stored seed", () => { + seek(2.5); + const { host } = mount(carved); + openModule(host); + const gain = gainReadout(host); + // Halfway along a 0 → −6 dB ramp. + expect(gain?.textContent).toContain("-3 dB"); + expect(gain?.hasAttribute("data-automation-live")).toBe(true); + }); + + it("follows the playhead as it moves", () => { + seek(0); + const { host } = mount(carved); + openModule(host); + expect(gainReadout(host)?.textContent).toContain("0 dB"); + seek(5); + expect(gainReadout(host)?.textContent).toContain("-6 dB"); + seek(1); + expect(gainReadout(host)?.textContent).toContain("-1.2 dB"); + }); + + it("follows the transport during playback, off the live-time channel", async () => { + // The RAF loop deliberately keeps every frame out of the store — it notifies + // `liveTime` instead — so a panel that only watched the store would sit still + // for the whole take and then jump when playback stopped. + seek(0); + const { host } = mount(carved); + openModule(host); + act(() => { + usePlayerStore.setState({ isPlaying: true }); + }); + act(() => liveTime.notify(4)); + // Throttled to 30 Hz rather than rendered per frame, so the readout lands on + // the next tick and not in this one. + await act(async () => { + await new Promise((resolve) => setTimeout(resolve, 60)); + }); + expect(gainReadout(host)?.textContent).toContain("-4.8 dB"); + + act(() => liveTime.notify(5)); + await act(async () => { + await new Promise((resolve) => setTimeout(resolve, 60)); + }); + expect(gainReadout(host)?.textContent).toContain("-6 dB"); + }); + + it("reserves the same width whatever the value reads", () => { + // The readout updates 30 times a second while the transport runs, and a value + // one character narrower shunts everything after it sideways. So the width comes + // from what the parameter CAN read, not from what it currently does. + seek(2.5); + const { host } = mount(carved); + openModule(host); + const widthAt = (): string | undefined => + gainReadout(host)?.querySelector(".tabular-nums")?.style.minWidth; + const narrow = widthAt(); + expect(narrow).toBe("8ch"); // -40..40 dB at one decimal: "-12.5 dB" + seek(5); // -6 dB — two characters shorter than -3 dB was + expect(gainReadout(host)?.textContent).toContain("-6 dB"); + expect(widthAt()).toBe(narrow); + }); + + it("moves a hand-built effect's own slider and number, not just the carve rack", () => { + // Every automated value follows its lane, whatever put the effect there. This + // one is a delay the author added and automated by hand: its control is locked + // (the lane owns the value), so the fader has no drag to fight and can simply + // show the truth. + const { host } = mount({ + start: "0", + duration: "10", + "fx-chain": JSON.stringify({ + version: 1, + nodes: [{ type: "delay", id: "n1", params: { time: 250, feedback: 0.35, mix: 0.4 } }], + }), + automation: JSON.stringify({ + version: 1, + lanes: [ + { + target: "fx.n1.mix", + points: [ + { t: 0, v: 0 }, + { t: 4, v: 1 }, + ], + }, + ], + }), + }); + const mixRow = rowFor(host, "Mix"); + const number = mixRow?.querySelector(".hf-fx-number"); + const slider = mixRow?.querySelector(".hf-fx-slider"); + expect(number?.disabled).toBe(true); // the lane owns it + + seek(1); // a quarter along a 0 → 1 ramp + expect(Number(number?.value)).toBeCloseTo(0.25, 2); + const quarter = Number(slider?.value); + + seek(3); + expect(Number(number?.value)).toBeCloseTo(0.75, 2); + expect(Number(slider?.value)).toBeGreaterThan(quarter); + }); + + it("shows the lane's own edge value off the clip, not the stored seed", () => { + // Past the bed's end, where a lane holds its last value — which is what would + // play if the playhead came back. The stored -1 dB is a seed the lane replaced + // and nothing will ever use it, so putting it on screen only made the fader + // jump when the clip came under the playhead. + seek(20); + const { host } = mount(carved); + openModule(host); + const gain = gainReadout(host); + expect(gain?.textContent).toContain("-6 dB"); // the ramp's last point + expect(gain?.hasAttribute("data-automation-live")).toBe(true); + expect(gain?.hasAttribute("data-automated")).toBe(true); + }); + + it("holds the lane's first value before the clip starts", () => { + // Same rule at the other end: a lane opens on its first point, so that is what + // the fader should read while the playhead is still upstream of the clip. + seek(-5); + const { host } = mount(carved); + openModule(host); + expect(gainReadout(host)?.textContent).toContain("0 dB"); + }); +}); diff --git a/packages/studio/src/components/editor/propertyPanelAudioFxGroup.tsx b/packages/studio/src/components/editor/propertyPanelAudioFxGroup.tsx index 0a04892e97..9401d6aae7 100644 --- a/packages/studio/src/components/editor/propertyPanelAudioFxGroup.tsx +++ b/packages/studio/src/components/editor/propertyPanelAudioFxGroup.tsx @@ -9,19 +9,30 @@ import { useState } from "react"; import { + defaultAudioFxParams, HF_AUDIO_FX_ATTR, + mintAudioFxNodeId, parseAudioFxChain, serializeAudioFxChain, type HfAudioFxChain, + type HfAudioFxNode, } from "@hyperframes/core/audio-fx"; import { analyseCarveBands, + analyseCarveDuck, + analyseCarveDynamics, carveBandsToChain, + carveProfile, HF_AUDIO_CARVE_ATTR, normalizeCarveSettings, type HfCarveSettings, } from "@hyperframes/core/audio-carve"; -import { fxAutomationTarget, type HfAutomation } from "@hyperframes/core/audio-automation"; +import { + fxAutomationTarget, + sampleAutomationLane, + type HfAutomation, + type HfAutomationLane, +} from "@hyperframes/core/audio-automation"; import { automatedTargetsOf, automationAttrValue, @@ -32,6 +43,7 @@ import { withSeededLane, } from "./propertyPanelAutomation"; import type { DomEditSelection } from "./domEditingTypes"; +import { useLivePlayheadTime } from "../../hooks/useLivePlayheadTime"; /** * Rate the carve source is decoded at. Analysis is self-consistent because it @@ -40,6 +52,22 @@ import type { DomEditSelection } from "./domEditingTypes"; const DECODE_SAMPLE_RATE = 48000; import { FxSection, type AudioTrackOption } from "./propertyPanelFxSection.js"; +/** Where a clip starts on the timeline, in seconds. */ +function clipStart(value: string | null | undefined): number { + const n = Number(value); + return Number.isFinite(n) ? n : 0; +} + +/** Lanes belonging to nodes the carve generated, which a re-run replaces. */ +function withoutCarveLanes(automation: HfAutomation, chain: HfAudioFxChain): HfAutomation { + const prefixes = chain.nodes.filter((n) => n.fromCarve && n.id).map((n) => `fx.${n.id}.`); + if (prefixes.length === 0) return automation; + return { + version: automation.version, + lanes: automation.lanes.filter((lane) => !prefixes.some((p) => lane.target.startsWith(p))), + }; +} + /** * Bridges the FX panel to the element/attribute world. Chain and carve are * serialised onto the element the way colour grading carries its config, so @@ -80,6 +108,37 @@ export function AudioFxGroup({ const automation = readPanelAutomation(element.dataAttributes?.["automation"], chain); const automatedTargets = automatedTargetsOf(automation); + /** + * What every automated knob is worth at the playhead, so the rack shows the + * value the audio is actually using rather than the one the attribute stores. + * + * An automated parameter has two values: the number sitting in the chain, which + * is only a seed once a lane exists, and the number the envelope is on right now. + * The second is the true one, and a rack that shows the first reads as broken + * during playback — the carve is visibly working and the readouts do not move. + * + * Sampled while paused as well, because the same argument applies to a scrub: + * the playhead is somewhere, and the envelope has a value there. + * + * Sampled off the clip too, which is not obvious. A lane holds its first value + * backwards and its last value forwards, so before the clip starts it already + * knows what it will open on — while the stored number is a seed the lane + * replaced and nothing will ever play. Showing that seed put a value on screen + * that the automation never uses, and made the fader jump the moment the clip + * came under the playhead. + */ + const playhead = useLivePlayheadTime(); + const localTime = playhead - clipStart(element.dataAttributes?.["start"]); + const liveAutomationValues = ((): Map => { + const values = new Map(); + for (const lane of automation.lanes) { + const range = resolveAutomationRange(lane.target, chain); + if (!range) continue; + values.set(lane.target, sampleAutomationLane(lane, localTime, range.scale)); + } + return values; + })(); + // Written through the live path on purpose. It persists to the source just // like the refreshing one, but skips the preview reload — and a reload // restarts every playing track, which is heard as the audio chopping. The @@ -126,6 +185,18 @@ export function AudioFxGroup({ * commit, which does not exist yet. */ const setCarve = async (next: HfCarveSettings | null): Promise => { + // Envelopes the carve wrote outlive it otherwise, and an automated gain + // ignores the panel's own depth — so switching dynamic off would leave the + // filters still following the voice with nothing saying they do. + if (!next || (carve?.dynamic && !next.dynamic)) { + const carriedOver = withoutCarveLanes(automation, chain); + if (carriedOver.lanes.length !== automation.lanes.length) { + await onSetAttributeQuiet( + HF_AUDIO_AUTOMATION_ATTR, + automationAttrValue(carriedOver) || null, + ); + } + } if (!next) { const kept = chain.nodes.filter((n) => !n.fromCarve); if (kept.length !== chain.nodes.length) { @@ -136,6 +207,20 @@ export function AudioFxGroup({ } } await onSetAttributeQuiet(HF_AUDIO_CARVE_ATTR, next ? JSON.stringify(next) : null); + + // Every setting here describes the filters, so changing one rebuilds them. + // There is no apply button: a carve naming a voice with no filters behind it + // is a setting nobody applied, and the panel already knows everything it needs + // to. Picking the voice is what starts it; strength and dynamic re-derive what + // is already there. A carve with no source yet has nothing to analyse. + const changed = + next && + next.source && + (!carve || + next.source !== carve.source || + next.strength !== carve.strength || + next.dynamic !== carve.dynamic); + if (next && changed) await analyse(next); }; /** Every lane belonging to a node that is going away. */ @@ -157,6 +242,32 @@ export function AudioFxGroup({ } })(); + /** + * Is some other track carving against this one? + * + * A carve is a relationship — a bed is carved against a voice — and the voice is + * the far end of it. Offering the same control there offers to carve a track + * against itself by proxy, and switching it on left a setting with no source it + * could legally name. Read off the other elements' own carve attributes, because + * that is where the relationship is recorded. + */ + const carvedAgainstBy = ((): string | null => { + const doc = element.element?.ownerDocument; + if (!doc || !element.id) return null; + for (const other of Array.from(doc.querySelectorAll(`[${HF_AUDIO_CARVE_ATTR}]`))) { + if (other.id === element.id) continue; + try { + const raw = other.getAttribute(HF_AUDIO_CARVE_ATTR); + if (raw && normalizeCarveSettings(JSON.parse(raw)).source === element.id) { + return other.id || "another track"; + } + } catch { + // An unreadable carve on some other element says nothing about this one. + } + } + return null; + })(); + const sourceOptions: AudioTrackOption[] = (() => { const doc = element.element?.ownerDocument; if (!doc) return []; @@ -172,16 +283,14 @@ export function AudioFxGroup({ * on this one. The bands replace any previous carve output but leave * hand-added effects alone, so re-analysing does not discard other work. */ - const analyse = async (): Promise => { - if (!carve?.source) return; + const analyse = async (active: HfCarveSettings | null = carve): Promise => { + if (!active?.source) return; const doc = element.element?.ownerDocument; - const voice = doc?.getElementById(carve.source) as HTMLAudioElement | null; + const voice = doc?.getElementById(active.source) as HTMLAudioElement | null; const src = voice?.getAttribute("src"); if (!src) return; setAnalysing(true); try { - const res = await fetch(new URL(src, doc!.baseURI).href); - const bytes = await res.arrayBuffer(); // Decoded in an OfflineAudioContext, not a live one. Opening a second // output device mid-playback makes the running track glitch while the // hardware is reconfigured; an offline context touches no device. @@ -190,20 +299,116 @@ export function AudioFxGroup({ (window as unknown as { webkitOfflineAudioContext?: typeof OfflineAudioContext }) .webkitOfflineAudioContext; if (!Ctor) return; - const decoder = new Ctor(1, 1, DECODE_SAMPLE_RATE); - const buffer = await decoder.decodeAudioData(bytes); - const bands = analyseCarveBands(buffer.getChannelData(0), buffer.sampleRate, carve); + const decode = async (relative: string): Promise => { + const res = await fetch(new URL(relative, doc!.baseURI).href); + return new Ctor(1, 1, DECODE_SAMPLE_RATE).decodeAudioData(await res.arrayBuffer()); + }; + const buffer = await decode(src); + // Strength is what the author set; these are the numbers it means. + const profile = carveProfile(active.strength); + // The bed as well as the voice, when the carve is asked to match levels: + // "how far over the voice is this bed" cannot be answered by listening to + // one of them. + const bedSrc = profile.duckDb > 0 ? element.element?.getAttribute("src") : null; + const bedBuffer = bedSrc ? await decode(bedSrc).catch(() => null) : null; + const bands = analyseCarveBands(buffer.getChannelData(0), buffer.sampleRate, profile); const carved = carveBandsToChain(bands); + + // The level half of the carve, measured against the voice it has to sit + // under. Times come back relative to the voice clip; the gap between the + // two clips' starts is what aligns them. + const offset = + clipStart(voice?.getAttribute("data-start")) - clipStart(element.dataAttributes?.["start"]); + const duck = bedBuffer + ? analyseCarveDuck( + buffer.getChannelData(0), + bedBuffer.getChannelData(0), + buffer.sampleRate, + profile, + offset, + ) + : []; + // Static carve holds one value, so the level match becomes the duck the + // voice needs while it is actually speaking — the median of it, which + // ignores both the pauses and any single loudest bar. + const speaking = duck.filter((p) => p.v < 0).map((p) => p.v); + const staticDuckDb = speaking.length + ? (speaking.sort((a, b) => a - b)[Math.floor(speaking.length / 2)] ?? 0) + : 0; + // Carve output is tagged so a re-run replaces it instead of stacking. const kept = chain.nodes.filter((n) => !n.fromCarve); + // Ids, minted against the nodes already claiming one, because a dynamic + // carve automates these filters and a lane addresses its node by id. + let claimed: HfAudioFxChain = { version: 1, nodes: kept }; + const mint = (node: HfAudioFxNode): HfAudioFxNode => { + const withId = { ...node, id: mintAudioFxNodeId(claimed), fromCarve: true }; + claimed = { version: 1, nodes: [...claimed.nodes, withId] }; + return withId; + }; + const carvedNodes: HfAudioFxNode[] = carved.nodes.map(mint); + // The gain stage sits after the filters, and only exists when the carve was + // asked to make level room. Dynamic drives it from the envelope; static + // holds the one value above. + const duckNode = + duck.length > 0 + ? mint({ + type: "gain", + enabled: true, + params: { ...defaultAudioFxParams("gain"), gain: active.dynamic ? 0 : staticDuckDb }, + }) + : null; const next = { version: 1, - nodes: [...carved.nodes.map((n) => ({ ...n, fromCarve: true })), ...kept], + nodes: [...carvedNodes, ...(duckNode ? [duckNode] : []), ...kept], }; // Live, like every other chain write: the runtime swaps the graph in // place, so a reload would only interrupt the audio to reach the same // filters. - onSetAttributeQuiet(HF_AUDIO_FX_ATTR, serializeAudioFxChain(next)); + // + // Awaited, because the automation write below is a second read-modify-write + // against the same file — fired together the later one would drop the + // earlier — and because a lane naming a node the chain does not have yet is + // pruned when it is read back. + await onSetAttributeQuiet(HF_AUDIO_FX_ATTR, serializeAudioFxChain(next)); + + /** + * One carve envelope as a lane on this bed's clock. + * + * Everything the analysis returns is timed from the start of the voice, so + * it shifts by the gap between the two clips; and a lane holds its first + * value backwards to the start of its own clip, so a bed that begins before + * the voice needs an explicit "no cut" at zero or it starts out ducked. + */ + const laneFor = (id: string, points: { t: number; v: number }[]): HfAutomationLane[] => { + const shifted = points + .map((p) => ({ t: Number((p.t + offset).toFixed(3)), v: p.v })) + .filter((p) => p.t >= 0); + if ((shifted[0]?.t ?? 0) > 0) shifted.unshift({ t: 0, v: 0 }); + return shifted.length > 1 + ? [{ target: fxAutomationTarget(id, "gain"), points: shifted }] + : []; + }; + + // Dynamic carve: each filter's depth becomes an envelope of the voice's + // level in that band, so pauses leave the bed alone. + const lanes: HfAutomationLane[] = active.dynamic + ? analyseCarveDynamics(buffer.getChannelData(0), buffer.sampleRate, bands).flatMap( + (dyn, i) => { + const id = carvedNodes[i]?.id; + if (!id) return []; + return laneFor(id, dyn.points); + }, + ) + : []; + // The level envelope rides the gain stage, on the same clock as the bands. + if (active.dynamic && duckNode?.id && duck.length > 0) { + lanes.push(...laneFor(duckNode.id, duck)); + } + const carriedOver = withoutCarveLanes(automation, chain); + if (lanes.length > 0 || carriedOver.lanes.length !== automation.lanes.length) { + writeAutomation({ version: 1, lanes: [...carriedOver.lanes, ...lanes] }); + } } catch { // Leave the chain as it was; the button simply re-enables. } finally { @@ -215,6 +420,7 @@ export function AudioFxGroup({ void setCarve(next)} onCarvePreview={(next) => onSetAttributeLive(HF_AUDIO_CARVE_ATTR, JSON.stringify(next))} sourceOptions={sourceOptions} - onAnalyseCarve={() => void analyse()} + carvedAgainstBy={carvedAgainstBy} analysing={analysing} /> ); diff --git a/packages/studio/src/components/editor/propertyPanelFxControls.tsx b/packages/studio/src/components/editor/propertyPanelFxControls.tsx index ffc7ffd88c..67f7e07ee9 100644 --- a/packages/studio/src/components/editor/propertyPanelFxControls.tsx +++ b/packages/studio/src/components/editor/propertyPanelFxControls.tsx @@ -42,6 +42,12 @@ function display(p: HfAudioFxNumberParam, value: number): string { interface ParamRowProps { param: HfAudioFxParam; value: number | string; + /** + * The envelope's value at the playhead, when a lane drives this parameter and + * the playhead is over the clip. Shown in place of `value`, which by then is + * only the seed the lane replaced. + */ + liveValue?: number; /** Fires continuously while dragging — cheap, not persisted. */ onChange(key: string, value: number | string): void; /** Fires once when the gesture ends — this is the write that persists. */ @@ -100,6 +106,7 @@ export function AutomationToggle({ export function FxParamRow({ param, value, + liveValue, onChange, onCommit, disabled, @@ -159,7 +166,11 @@ export function FxParamRow({ ); } - const shown = dragging ? local : value; + // An envelope's value at the playhead outranks the one stored in the chain, + // because it is the one the audio is using — the stored number is only the seed + // the lane replaced. Safe against the pointer: an automated control is locked + // (see `locked` below), so there is no drag for this to fight. + const shown = dragging ? local : (liveValue ?? value); const numeric = typeof shown === "number" ? shown : Number(shown); const current = Number.isFinite(numeric) ? numeric : param.default; @@ -229,6 +240,8 @@ export function FxParamRow({ interface FxParamsProps { def: HfAudioFxDef; params: HfAudioFxParamValues; + /** What automated knobs are worth at the playhead, by parameter key. */ + liveValues?: ReadonlyMap; onChange(params: HfAudioFxParamValues): void; onCommit?(params: HfAudioFxParamValues): void; disabled?: boolean; @@ -243,6 +256,7 @@ interface FxParamsProps { export function FxParams({ def, params, + liveValues, onChange, onCommit, disabled, @@ -270,6 +284,7 @@ export function FxParams({ key={p.key} param={p} value={params[p.key] ?? p.default} + liveValue={liveValues?.get(p.key)} onChange={set} onCommit={commit} disabled={disabled} diff --git a/packages/studio/src/components/editor/propertyPanelFxSection.test.tsx b/packages/studio/src/components/editor/propertyPanelFxSection.test.tsx index bbd20f63ed..3cb99a83c8 100644 --- a/packages/studio/src/components/editor/propertyPanelFxSection.test.tsx +++ b/packages/studio/src/components/editor/propertyPanelFxSection.test.tsx @@ -28,8 +28,6 @@ const chainOf = (...types: string[]): HfAudioFxChain => ({ nodes: types.map((t) => ({ type: t, enabled: true, params: defaultAudioFxParams(t) })), }); -const noop = () => {}; - function mount(overrides: Partial[0]> = {}) { const onChainChange = vi.fn(); const onChainPreview = vi.fn(); @@ -42,7 +40,6 @@ function mount(overrides: Partial[0]> = {}) { carve={overrides.carve ?? null} onCarveChange={overrides.onCarveChange ?? onCarveChange} sourceOptions={overrides.sourceOptions ?? [{ id: "vo", label: "Voiceover" }]} - onAnalyseCarve={overrides.onAnalyseCarve ?? noop} analysing={overrides.analysing} disabled={overrides.disabled} automatedTargets={overrides.automatedTargets} @@ -198,6 +195,123 @@ describe("FxSection chain", () => { }); }); +describe("FxSection carve module", () => { + /** + * A carve is one thing an author turned on, not the six filters it happens to + * compile to. Listed individually they read as hand-built effects: removable + * one at a time, reorderable, each with knobs that the next strength change + * overwrites without warning. + */ + const carved = { + version: 1, + nodes: [ + { type: "peaking", id: "n1", fromCarve: true, params: { frequency: 400, gain: -6, q: 1.4 } }, + { type: "peaking", id: "n2", fromCarve: true, params: { frequency: 1600, gain: -9, q: 1.4 } }, + { type: "gain", id: "n3", fromCarve: true, params: { gain: -6 } }, + { type: "lowpass", id: "n4", params: { frequency: 8000, q: 0.7, poles: "2" } }, + ], + } as unknown as HfAudioFxChain; + + it("shows the carve's effects as one module, alongside hand-built ones", () => { + const { host } = mount({ chain: carved }); + const rows = Array.from(host.querySelectorAll(".hf-fx-node")); + // One row for the carve, one for the low-pass the author added. + expect(rows).toHaveLength(2); + expect(host.querySelector(".hf-fx-carve-module")).not.toBeNull(); + expect(host.querySelector(".hf-fx-carve-module")?.textContent).toContain("Voiceover carve"); + }); + + it("says what the module contains, since its parts are not listed", () => { + const { host } = mount({ chain: carved }); + const text = host.querySelector(".hf-fx-carve-module")?.textContent ?? ""; + expect(text).toMatch(/2 bands/); + expect(text).toMatch(/level/); + }); + + it("removes every carve effect together, never one of them", () => { + const onChainChange = vi.fn(); + const { host } = mount({ chain: carved, onChainChange }); + const removes = Array.from( + host.querySelectorAll(".hf-fx-carve-module .hf-fx-remove"), + ); + expect(removes).toHaveLength(1); + act(() => removes[0]!.click()); + const nodes = onChainChange.mock.calls[0]![0].nodes as { id: string }[]; + expect(nodes.map((n) => n.id)).toEqual(["n4"]); + }); + + it("bypasses the whole module at once", () => { + const onChainChange = vi.fn(); + const { host } = mount({ chain: carved, onChainChange }); + const bypass = host.querySelector(".hf-fx-carve-module .hf-fx-bypass")!; + act(() => bypass.click()); + const nodes = onChainChange.mock.calls[0]![0].nodes as { id: string; enabled?: boolean }[]; + expect(nodes.filter((n) => n.id !== "n4").every((n) => n.enabled === false)).toBe(true); + // The author's own effect is not touched. + expect(nodes.find((n) => n.id === "n4")?.enabled).not.toBe(false); + }); + + it("lists what each effect inside it is set to", () => { + // Grouped is not hidden: the carve compiles to real filters and an author has + // to be able to see where they landed. What they cannot do is edit them by + // hand — strength owns those numbers — so the settings read out rather than + // offering controls that the next adjustment would overwrite. + const { host } = mount({ chain: carved }); + const module = host.querySelector(".hf-fx-carve-module")!; + act(() => module.querySelector(".hf-fx-node-name")!.click()); + const members = Array.from(module.querySelectorAll(".hf-fx-carve-member")); + expect(members).toHaveLength(3); + // Named by what tells them apart, the way the timeline lanes name them. + expect(members.map((m) => m.querySelector(".hf-fx-carve-member-name")?.textContent)).toEqual([ + "Peaking EQ 400 Hz", + "Peaking EQ 1.6 kHz", + "Gain", + ]); + // And every one of its settings is visible. + const first = members[0]!.textContent ?? ""; + expect(first).toContain("Gain"); + expect(first).toMatch(/-6(\.0)? dB/); + expect(first).toMatch(/1\.4/); // Q + }); + + it("reads its settings out rather than offering controls", () => { + const { host } = mount({ chain: carved }); + const module = host.querySelector(".hf-fx-carve-module")!; + act(() => module.querySelector(".hf-fx-node-name")!.click()); + expect(module.querySelectorAll("input")).toHaveLength(0); + }); + + it("says which of them the timeline is driving", () => { + // A carve in dynamic mode automates every one of these, and that is where the + // values come from — so the module has to point at the lane rather than look + // like a static setting. + const { host } = mount({ + chain: carved, + automatedTargets: new Set(["fx.n1.gain", "fx.n3.gain"]), + }); + const module = host.querySelector(".hf-fx-carve-module")!; + act(() => module.querySelector(".hf-fx-node-name")!.click()); + const automated = Array.from(module.querySelectorAll("[data-automated]")); + expect(automated).toHaveLength(2); + }); + + it("keeps the summary readable while collapsed", () => { + const { host } = mount({ chain: carved }); + const module = host.querySelector(".hf-fx-carve-module")!; + expect(module.querySelectorAll(".hf-fx-carve-member")).toHaveLength(0); + expect(module.textContent).toContain("2 bands + level"); + }); + + it("offers no per-effect controls inside the module", () => { + // Reordering or editing one band is meaningless: the next strength change + // rewrites every one of them. + const { host } = mount({ chain: carved }); + const module = host.querySelector(".hf-fx-carve-module")!; + expect(module.querySelectorAll(".hf-fx-move")).toHaveLength(0); + expect(module.querySelectorAll("input[type=range]")).toHaveLength(0); + }); +}); + describe("FxSection carve", () => { it("is off by default and is not an entry in the chain", () => { const { host } = mount(); @@ -230,16 +344,17 @@ describe("FxSection carve", () => { expect(options).toContain("Narration"); }); - it("will not analyse until a source is chosen", () => { - const { host } = mount({ carve: { ...DEFAULT_CARVE, source: "" } }); - expect(host.querySelector(".hf-fx-analyse")!.disabled).toBe(true); + it("offers no analyse button — picking a voice is the whole gesture", () => { + // A carve with a source and no filters is a setting nobody applied; the + // button was a second step for something the panel already knew to do. + const { host } = mount({ carve: { ...DEFAULT_CARVE, source: "vo" } }); + expect(host.querySelector(".hf-fx-analyse")).toBeNull(); + expect(host.textContent).not.toMatch(/Analyse/i); }); - it("analyses once a source is chosen", () => { - const onAnalyseCarve = vi.fn(); - const { host } = mount({ carve: { ...DEFAULT_CARVE, source: "vo" }, onAnalyseCarve }); - click(host.querySelector(".hf-fx-analyse")); - expect(onAnalyseCarve).toHaveBeenCalledTimes(1); + it("says when it is working, since there is no button to grey out", () => { + const { host } = mount({ carve: { ...DEFAULT_CARVE, source: "vo" }, analysing: true }); + expect(host.querySelector(".hf-fx-carve-working")?.textContent).toMatch(/Analysing/i); }); it("disables everything when the panel is read-only", () => { diff --git a/packages/studio/src/components/editor/propertyPanelFxSection.tsx b/packages/studio/src/components/editor/propertyPanelFxSection.tsx index 56b7b09132..3a99c77862 100644 --- a/packages/studio/src/components/editor/propertyPanelFxSection.tsx +++ b/packages/studio/src/components/editor/propertyPanelFxSection.tsx @@ -18,11 +18,15 @@ import { type HfAudioFxDef, type HfAudioFxGroup, type HfAudioFxNode, + type HfAudioFxParam, type HfAudioFxParamValues, } from "@hyperframes/core/audio-fx"; import { DEFAULT_CARVE, type HfCarveSettings } from "@hyperframes/core/audio-carve"; import { fxAutomationTarget } from "@hyperframes/core/audio-automation"; import { FxParams, FxParamRow } from "./propertyPanelFxControls.js"; +// Shared with the timeline's lane labels: a band is named by its frequency in +// both places, and two formatters would drift. +import { formatHz } from "../../player/components/automationLaneData"; const GROUP_ORDER: HfAudioFxGroup[] = ["filter", "dynamics", "nonlinear", "time"]; const GROUP_LABEL: Record = { @@ -41,6 +45,7 @@ interface FxNodeRowProps { node: HfAudioFxNode; index: number; automatedTargets?: ReadonlySet; + liveAutomationValues?: ReadonlyMap; onAutomateParam?(nodeId: string, paramKey: string): void; onRemoveParamAutomation?(nodeId: string, paramKey: string): void; open: boolean; @@ -148,6 +153,205 @@ function FxNodeHeader({ ); } +/** + * The carve's own effects, as one module. + * + * A carve is one thing the author switched on; the peaking filters and the level + * stage are how it is built. Listed individually they read as hand-built effects + * — removable one at a time, reorderable, each with knobs the next strength + * change silently overwrites. So the rack shows the unit, says what is inside it, + * and offers the two actions that mean anything for a whole module: bypass it, + * or remove it. + */ +/** What one effect inside the module is called: its own name, plus the band. */ +function carveMemberName(node: HfAudioFxNode): string { + const def = getAudioFxDef(node.type); + const freq = node.params?.["frequency"]; + const label = def?.label ?? node.type; + return typeof freq === "number" ? `${label} ${formatHz(freq)}` : label; +} + +/** A parameter's value as the rack shows it: rounded to the step, with its unit. */ +function formatParamValue(param: HfAudioFxParam, raw: number | string | undefined): string { + if (param.kind !== "number" || typeof raw !== "number") return String(raw ?? ""); + const places = param.step >= 1 ? 0 : param.step >= 0.1 ? 1 : 2; + return `${Number(raw.toFixed(places))}${param.unit ? ` ${param.unit}` : ""}`; +} + +/** + * Width to reserve for a parameter's value, in characters. + * + * Derived from what the parameter CAN read rather than what it currently reads, so + * the column never moves: an automated value updates 30 times a second, and + * `-1 dB` is two characters narrower than `-3.2 dB`, which was enough to shunt + * everything after it sideways on every frame. `ch` is exact here because the + * readouts are monospace and already `tabular-nums`. + */ +function paramValueWidthCh(param: HfAudioFxParam): number { + if (param.kind === "enum") { + return Math.max(1, ...param.options.map((option) => option.value.length)); + } + const places = param.step >= 1 ? 0 : param.step >= 0.1 ? 1 : 2; + const digits = Math.max( + String(Math.floor(Math.abs(param.min))).length, + String(Math.floor(Math.abs(param.max))).length, + ); + const sign = param.min < 0 ? 1 : 0; + const decimals = places > 0 ? places + 1 : 0; + const unit = param.unit ? param.unit.length + 1 : 0; + return sign + digits + decimals + unit; +} + +/** One member of the module: what it is, and what every knob is set to. */ +function FxCarveMember({ + node, + automatedTargets, + liveAutomationValues, +}: { + node: HfAudioFxNode; + automatedTargets?: ReadonlySet; + liveAutomationValues?: ReadonlyMap; +}) { + const def = getAudioFxDef(node.type); + if (!def) return null; + const params = node.params ?? defaultAudioFxParams(node.type); + return ( +
+ + {carveMemberName(node)} + +
+ {def.params.map((param) => { + const target = node.id ? fxAutomationTarget(node.id, param.key) : null; + const automated = Boolean(target && automatedTargets?.has(target)); + // The envelope's value at the playhead when there is one, which is what + // the audio is using; the stored number is only the seed behind it. + const live = target ? liveAutomationValues?.get(target) : undefined; + const driven = automated && live !== undefined; + const value = formatParamValue(param, driven ? live : params[param.key]); + return ( + + {param.label} + + {value} + + {/* The lane is where an automated value comes from, and where it is + edited — saying so is the difference between a stale readout and + a pointer to the thing that owns it. */} + {automated ? A : null} + + ); + })} +
+
+ ); +} + +/** + * The carve's own effects, as one module. + * + * A carve is one thing the author switched on; the peaking filters and the level + * stage are how it is built. Listed individually in the rack they read as + * hand-built effects — removable one at a time, reorderable, each with knobs the + * next strength change silently overwrites. So the rack shows the unit, and the + * unit owns the actions that mean anything for a whole module: bypass, remove. + * + * Grouped is not hidden. Opening it lists every effect inside with all of its + * settings, because an author has to be able to see where the analysis landed — + * as readouts rather than controls, since strength is what sets them and a knob + * here would be overwritten by the next adjustment. A value the timeline drives + * says so, and points at the lane that owns it. + */ +function FxCarveModule({ + nodes, + automatedTargets, + liveAutomationValues, + open, + disabled, + onToggleOpen, + onToggleBypass, + onRemove, +}: { + nodes: HfAudioFxNode[]; + automatedTargets?: ReadonlySet; + liveAutomationValues?: ReadonlyMap; + open: boolean; + disabled?: boolean; + onToggleOpen(): void; + onToggleBypass(): void; + onRemove(): void; +}) { + const bands = nodes.filter((n) => n.type === "peaking").length; + const hasLevel = nodes.some((n) => n.type === "gain"); + const bypassed = nodes.every((n) => n.enabled === false); + const summary = [`${bands} band${bands === 1 ? "" : "s"}`, ...(hasLevel ? ["level"] : [])].join( + " + ", + ); + return ( +
+
+ + + {summary} + + + +
+ {open ? ( + // Divided rows rather than boxes: these are parts of one module, and a + // border around each would read as the separate effects this replaced. +
+ {nodes.map((node, i) => ( + + ))} +
+ ) : null} +
+ ); +} + /** * Which of an effect's knobs already have a lane. * @@ -174,6 +378,7 @@ function FxNodeParams({ index, disabled, automatedTargets, + liveAutomationValues, onUpdate, onPreview, onAutomateParam, @@ -184,16 +389,29 @@ function FxNodeParams({ index: number; disabled: boolean; automatedTargets?: ReadonlySet; + liveAutomationValues?: ReadonlyMap; onUpdate(index: number, patch: Partial): void; onPreview(index: number, params: HfAudioFxParamValues): void; onAutomateParam?(nodeId: string, paramKey: string): void; onRemoveParamAutomation?(nodeId: string, paramKey: string): void; }) { const nodeId = node.id; + // Lanes address a node by id; the controls know their own parameter keys. This + // is the one place that translation belongs. + const liveValues = ((): Map | undefined => { + if (!nodeId || !liveAutomationValues?.size) return undefined; + const byKey = new Map(); + for (const param of def.params) { + const live = liveAutomationValues.get(fxAutomationTarget(nodeId, param.key)); + if (live !== undefined) byKey.set(param.key, live); + } + return byKey; + })(); return ( onPreview(index, params)} onCommit={(params: HfAudioFxParamValues) => onUpdate(index, { params })} @@ -213,6 +431,7 @@ function FxNodeRow({ node, index, automatedTargets, + liveAutomationValues, onAutomateParam, onRemoveParamAutomation, open, @@ -251,6 +470,7 @@ function FxNodeRow({ index={index} disabled={Boolean(disabled) || bypassed} automatedTargets={automatedTargets} + liveAutomationValues={liveAutomationValues} onUpdate={onUpdate} onPreview={onPreview} onAutomateParam={onAutomateParam} @@ -265,6 +485,15 @@ export interface FxSectionProps { chain: HfAudioFxChain; /** Targets this track already automates, as `fx..` strings. */ automatedTargets?: ReadonlySet; + /** + * What each automated target is worth at the playhead, by the same key. + * + * An automated parameter's stored number is only the seed the lane replaced, so + * a rack that shows it stands still while the carve is audibly working. Absent, + * or missing a key, means there is no playhead over this clip and the stored + * value is the honest one. + */ + liveAutomationValues?: ReadonlyMap; /** Add a lane for one effect parameter, seeded at its current value. */ onAutomateParam?(nodeId: string, paramKey: string): void; /** Delete one effect parameter's lane. */ @@ -281,10 +510,13 @@ export interface FxSectionProps { /** Continuous updates while a carve slider is dragged. Without this every * pointermove patched the source file and resynced the selection. */ onCarvePreview?(carve: HfCarveSettings): void; + /** + * Set when another track's carve listens to this one, naming it. The carve block + * is then not offered here at all: this track is the voice, not the bed. + */ + carvedAgainstBy?: string | null; /** Other audio elements that could act as the carve source. */ sourceOptions: AudioTrackOption[]; - /** Re-run analysis against the current source audio. */ - onAnalyseCarve?(): void; analysing?: boolean; disabled?: boolean; } @@ -292,16 +524,17 @@ export interface FxSectionProps { export function FxSection({ chain, automatedTargets, + liveAutomationValues, onAutomateParam, onRemoveParamAutomation, onRemoveNodeAutomation, onChainChange, onChainPreview, carve, + carvedAgainstBy, onCarveChange, onCarvePreview, sourceOptions, - onAnalyseCarve, analysing, disabled, }: FxSectionProps) { @@ -310,7 +543,10 @@ export function FxSection({ const previewCarve = onCarvePreview ?? onCarveChange; // Nothing to carve against means nothing to show — see the block below. - const showCarve = sourceOptions.length > 0 || carve !== null; + // Not offered on the voice another track is already carving against — that + // track is the far end of someone else's relationship, and a carve of its own + // could only name a source it must not. + const showCarve = !carvedAgainstBy && (sourceOptions.length > 0 || carve !== null); const [adding, setAdding] = useState(false); const [openNode, setOpenNode] = useState(0); @@ -368,6 +604,24 @@ export function FxSection({ [chain.nodes, mutate, onRemoveNodeAutomation], ); + const [carveOpen, setCarveOpen] = useState(false); + const carveNodes = useMemo(() => chain.nodes.filter((n) => n.fromCarve), [chain.nodes]); + + /** Remove the carve's effects together, with the envelopes they carried. */ + const removeCarve = useCallback(() => { + for (const node of carveNodes) { + if (node.id) onRemoveNodeAutomation?.(node.id); + } + mutate(chain.nodes.filter((n) => !n.fromCarve)); + setOpenNode(null); + }, [carveNodes, chain.nodes, mutate, onRemoveNodeAutomation]); + + /** Bypass or enable every carve effect at once — the module is the unit. */ + const toggleCarveBypass = useCallback(() => { + const bypassed = carveNodes.every((n) => n.enabled === false); + mutate(chain.nodes.map((n) => (n.fromCarve ? { ...n, enabled: bypassed } : n))); + }, [carveNodes, chain.nodes, mutate]); + const moveNode = useCallback( (index: number, delta: number) => { const target = index + delta; @@ -389,24 +643,46 @@ export function FxSection({ No effects on this track.

) : ( - chain.nodes.map((node, i) => ( - setOpenNode(openNode === i ? null : i)} - onUpdate={updateNode} - onMove={moveNode} - onRemove={removeNode} - onPreview={previewNode} - /> - )) + chain.nodes.map((node, i) => { + if (node.fromCarve) { + // The module stands in for the whole run of carve nodes, drawn once + // at the first of them. + const first = chain.nodes.findIndex((n) => n.fromCarve); + if (i !== first) return null; + return ( + setCarveOpen((was) => !was)} + onToggleBypass={toggleCarveBypass} + onRemove={removeCarve} + /> + ); + } + return ( + setOpenNode(openNode === i ? null : i)} + onUpdate={updateNode} + onMove={moveNode} + onRemove={removeNode} + onPreview={previewNode} + /> + ); + }) )} @@ -484,63 +760,54 @@ export function FxSection({ ))} + {/* One knob for the whole effect. Depth, band count, width, the + intelligibility weighting and both level-match numbers move + together anyway — a gentle carve is shallow in few bands with + little ducking, a hard one is deeper in more with more — so the + panel sets the strength and `carveProfile` derives the six + numbers the analysis works in. */} previewCarve({ ...carve, maxCutDb: Number(v) })} - onCommit={(_k, v) => onCarveChange({ ...carve, maxCutDb: Number(v) })} - /> - previewCarve({ ...carve, bands: Number(v) })} - onCommit={(_k, v) => onCarveChange({ ...carve, bands: Number(v) })} - /> - previewCarve({ ...carve, intelligibilityBias: Number(v) })} - onCommit={(_k, v) => onCarveChange({ ...carve, intelligibilityBias: Number(v) })} + onChange={(_k, v) => previewCarve({ ...carve, strength: Number(v) })} + onCommit={(_k, v) => onCarveChange({ ...carve, strength: Number(v) })} /> - + {/* A static carve holds its cuts for the whole clip, pauses + included. Dynamic hands every value to an envelope of the voice's + own level, so the bed is only worked on while there is something + to make room for. Written as ordinary automation, which is why the + lanes show up in the timeline and can be edited afterwards. */} + + {analysing ? ( +

+ Analysing… +

+ ) : null} ) : null} diff --git a/packages/studio/src/hooks/useAutomationSelectionKeyboard.test.tsx b/packages/studio/src/hooks/useAutomationSelectionKeyboard.test.tsx index f8ab1a3d76..ac7f48d3c8 100644 --- a/packages/studio/src/hooks/useAutomationSelectionKeyboard.test.tsx +++ b/packages/studio/src/hooks/useAutomationSelectionKeyboard.test.tsx @@ -16,6 +16,16 @@ import type { } from "../player/components/useAutomationLanes"; import type { TimelineElement } from "../player/store/timelineElement"; +/** + * A selection box spanning the lane's whole value axis. + * + * What almost every test here is about is the time span — which breakpoints a + * Delete or a copy covers. The box's value bounds have their own tests; giving + * these an unbounded axis keeps them testing the one thing they name. + */ +function wholeAxis(sel: T): T & { v0: number; v1: number } { + return { ...sel, v0: Number.NEGATIVE_INFINITY, v1: Number.POSITIVE_INFINITY }; +} /** Minimal valid fixture — TimelineElement only requires these five fields. */ const bgmElement: TimelineElement = { id: "bgm", @@ -106,22 +116,88 @@ describe("useAutomationSelectionKeyboard", () => { return { onCommit }; }; - it("Delete empties the selected range and pins anchors", () => { + it("Delete removes every breakpoint the selection covers", () => { + // Deleted, not emptied. Pinning anchors at the selection's edges keeps the + // envelope either side from moving, which is right for a shape insert or a + // paste — but answering "delete these points" with two NEW points at the edges + // reads as the delete not having worked. usePlayerStore.setState({ elements: [bgmElement], selectedElementId: "bgm" }); usePlayerStore .getState() - .setAutomationSelection({ elementKey: "bgm", target: "volume", t0: 1, t1: 3 }); + .setAutomationSelection(wholeAxis({ elementKey: "bgm", target: "volume", t0: 1, t1: 3 })); const { onCommit } = setup({}); key("Delete"); const written = onCommit.mock.calls.at(-1)?.[0]; const points = written?.lanes?.[0]?.points ?? []; - expect(points.map((p: { t: number }) => p.t)).toEqual([0, 1, 3, 4]); + // The fixture lane is 0, 2, 4: only t=2 was inside. + expect(points.map((p: { t: number }) => p.t)).toEqual([0, 4]); + }); + + it("Delete leaves a point the box's value bounds exclude", () => { + // The box spans the whole clip but only its top, so Delete takes the one + // breakpoint up there and nothing else. A time range could not express this. + usePlayerStore.setState({ elements: [bgmElement], selectedElementId: "bgm" }); + usePlayerStore.getState().setAutomationSelection({ + elementKey: "bgm", + target: "volume", + t0: 0, + t1: 4, + v0: 0.9, + v1: 1, + }); + const { onCommit } = setup({}); + key("Delete"); + const written = onCommit.mock.calls.at(-1)?.[0]; + const points = written?.lanes?.[0]?.points ?? []; + // Fixture is (0, v=1), (2, v=0.5), (4, v=0): only the first was in the box. + expect(points.map((p: { t: number }) => p.t)).toEqual([2, 4]); + }); + + it("Delete takes points sitting exactly on the selection's edges", () => { + // Endpoint-inclusive, matching the copy path: a point the selection was dragged + // over is inside it, edge or not. Every range operation leaves a breakpoint + // exactly on an edge, so excluding them would leave those behind every time. + usePlayerStore.setState({ elements: [bgmElement], selectedElementId: "bgm" }); + usePlayerStore + .getState() + .setAutomationSelection(wholeAxis({ elementKey: "bgm", target: "volume", t0: 2, t1: 4 })); + const { onCommit } = setup({}); + key("Delete"); + const points = onCommit.mock.calls.at(-1)?.[0]?.lanes?.[0]?.points ?? []; + expect(points.map((p: { t: number }) => p.t)).toEqual([0]); + }); + + it("Delete over a stretch with no breakpoints writes nothing at all", () => { + // A no-op rather than a write: emptying a span that had nothing in it used to + // push an undo entry that changed nothing but the anchors it invented. + usePlayerStore.setState({ elements: [bgmElement], selectedElementId: "bgm" }); + usePlayerStore + .getState() + .setAutomationSelection(wholeAxis({ elementKey: "bgm", target: "volume", t0: 2.5, t1: 3.5 })); + const { onCommit } = setup({}); + const e = new KeyboardEvent("keydown", { key: "Delete", bubbles: true, cancelable: true }); + act(() => void document.dispatchEvent(e)); + expect(onCommit).not.toHaveBeenCalled(); + expect(e.defaultPrevented).toBe(false); + }); + + it("Delete clears the lane when the selection covers all of it", () => { + usePlayerStore.setState({ elements: [bgmElement], selectedElementId: "bgm" }); + usePlayerStore + .getState() + .setAutomationSelection(wholeAxis({ elementKey: "bgm", target: "volume", t0: 0, t1: 6 })); + const { onCommit } = setup({}); + key("Delete"); + const written = onCommit.mock.calls.at(-1)?.[0]; + // withLane drops a lane with no points left, so the attribute goes empty and + // the clip is back to its plain data-volume. + expect(written?.lanes ?? []).toEqual([]); }); it("Escape clears the selection", () => { usePlayerStore .getState() - .setAutomationSelection({ elementKey: "bgm", target: "volume", t0: 1, t1: 3 }); + .setAutomationSelection(wholeAxis({ elementKey: "bgm", target: "volume", t0: 1, t1: 3 })); setup({}); key("Escape"); expect(usePlayerStore.getState().automationSelection).toBeNull(); @@ -130,7 +206,7 @@ describe("useAutomationSelectionKeyboard", () => { it("is inert while a text input has focus", () => { usePlayerStore .getState() - .setAutomationSelection({ elementKey: "bgm", target: "volume", t0: 1, t1: 3 }); + .setAutomationSelection(wholeAxis({ elementKey: "bgm", target: "volume", t0: 1, t1: 3 })); const { onCommit } = setup({}); const input = document.createElement("input"); document.body.append(input); @@ -145,7 +221,7 @@ describe("useAutomationSelectionKeyboard", () => { usePlayerStore.setState({ elements: [bgmElement], selectedElementId: "bgm" }); usePlayerStore .getState() - .setAutomationSelection({ elementKey: "bgm", target: "volume", t0: 2, t1: 4 }); + .setAutomationSelection(wholeAxis({ elementKey: "bgm", target: "volume", t0: 2, t1: 4 })); setup({}); combo("c"); const entry = readClipboard(null); @@ -165,7 +241,7 @@ describe("useAutomationSelectionKeyboard", () => { }); usePlayerStore .getState() - .setAutomationSelection({ elementKey: "bgm", target: "volume", t0: 2, t1: 4 }); + .setAutomationSelection(wholeAxis({ elementKey: "bgm", target: "volume", t0: 2, t1: 4 })); const { onCommit } = setup({}); combo("c"); expect(readClipboard(null)?.span).toBe(2); @@ -183,6 +259,10 @@ describe("useAutomationSelectionKeyboard", () => { target: "volume", t0: 5, t1: 7, + // Full height: everything the paste landed is selected, so Delete straight + // after undoes it in one press. + v0: 0, + v1: 1, }); }); @@ -196,7 +276,7 @@ describe("useAutomationSelectionKeyboard", () => { }); usePlayerStore .getState() - .setAutomationSelection({ elementKey: "bgm", target: "volume", t0: 2, t1: 4 }); + .setAutomationSelection(wholeAxis({ elementKey: "bgm", target: "volume", t0: 2, t1: 4 })); const { onCommit } = setup({}); combo("c"); @@ -218,6 +298,10 @@ describe("useAutomationSelectionKeyboard", () => { target: "volume", t0: 4, t1: 6, + // Full height: everything the paste landed is selected, so Delete straight + // after undoes it in one press. + v0: 0, + v1: 1, }); }); @@ -230,7 +314,7 @@ describe("useAutomationSelectionKeyboard", () => { usePlayerStore.setState({ elements: [bgmElement], selectedElementId: "bgm" }); usePlayerStore .getState() - .setAutomationSelection({ elementKey: "bgm", target: "volume", t0: 2, t1: 4 }); + .setAutomationSelection(wholeAxis({ elementKey: "bgm", target: "volume", t0: 2, t1: 4 })); const { onCommit } = setup({}); combo("c"); expect(readClipboard(null)?.span).toBe(2); @@ -238,7 +322,7 @@ describe("useAutomationSelectionKeyboard", () => { // A 0.1s-wide selection right near the clip's 6s end. usePlayerStore .getState() - .setAutomationSelection({ elementKey: "bgm", target: "volume", t0: 5.5, t1: 5.6 }); + .setAutomationSelection(wholeAxis({ elementKey: "bgm", target: "volume", t0: 5.5, t1: 5.6 })); combo("v"); const written = onCommit.mock.calls.at(-1)?.[0]; const times = (written?.lanes?.[0]?.points ?? []).map((p: { t: number }) => p.t); @@ -252,6 +336,10 @@ describe("useAutomationSelectionKeyboard", () => { target: "volume", t0: 4, t1: 6, + // Full height: everything the paste landed is selected, so Delete straight + // after undoes it in one press. + v0: 0, + v1: 1, }); }); @@ -264,7 +352,7 @@ describe("useAutomationSelectionKeyboard", () => { usePlayerStore.setState({ elements: [bgmElement], selectedElementId: "bgm" }); usePlayerStore .getState() - .setAutomationSelection({ elementKey: "bgm", target: "volume", t0: 2, t1: 4 }); + .setAutomationSelection(wholeAxis({ elementKey: "bgm", target: "volume", t0: 2, t1: 4 })); const { onCommit } = setup({ commitTargetKey: "some-other-clip" }); const e = combo("v"); expect(e.defaultPrevented).toBe(false); @@ -282,7 +370,7 @@ describe("useAutomationSelectionKeyboard", () => { }); usePlayerStore .getState() - .setAutomationSelection({ elementKey: "bgm", target: "volume", t0: 2, t1: 4 }); + .setAutomationSelection(wholeAxis({ elementKey: "bgm", target: "volume", t0: 2, t1: 4 })); const { onCommit } = setup({}); combo("c"); usePlayerStore.getState().clearAutomationSelection(); @@ -302,7 +390,7 @@ describe("useAutomationSelectionKeyboard", () => { usePlayerStore.setState({ elements: [bgmElement], selectedElementId: "bgm" }); usePlayerStore .getState() - .setAutomationSelection({ elementKey: "bgm", target: "volume", t0: 1, t1: 2 }); + .setAutomationSelection(wholeAxis({ elementKey: "bgm", target: "volume", t0: 1, t1: 2 })); setup({ automation: { version: 1, lanes: [{ target: "volume", points: [] }] } }); const e = combo("c"); @@ -315,7 +403,7 @@ describe("useAutomationSelectionKeyboard", () => { usePlayerStore.setState({ elements: [bgmElement], selectedElementId: "bgm" }); usePlayerStore .getState() - .setAutomationSelection({ elementKey: "bgm", target: "volume", t0: 2, t1: 4 }); + .setAutomationSelection(wholeAxis({ elementKey: "bgm", target: "volume", t0: 2, t1: 4 })); const { onCommit } = setup({}); combo("C"); expect(readClipboard(null)?.span).toBe(2); diff --git a/packages/studio/src/hooks/useAutomationSelectionKeyboard.ts b/packages/studio/src/hooks/useAutomationSelectionKeyboard.ts index 354fd7342b..5e73ab52f8 100644 --- a/packages/studio/src/hooks/useAutomationSelectionKeyboard.ts +++ b/packages/studio/src/hooks/useAutomationSelectionKeyboard.ts @@ -1,8 +1,8 @@ /** * Keyboard surface for the active automation selection: Escape clears, - * Delete/Backspace empties the range (anchors pinned, envelope outside - * untouched), Cmd/Ctrl+C copies it, Cmd/Ctrl+V pastes at the selection's - * start (or the playhead) onto the selected clip's lane. Sibling of + * Delete/Backspace deletes every breakpoint inside the selection box, Cmd/Ctrl+C + * copies its span, Cmd/Ctrl+V pastes at the selection's start (or the playhead) onto the + * selected clip's lane. Sibling of * useKeyframeKeyboard and copies its contract: capture phase so playback * shortcuts cannot swallow keys we act on, inert while any text input has * focus, and a key is only consumed when it does something. @@ -16,7 +16,7 @@ import { useEffect } from "react"; import { usePlayerStore, type TimelineElement } from "../player/store/playerStore"; import { laneFor, withLane } from "../player/components/automationLaneGeometry"; -import { replaceRange } from "../player/components/automationLaneSelection"; +import { pointInSelection, replaceRange } from "../player/components/automationLaneSelection"; import { copyRange, isLastPasteSpan, @@ -87,11 +87,22 @@ function resolveSelectionContext( } /** - * The write that empties the active selection, or null when there is nothing - * to do: the clip is gone, its lane is read-only, the target no longer - * resolves to a range, or the lane already has no points in it. Split out of - * the keydown handler so each stays under the complexity a single branch of - * keyboard dispatch should carry. + * The write that deletes the breakpoints inside the active selection, or null when + * there is nothing to do: the clip is gone, its lane is read-only, the target no + * longer resolves to a range, or the selection covers no breakpoints. + * + * Deletes them outright rather than emptying the span behind anchor points. Anchors + * are what `replaceRange` exists for, and they are right for a shape insert or a + * paste — the envelope either side of the edit must not move. But Delete over a + * selection is the author saying "these points, gone", and answering that with two + * NEW points at the selection's edges reads as the delete not having worked. The + * envelope between the surviving neighbours re-interpolates, which is what deleting + * a breakpoint means everywhere else in the lane (right-clicking one does exactly + * this). + * + * Both axes of the selection box, edges included: what Delete removes is exactly + * what the lane drew a ring around. A point at the right time but outside the box's + * value bounds stays — which is the whole reason the box has them. */ function resolveDeleteWrite( state: PlayerState, @@ -99,14 +110,12 @@ function resolveDeleteWrite( sel: AutomationSelection, ): { onCommit(next: HfAutomation): void; next: HfAutomation } | null { const ctx = resolveSelectionContext(state, lanes, sel); - if (!ctx || ctx.lane.points.length === 0) return null; - const points = replaceRange({ - lane: ctx.lane, - range: ctx.range, - t0: sel.t0, - t1: sel.t1, - inner: [], - }); + if (!ctx) return null; + const points = ctx.lane.points.filter((p) => !pointInSelection(p, sel)); + // Nothing inside is nothing to do — and it must stay a no-op rather than + // writing, or Delete over a smooth stretch would push an undo entry that + // changed nothing. + if (points.length === ctx.lane.points.length) return null; return { onCommit: ctx.binding.onCommit, next: withLane(ctx.binding.automation, { target: sel.target, points }), @@ -247,7 +256,16 @@ function handlePaste( // Select the pasted span — the only feedback that it landed — and mark it, so // an immediate second Cmd+V recognises this selection as the paste's own and // chains right after it instead of overwriting it. - const mark = { elementKey: paste.elementKey, target: paste.target, t0: atT, t1 }; + // Full-height box over the pasted span: everything that landed is selected, so + // Delete straight after a paste undoes it in one press. + const mark = { + elementKey: paste.elementKey, + target: paste.target, + t0: atT, + t1, + v0: paste.range.min, + v1: paste.range.max, + }; state.setAutomationSelection(mark); markLastPaste(mark); return true; diff --git a/packages/studio/src/hooks/useDomEditAttributeCommits.ts b/packages/studio/src/hooks/useDomEditAttributeCommits.ts index d4e76c2cb8..7d72bd9620 100644 --- a/packages/studio/src/hooks/useDomEditAttributeCommits.ts +++ b/packages/studio/src/hooks/useDomEditAttributeCommits.ts @@ -8,6 +8,7 @@ import { import type { PersistDomEditOperations } from "./domEditCommitTypes"; import { reportDomEditPersistFailure } from "./domEditPersistFailure"; import { bumpDomEditCommitMapVersion, runDomEditCommit } from "./domEditCommitRunner"; +import { syncStoredAutomationFromPreview } from "../player/lib/automationStoreSync"; // ── Types ── @@ -26,6 +27,26 @@ interface DataAttributeCommitOptions { skipRefresh: boolean; refreshAfter?: boolean; onSettled?: (ok: boolean) => void; + /** + * Undo grouping for a gesture that spans several commits. + * + * Without it the key is derived from the prefix, attribute and element, and the + * window is history's own 300ms — so a drag's moves and the release that ends it + * landed in different entries, and a drag slower than the window split further. + * A caller that knows a gesture is in progress passes one key for all of it. + */ + coalesce?: { key: string; ms: number }; + /** + * Apply to the preview and stop there — no file write, no history entry. + * + * What a gesture wants from every pointermove: the preview document and the + * audio graph following the pointer, with the file written once on release. + * Persisting each move put a fragment of one drag in the undo stack, and since + * those writes race, a follow-up's "before" was often not the previous entry's + * "after" — history refuses to coalesce across that gap, so undo took back a + * few milliseconds of the gesture and looked like it had done nothing. + */ + previewOnly?: boolean; } function resolveFullAttrName(attr: string, prefixData: boolean | undefined): string { @@ -108,7 +129,9 @@ export function useDomEditAttributeCommits({ if (!domEditSelection) return; const iframe = previewIframeRef.current; const fullAttr = resolveFullAttrName(attr, true); - const commitKey = `${options.coalescePrefix}:${attr}:${getDomEditTargetKey(domEditSelection)}`; + const commitKey = + options.coalesce?.key ?? + `${options.coalescePrefix}:${attr}:${getDomEditTargetKey(domEditSelection)}`; const isLatestCommit = bumpDomEditCommitMapVersion( domAttributeCommitVersionRef.current, commitKey, @@ -134,12 +157,15 @@ export function useDomEditAttributeCommits({ const nextValue = value === null || value === "" ? null : value; setOrRemovePreviewAttribute(editedElement, fullAttr, nextValue); }, - persist: () => - persistDomEditOperations(domEditSelection, [op], { - label: options.label, - coalesceKey: commitKey, - skipRefresh: options.skipRefresh, - }), + persist: options.previewOnly + ? async () => {} + : () => + persistDomEditOperations(domEditSelection, [op], { + label: options.label, + coalesceKey: commitKey, + ...(options.coalesce ? { coalesceMs: options.coalesce.ms } : {}), + skipRefresh: options.skipRefresh, + }), shouldRevert: () => isLatestCommit(), revert: () => { if (!editedElement) return; @@ -147,7 +173,19 @@ export function useDomEditAttributeCommits({ }, onError: (error) => reportDomEditPersistFailure(domEditSelection, [op], error, showToast), shouldResync: () => isLatestCommit() && !!options.refreshAfter, - resync: () => refreshDomEditSelectionFromPreview(domEditSelection), + resync: () => { + refreshDomEditSelectionFromPreview(domEditSelection); + // The player store keeps its own copy of each element's attributes, and + // that copy is what the timeline's automation lanes draw from. Nothing + // else refreshes it: a commit patches the preview document and the file, + // and resyncs the dom-edit SELECTION for the panel. So every writer that + // did not also update the store by hand — the FX panel's automate and + // un-automate buttons, the keyboard Delete, a paste — changed the file and + // the audio while the lane went on drawing what it had, until a reload. + // One sink here rather than a sync in each writer, because three of them + // shipped without one. + syncStoredAutomationFromPreview(previewIframeRef.current?.contentDocument ?? null); + }, onSettled: options.onSettled, }); }, @@ -273,12 +311,19 @@ export function useDomEditAttributeCommits({ ); const handleDomAttributeLiveCommit = useCallback( - async (attr: string, value: string | null, onSettled?: (ok: boolean) => void) => { + async ( + attr: string, + value: string | null, + onSettled?: (ok: boolean) => void, + live?: { coalesce?: { key: string; ms: number }; previewOnly?: boolean }, + ) => { await commitDataAttribute(attr, value, { label: `Edit ${attr.replace(/^(data-)?/, "").replace(/-/g, " ")}`, coalescePrefix: "attr-live", skipRefresh: true, onSettled, + ...(live?.coalesce ? { coalesce: live.coalesce } : {}), + ...(live?.previewOnly ? { previewOnly: true } : {}), }); }, [commitDataAttribute], @@ -294,12 +339,13 @@ export function useDomEditAttributeCommits({ * edit computes from a pre-edit value and appears to do nothing. */ const handleDomAttributeQuietCommit = useCallback( - async (attr: string, value: string | null) => { + async (attr: string, value: string | null, coalesce?: { key: string; ms: number }) => { await commitDataAttribute(attr, value, { label: `Edit ${attr.replace(/^(data-)?/, "").replace(/-/g, " ")}`, coalescePrefix: "attr-quiet", skipRefresh: true, refreshAfter: true, + ...(coalesce ? { coalesce } : {}), }); }, [commitDataAttribute], @@ -348,7 +394,19 @@ export function useDomEditAttributeCommits({ }, onError: (error) => reportDomEditPersistFailure(domEditSelection, [op], error, showToast), shouldResync: () => isLatestCommit(), - resync: () => refreshDomEditSelectionFromPreview(domEditSelection), + resync: () => { + refreshDomEditSelectionFromPreview(domEditSelection); + // The player store keeps its own copy of each element's attributes, and + // that copy is what the timeline's automation lanes draw from. Nothing + // else refreshes it: a commit patches the preview document and the file, + // and resyncs the dom-edit SELECTION for the panel. So every writer that + // did not also update the store by hand — the FX panel's automate and + // un-automate buttons, the keyboard Delete, a paste — changed the file and + // the audio while the lane went on drawing what it had, until a reload. + // One sink here rather than a sync in each writer, because three of them + // shipped without one. + syncStoredAutomationFromPreview(previewIframeRef.current?.contentDocument ?? null); + }, }); }, [ diff --git a/packages/studio/src/hooks/useDomEditCommits.test.tsx b/packages/studio/src/hooks/useDomEditCommits.test.tsx index 1f0af1c06e..7a3c8f2c36 100644 --- a/packages/studio/src/hooks/useDomEditCommits.test.tsx +++ b/packages/studio/src/hooks/useDomEditCommits.test.tsx @@ -1256,6 +1256,44 @@ describe("useDomEditCommits attribute persist handling", () => { } }); + it("applies a preview-only write without persisting it", async () => { + // What a drag needs from every pointermove: the preview and the audio graph + // follow, the file does not. Persisting each move filled the undo stack with + // fragments of one gesture — and because those writes race, a follow-up's + // "before" often was not the previous entry's "after", so history refused to + // coalesce them and undo took back a few milliseconds of the drag. + const fetchSpy = stubPatchFetch({ ok: true, changed: true, matched: true }); + const { iframe, element } = createPreviewElement(); + const rendered = renderDomEditCommits(createSelection(element), iframe); + + try { + await act(async () => { + await rendered.hook.handleDomAttributeLiveCommit("volume", "0.7", undefined, { + previewOnly: true, + }); + }); + expect(element.getAttribute("data-volume")).toBe("0.7"); + expect(fetchSpy).not.toHaveBeenCalled(); + } finally { + rendered.cleanup(); + } + }); + + it("still persists a live write that does not ask to be preview-only", async () => { + const fetchSpy = stubPatchFetch({ ok: true, changed: true, matched: true }); + const { iframe, element } = createPreviewElement(); + const rendered = renderDomEditCommits(createSelection(element), iframe); + + try { + await act(async () => { + await rendered.hook.handleDomAttributeLiveCommit("volume", "0.7"); + }); + expect(fetchSpy).toHaveBeenCalled(); + } finally { + rendered.cleanup(); + } + }); + it("keeps a data-attribute commit on success", async () => { stubPatchFetch({ ok: true, diff --git a/packages/studio/src/hooks/useLivePlayheadTime.ts b/packages/studio/src/hooks/useLivePlayheadTime.ts new file mode 100644 index 0000000000..72c0d2972b --- /dev/null +++ b/packages/studio/src/hooks/useLivePlayheadTime.ts @@ -0,0 +1,49 @@ +/** + * The playhead in composition seconds, live while the transport runs. + * + * The RAF loop deliberately does not push every frame through the store — it + * notifies `liveTime` instead, so the playhead can move without re-rendering the + * app. A panel that wants to follow it therefore has to subscribe itself, and + * throttle: 30 Hz reads as continuous and costs an order of magnitude less than a + * render per frame. + * + * Paused, the store is the truth — a seek or a scrub lands there — so this returns + * that instead, which is what lets a readout follow the playhead while it is being + * dragged as well as while it is playing. + */ +import { useEffect, useRef, useState } from "react"; +import { liveTime, usePlayerStore } from "../player"; + +/** Long enough to be much cheaper than a frame, short enough to read as motion. */ +const THROTTLE_MS = 33; + +export function useLivePlayheadTime(): number { + const storeTime = usePlayerStore((s) => s.currentTime); + const isPlaying = usePlayerStore((s) => s.isPlaying); + const liveRef = useRef(storeTime); + const [, forceRender] = useState(0); + + // Paused, the ref tracks the store so the first frame of playback is never a + // stale value from the last time the transport ran. + if (!isPlaying) liveRef.current = storeTime; + + useEffect(() => { + if (!isPlaying) return; + let timerId: ReturnType | 0 = 0; + const unsubscribe = liveTime.subscribe((t) => { + liveRef.current = t; + if (!timerId) { + timerId = setTimeout(() => { + timerId = 0; + forceRender((v) => v + 1); + }, THROTTLE_MS); + } + }); + return () => { + unsubscribe(); + if (timerId) clearTimeout(timerId); + }; + }, [isPlaying]); + + return isPlaying ? liveRef.current : storeTime; +} diff --git a/packages/studio/src/hooks/usePreviewPersistence.ts b/packages/studio/src/hooks/usePreviewPersistence.ts index 5530122713..04cc2df09b 100644 --- a/packages/studio/src/hooks/usePreviewPersistence.ts +++ b/packages/studio/src/hooks/usePreviewPersistence.ts @@ -12,6 +12,7 @@ import { flushStudioPendingEdits } from "../utils/studioPendingEdits"; import { trackStudioEvent } from "../utils/studioTelemetry"; import { applyUndoRestoreToPreview, type UndoRestoreFile } from "../utils/gsapUndoRestore"; import { usePlayerStore } from "../player"; +import { syncStoredAutomationFromPreview } from "../player/lib/automationStoreSync"; /** The restore payload the undo/redo preview-sync consumes (from the history store). */ interface HistoryPreviewRestore { @@ -220,7 +221,13 @@ export function usePreviewPersistence({ player.setElements([]); player.setSelectedElementId(null); player.setTimelineReady(false); + return; } + // A soft restore patched the reverted attributes onto the live preview, but the + // player store keeps its own copy and that copy is what the automation lanes + // draw — so without this an undone envelope edit stayed invisible until a + // reload. The full path above clears the store and waits for discovery instead. + syncStoredAutomationFromPreview(previewIframeRef.current?.contentDocument ?? null); }, [previewIframeRef, activeCompPathRef, reloadPreview], ); From f423ea293fcb4893cfd3b9b02bded3bbd32abc97 Mon Sep 17 00:00:00 2001 From: Vance Ingalls Date: Fri, 7 Aug 2026 13:26:25 -0700 Subject: [PATCH 10/25] docs(skills): add /hyperframes-audio, with a headless carve MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mixing was the one audio surface no skill covered: the effect chain, automation lanes and the voiceover carve existed with no guidance, so an agent had the attributes and none of the judgement for using them. The skill teaches the carve as what it is — a relationship between two tracks, wired like a sidechain, with the settings on the bed naming the voice — and routes the effect families by the problem each solves rather than listing parameters. It ships scripts/carve.mjs because a skill teaching a feature agents cannot operate is not much of a skill: the analysis needs decoded PCM, and the only other way in is clicking a Studio panel. Same core functions and same decode rate, so headless output and panel output are the same three attributes. It finds the voice and the bed itself — names first, then by ear, measuring how much of each track is quiet, since a voice stops between phrases and a bed does not — and refuses rather than guessing when two tracks are too close to call. References carry the full registry with which parameters can actually be automated (the four worklet effects expose none, so a lane on one is silently inert) and the exact JSON of the three attributes. Counts, catalogues and the core-skill manifest updated in lockstep; the pinning test in skillsManifest is what caught the two surfaces the maintenance checklist in CLAUDE.md does not name. Co-Authored-By: Claude Opus 5 (1M context) --- .claude-plugin/marketplace.json | 1 + CLAUDE.md | 9 +- README.md | 7 +- packages/cli/src/templates/_shared/AGENTS.md | 2 +- packages/cli/src/templates/_shared/CLAUDE.md | 2 +- packages/cli/src/utils/skillsManifest.ts | 1 + skills/hyperframes-audio/SKILL.md | 272 ++++++++++++ .../references/attributes.md | 88 ++++ .../references/fx-registry.md | 84 ++++ skills/hyperframes-audio/scripts/carve.mjs | 418 ++++++++++++++++++ skills/hyperframes/SKILL.md | 1 + 11 files changed, 876 insertions(+), 9 deletions(-) create mode 100644 skills/hyperframes-audio/SKILL.md create mode 100644 skills/hyperframes-audio/references/attributes.md create mode 100644 skills/hyperframes-audio/references/fx-registry.md create mode 100644 skills/hyperframes-audio/scripts/carve.mjs diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 936b1c0945..c472c1282f 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -15,6 +15,7 @@ "skills": [ "./skills/hyperframes", "./skills/hyperframes-animation", + "./skills/hyperframes-audio", "./skills/hyperframes-cli", "./skills/hyperframes-core", "./skills/hyperframes-creative", diff --git a/CLAUDE.md b/CLAUDE.md index 638926508f..31edd2a183 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,12 +4,12 @@ Open-source video rendering framework: write HTML, render video. ## Skills -This repo ships 19 AI agent skills via [vercel-labs/skills](https://github.com/vercel-labs/skills). Install them before writing compositions — they encode framework-specific patterns that generic docs don't cover. **Default to the core set**: the `/hyperframes` router installs each creation workflow on demand; install all 19 only when the user explicitly asks for the full set. +This repo ships 20 AI agent skills via [vercel-labs/skills](https://github.com/vercel-labs/skills). Install them before writing compositions — they encode framework-specific patterns that generic docs don't cover. **Default to the core set**: the `/hyperframes` router installs each creation workflow on demand; install all 20 only when the user explicitly asks for the full set. ```bash npx hyperframes skills update # default: installs/refreshes the core set — workflows install on demand -npx skills add heygen-com/hyperframes --full-depth # interactive picker (terminal only — non-interactive without --skill installs all 19) -npx skills add heygen-com/hyperframes --all --full-depth # all 19 at once — only on explicit request +npx skills add heygen-com/hyperframes --full-depth # interactive picker (terminal only — non-interactive without --skill installs all 20) +npx skills add heygen-com/hyperframes --all --full-depth # all 20 at once — only on explicit request npx skills add heygen-com/hyperframes --skill --full-depth # just one (bare name, no leading slash) ``` @@ -39,6 +39,7 @@ Atomic capabilities the creation workflows compose against — pull one when you - `/hyperframes-keyframes` — seek-safe keyframe authoring across runtimes: GSAP timelines, CSS keyframes, Anime.js, WAAPI, FLIP, paths, masks, SVG morph/draw, text trails, 3D depth; plus `hyperframes keyframes` diagnostics for surfacing and verifying rendered motion. - `/hyperframes-creative` — non-animation creative direction: `frame.md` / `design.md` handling, palettes, typography, narration, beat planning, audio-reactive visuals, composition patterns. - `/media-use` — the media OS: resolve any media need (BGM, SFX, image, icon, logo, voice, color grade, LUT) into a frozen local file or paste-ready block + ledger record; generate via TTS / music / image models when the catalog misses; transcribe, caption, remove backgrounds, and reuse assets across projects. One shared `scripts/audio.mjs` engine + manifest tracking; keeps search noise on disk. +- `/hyperframes-audio` — mix the audio already placed in a composition: voiceover carve (dip a music bed only in the bands the voice occupies, static or dynamic, level match included), the effect chain (EQ, compressor, limiter, gate, saturation, delay, reverb, chorus, phaser, bitcrush), and automation envelopes on volume or any effect parameter. Sourcing the audio is `/media-use`; this is what happens to it afterwards. - `/hyperframes-cli` — CLI dev loop: `init`, `add`, `lint`, `check`, `snapshot`, `preview`, `render`, `publish`, `doctor`, `lambda` (AWS Lambda cloud rendering). - `/hyperframes-registry` — install and wire registry blocks and components into compositions via `hyperframes add`. Covers authoring a new block or component to contribute upstream. - `/figma` — import Figma assets, tokens, components, and storyboard sections → reconstructed motion (frames read as states, not slides) (REST/CLI) plus Motion animations (MCP) and shaders (MCP source / native export) into a composition. @@ -51,7 +52,7 @@ When adding a new skill, or substantially renaming / repurposing an existing one 2. The scaffolded project template `packages/cli/src/templates/_shared/CLAUDE.md` + `AGENTS.md` — written into every `hyperframes init` project, so a stale entry there ships to users. The two template files must stay byte-identical. 3. If the skill changes the routing surface for "make a video" requests, also update the routing table + intent layer in `skills/hyperframes/SKILL.md` AND that workflow's own route file, `skills/hyperframes/references/routes/.md`. One file carries both halves: the input/output/trigger contract the router reads before the workflow is installed, and its interview entry (must-haves, conditionals, deferred asks, run-shape). The older `references/workflow-catalog.md` and `references/route-briefs.md` are now "moved" stubs pointing at `routes/` — don't edit them. 4. Mirror the Router / Creation workflows / Domain skills grouping across all surfaces so a skill always lives in the same column. -5. Skill count appears in the README and CLAUDE.md intro lines ("19 AI agent skills…") — update on add/remove. The `docs/guides/skills.mdx` page and the CLI templates deliberately omit a count to avoid drift; keep them count-free. +5. Skill count appears in the README and CLAUDE.md intro lines ("20 AI agent skills…") — update on add/remove. The `docs/guides/skills.mdx` page and the CLI templates deliberately omit a count to avoid drift; keep them count-free. The skill's own `SKILL.md` frontmatter `description:` is the source of truth for the one-line "use when" blurb; copy from there into the catalog rather than paraphrasing. diff --git a/README.md b/README.md index 2a34005a40..df182d6b42 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ Install the HyperFrames skills, then describe the video you want: npx skills add heygen-com/hyperframes --full-depth ``` -> The picker opens with nothing pre-selected — the **Core Skills** group is all you need: the `/hyperframes` router installs each creation workflow on demand. Agents and non-interactive runs should use `npx hyperframes skills update` instead — it installs exactly the core set, whereas a non-interactive `skills add` without `--skill` installs all 19. +> The picker opens with nothing pre-selected — the **Core Skills** group is all you need: the `/hyperframes` router installs each creation workflow on demand. Agents and non-interactive runs should use `npx hyperframes skills update` instead — it installs exactly the core set, whereas a non-interactive `skills add` without `--skill` installs all 20. > > `--full-depth` does a full clone of the repo's current `main`. Without it, `skills add` fetches the skills.sh registry blob, which lags `main` by hours — you'd get an older copy of a skill. (`hyperframes skills update` already installs full-depth.) @@ -53,9 +53,9 @@ The skills teach agents the HyperFrames production loop: plan the video, write v ## Skills -HyperFrames ships 19 skills agents load on demand. Read `/hyperframes` first — it's the router and capability map; it picks a workflow for any "make me a…" request — video, deck, or composition port — and points to the domain skills below. +HyperFrames ships 20 skills agents load on demand. Read `/hyperframes` first — it's the router and capability map; it picks a workflow for any "make me a…" request — video, deck, or composition port — and points to the domain skills below. -Default to the **core set** — the router installs each creation workflow on demand. `npx hyperframes skills update` installs exactly that from anywhere; the interactive picker (`npx skills add heygen-com/hyperframes --full-depth`) lists it as the "Core Skills" group, nothing pre-selected. The picker is interactive-only — a non-interactive or agent run without `--skill` installs all 19. Use `npx skills add heygen-com/hyperframes --all --full-depth` to install all 19 deliberately (skips the picker), or `npx skills add heygen-com/hyperframes --skill --full-depth` for just one (bare name, no leading `/`). Keep `--full-depth` — it installs the current `main`; without it `skills add` fetches the skills.sh blob, which lags by hours. +Default to the **core set** — the router installs each creation workflow on demand. `npx hyperframes skills update` installs exactly that from anywhere; the interactive picker (`npx skills add heygen-com/hyperframes --full-depth`) lists it as the "Core Skills" group, nothing pre-selected. The picker is interactive-only — a non-interactive or agent run without `--skill` installs all 20. Use `npx skills add heygen-com/hyperframes --all --full-depth` to install all 20 deliberately (skips the picker), or `npx skills add heygen-com/hyperframes --skill --full-depth` for just one (bare name, no leading `/`). Keep `--full-depth` — it installs the current `main`; without it `skills add` fetches the skills.sh blob, which lags by hours. Installs stay lean after that: `npx hyperframes init` keeps the **core set** fresh (the router, the `hyperframes-*` domain skills, and `media-use` — plus whatever is already installed; `/figma` stays on demand) and never expands a partial install; the creation workflows install **on demand** — the router runs `npx hyperframes skills update ` before entering one. Nothing re-pulls the full set behind your back. @@ -102,6 +102,7 @@ Atomic capabilities the creation workflows compose against — pull one when you | `/hyperframes-creative` | Non-animation creative direction — `frame.md` / `design.md`, palettes, typography, narration, beat planning, audio-reactive visuals, composition patterns. | | `/media-use` | The media OS — resolve any media need (BGM, SFX, image, icon, logo, voice, color grade, LUT) into a frozen local file or paste-ready block + ledger record, generate via TTS/music/image models when the catalog misses, transcribe, caption, remove backgrounds, and reuse assets across projects. One shared audio engine + manifest tracking. | | `/hyperframes-cli` | CLI dev loop — `init`, `lint`, `check`, `snapshot`, `preview`, `render`, `publish`, `doctor`, plus HeyGen-hosted cloud rendering (`cloud render`) and AWS Lambda rendering (`lambda deploy / render / progress`). | +| `/hyperframes-audio` | Mix the audio already placed in a composition — voiceover carve (dip a music bed only in the bands the voice occupies, static or dynamic, level match included), the effect chain (EQ, compressor, limiter, gate, saturation, delay, reverb, chorus, phaser, bitcrush), and automation envelopes on volume or any effect parameter. Sourcing the audio is `/media-use`. | | `/hyperframes-registry` | Install and wire registry blocks and components into compositions via `hyperframes add`. Authoring a new block or component to contribute upstream. | | `/figma` | Import Figma assets, tokens, components, and storyboard sections → reconstructed motion (frames read as states, not slides) (REST/CLI) plus Motion animations (MCP) and shaders (MCP source / native export) into a composition. | diff --git a/packages/cli/src/templates/_shared/AGENTS.md b/packages/cli/src/templates/_shared/AGENTS.md index 5c8126aeb2..1762ea7a66 100644 --- a/packages/cli/src/templates/_shared/AGENTS.md +++ b/packages/cli/src/templates/_shared/AGENTS.md @@ -18,7 +18,7 @@ **Porting an existing composition?** `/remotion-to-hyperframes` translates a Remotion (React) composition into HyperFrames HTML — a source migration, separate from the creation workflows above. -The domain skills (`/hyperframes-core`, `/hyperframes-animation`, `/hyperframes-keyframes`, `/hyperframes-creative`, `/hyperframes-cli`, `/media-use`, `/hyperframes-registry`, `/figma`) and the full capability map live inside `/hyperframes` — it is the single source of truth for which skill handles which intent. +The domain skills (`/hyperframes-core`, `/hyperframes-animation`, `/hyperframes-keyframes`, `/hyperframes-creative`, `/hyperframes-cli`, `/media-use`, `/hyperframes-audio`, `/hyperframes-registry`, `/figma`) and the full capability map live inside `/hyperframes` — it is the single source of truth for which skill handles which intent. **Changing how real footage or images look or reveal?** Load `/media-use` and read its `references/media-treatments.md` before editing, even when the request only says dark, flat, boring, retro, private, or “make the reveal cooler.” It governs how footage is treated, never whether media may be used. Use canonical media treatments and seek-safe motion; do not improvise equivalent CSS/SVG filters or overlays. diff --git a/packages/cli/src/templates/_shared/CLAUDE.md b/packages/cli/src/templates/_shared/CLAUDE.md index 5c8126aeb2..1762ea7a66 100644 --- a/packages/cli/src/templates/_shared/CLAUDE.md +++ b/packages/cli/src/templates/_shared/CLAUDE.md @@ -18,7 +18,7 @@ **Porting an existing composition?** `/remotion-to-hyperframes` translates a Remotion (React) composition into HyperFrames HTML — a source migration, separate from the creation workflows above. -The domain skills (`/hyperframes-core`, `/hyperframes-animation`, `/hyperframes-keyframes`, `/hyperframes-creative`, `/hyperframes-cli`, `/media-use`, `/hyperframes-registry`, `/figma`) and the full capability map live inside `/hyperframes` — it is the single source of truth for which skill handles which intent. +The domain skills (`/hyperframes-core`, `/hyperframes-animation`, `/hyperframes-keyframes`, `/hyperframes-creative`, `/hyperframes-cli`, `/media-use`, `/hyperframes-audio`, `/hyperframes-registry`, `/figma`) and the full capability map live inside `/hyperframes` — it is the single source of truth for which skill handles which intent. **Changing how real footage or images look or reveal?** Load `/media-use` and read its `references/media-treatments.md` before editing, even when the request only says dark, flat, boring, retro, private, or “make the reveal cooler.” It governs how footage is treated, never whether media may be used. Use canonical media treatments and seek-safe motion; do not improvise equivalent CSS/SVG filters or overlays. diff --git a/packages/cli/src/utils/skillsManifest.ts b/packages/cli/src/utils/skillsManifest.ts index 5145cc0348..ad613270f4 100644 --- a/packages/cli/src/utils/skillsManifest.ts +++ b/packages/cli/src/utils/skillsManifest.ts @@ -150,6 +150,7 @@ export function isCoreSkill(name: string): boolean { export const FALLBACK_CORE_SKILLS: readonly string[] = [ "hyperframes", "hyperframes-animation", + "hyperframes-audio", "hyperframes-cli", "hyperframes-core", "hyperframes-creative", diff --git a/skills/hyperframes-audio/SKILL.md b/skills/hyperframes-audio/SKILL.md new file mode 100644 index 0000000000..9bae288c13 --- /dev/null +++ b/skills/hyperframes-audio/SKILL.md @@ -0,0 +1,272 @@ +--- +name: hyperframes-audio +description: > + Use when audio already placed in a HyperFrames composition needs to be mixed: + a music bed that fights a voiceover (voiceover carve), effects on a track + (EQ, compressor, limiter, gate, saturation, delay, reverb, chorus, phaser, + bitcrush), or automation envelopes drawn on a track's volume or any effect + parameter. + Don't use for sourcing or generating audio — finding BGM, SFX, or making a + voiceover is `/media-use`. Don't use for clip timing or track layout, which is + `/hyperframes-core`. +--- + +# HyperFrames Audio + +A mix is a set of relationships, not a stack of processors. Two tracks that each +sound right alone can be unlistenable together, and the fix is almost never "turn +one down" — it is finding what they are fighting over and giving it to whichever +one needs it. Every tool here exists to express one of those relationships. + +Effects live on the element as `data-fx-chain`, and preview and render run the +same Web Audio graph — the studio in a live context, the engine in an offline one +inside the browser it already drives. There is one implementation of each effect, +so what you hear while scrubbing is what gets written. You never tune twice. + +Three attributes carry everything, all on the audio/video element itself: + +| Attribute | Holds | +| ----------------- | --------------------------------------------------------- | +| `data-fx-chain` | the effects, in signal order | +| `data-automation` | envelopes on this track's volume or its effect parameters | +| `data-fx-carve` | the carve's own settings, so it can be re-derived | + +Exact JSON for each, and the rules a lane must satisfy: `references/attributes.md`. +Every effect with its parameters, ranges and units: `references/fx-registry.md`. + +## How it fits together + +Two authoring surfaces write those attributes; two runtimes read them through the +same builders. That shared middle is why preview predicts the render. + +```mermaid +flowchart TB + voice["voice track
media file"] + bed["music bed
media file"] + + subgraph AUTHOR["Authoring — the only things that write attributes"] + panel["Studio
Voiceover carve control"] + script["scripts/carve.mjs
detects the pair, dynamic by default"] + analysis["core/audioCarve.ts
carveProfile · analyseCarveBands
analyseCarveDuck · analyseCarveDynamics"] + panel --> analysis + script --> analysis + end + + voice --> analysis + bed --> analysis + + subgraph ATTRS["Written onto the bed element"] + carveAttr["data-fx-carve
source · strength · dynamic"] + chainAttr["data-fx-chain
peaking xN + gain, tagged fromCarve"] + autoAttr["data-automation
a lane per carved parameter"] + end + + analysis --> carveAttr + analysis --> chainAttr + analysis --> autoAttr + + subgraph SHARED["One implementation, read by both"] + build["audioFxGraph.ts · buildFxChain"] + sched["audioFxAutomation.ts · scheduleChainAutomation"] + end + + chainAttr --> build + autoAttr --> sched + + build --> preview["Preview
live AudioContext
attachElementFxChain"] + sched --> preview + build --> render["Render
OfflineAudioContext in the headless browser
applyAudioFxChain"] + sched --> render + + preview --> heard["what you hear while scrubbing"] + render --> wav["processed WAV
+ chainTailSeconds so the mix lets the tail through"] + wav --> mix["engine · audioMixer
volume lane baked into the PCM here, not in the graph"] + mix --> out["the rendered mix"] + + edit["editing the attribute mid-playback"] -.->|MutationObserver| preview +``` + +The carve's own settings are never read at playback — the chain and lanes it +produced are what play. `data-fx-carve` exists so strength can be changed on an +existing carve instead of guessed back out of the filters. + +Inside a carved bed the signal runs through the dips first, then the level match, +then anything you built yourself — which is why a limiter you add still acts as +the last ceiling: + +```mermaid +flowchart LR + src["decoded bed"] --> p1["peaking
400 Hz"] + p1 --> p2["peaking
1 kHz"] + p2 --> p3["peaking
1.6 kHz"] + p3 --> g["gain
level match"] + g --> hand["your own effects
e.g. limiter"] + hand --> dest["track gain, then out"] + + l1["lane fx.n1.gain"] -.->|"envelope of the voice's
level in that band"| p1 + l4["lane fx.n4.gain"] -.->|"how far the bed
ducks overall"| g +``` + +A static carve is the same graph with fixed values and no lanes at all. + +## Reach for a family by the problem, not the name + +**Filters** (`highpass`, `lowpass`, `peaking`, `lowshelf`, `highshelf`) decide +which frequencies a track is allowed to occupy. This is the first tool for two +sources colliding, because collisions happen in bands: a bed and a voice both +want 1–3 kHz, and taking that from the bed costs the bed far less than turning +the whole thing down costs the mix. A high-pass on a voice is the standard fix +for rumble; a low-pass darkens or muffles deliberately. + +**Dynamics** (`gain`, `compressor`, `limiter`, `gate`) decide how a track's level +behaves over time. Compression narrows the distance between loud and quiet so the +quiet parts can come up. A limiter is a ceiling — it does not shape anything, it +guarantees nothing gets past. A gate removes what is below a threshold, which is +how you silence room tone between phrases. `gain` is a plain level stage, and it +is what an automation lane rides when a track has to move out of the way. + +**Nonlinear** (`saturate`, `bitcrush`) changes the waveform's shape, which adds +harmonics that were not there. Reach for it when a track needs character or +grit rather than correction — and remember it is generative: it makes a thin +source denser, not cleaner. + +**Time** (`delay`, `reverb`, `chorus`, `phaser`) puts a track in a space or gives +it width. These are the ones that most easily wreck a mix, because a tail or a +detuned copy occupies the same room a voice needs. Use them on the thing that +should sit *behind* something else, and keep the wet amount lower than sounds +right in isolation. + +The chain is serial: each effect processes what the one before it produced. So +corrective filtering goes early, character in the middle, and a limiter last +where it can actually act as a ceiling. + +## Voiceover carve + +**The problem it solves.** A music bed under a voice makes the voice hard to +follow. The reflex is to duck the whole bed, which works and costs the bed all of +its presence — the music goes limp for the entire voiceover. But the voice does +not need the whole spectrum. It needs the few bands it actually occupies. Carve +takes only those, and the bed keeps its low end and its top, so it is still music +while the voice is still intelligible. + +**It is a relationship, not an effect.** The settings live on the *bed* — the +track that gets processed — and they name the voice to listen to, exactly as a +sidechain compressor does: you select the track that gets quieter and pick what +makes it quieter. **Never put a carve on the voice track.** A voice carved +against itself is a bug, not a subtle mix choice. + +**One knob.** `strength` is 0..1 and derives everything: how deep to cut, how +many bands, how wide, how far to favour intelligibility over raw voice energy, +how far the level may drop, how far under the voice to aim. Those six move +together in any real mix — a gentle carve is a shallow cut in few bands with +little ducking, a hard one is deeper in more bands with more — so they are one +relationship written once, in `carveProfile`. Default is `0.25` — a 6 dB dip in +three bands with 6 dB of level room, audible without sounding like a hole. At +`0.5` the dip reaches 10 dB, which is where a carve starts being heard as an +effect rather than as room for the voice; above that is deliberate territory for +a loud bed under a quiet voice. `0` is spectral only — one band, no level match +at all. + +**Carve by default.** A bed playing under narration wants a carve; it is not a +polish step to get to if there is time. Place both tracks, run the command below, +listen. Skip it only when there is no narration for the music to sit under — a +music video, a title card, a montage cut to the track. + +**Static or dynamic — dynamic unless you know otherwise.** A static carve holds +its cuts for the whole clip, including every pause, so the bed is thinned where +there is nothing to make room for. Dynamic turns every value into an envelope of +the voice's own level: silence leaves the bed alone, a loud passage pushes the +carve to full depth. That is what almost every voiceover wants, so it is the +default. Reach for `--static` only for wall-to-wall narration with no real gaps, +where an envelope is hundreds of breakpoints describing a constant. + +**Level matching is part of it.** Spectral carving cannot fix a bed that is +simply louder than the voice. So the carve also measures how far over the voice +the bed sits and writes a `gain` stage: held at one value for a static carve, +driven by an envelope for a dynamic one. That envelope releases slowly on +purpose — music that snaps back to full the instant a word ends sounds like a +machine doing it. + +**Running it.** In Studio: pick the voice in the bed's Voiceover carve control; +turning it on adds the modules and strength adjusts what is there. Headless — +which is the path when you are authoring a composition rather than editing one: + +```bash +node /scripts/carve.mjs --comp index.html +``` + +That is the whole command. It finds the voice and the bed itself, carves +dynamically at the default strength, and prints what it decided: + +``` +bed music-bed (name looks like music) +voice narration (only track left) +carve strength 0.25 dynamic +bands 400Hz -6dB q1.4, 1000Hz -3dB q1.4, 1600Hz -3.17dB q1.4 +level 216-point envelope, floor -6 dB +``` + +Name the pair with `--bed` / `--voice` when the composition has several plausible +tracks, `--strength` to push it, `--static` to hold one depth, `--dry-run` to see +that report and write nothing. + +**How it picks the pair.** Names first, because that is what you already told it +and the answer is explainable: a track whose id or filename looks like music +(`music`, `bgm`, `bed`, `score`…) is the bed, one that looks like a voice +(`voice`, `vo`, `narration`, `speech`…) is the voice, and SFX-shaped names are not +candidates for either. If one role is filled and a single track is left, that +track takes the other role. Only when names decide nothing does it listen: it +measures how much of each track is quiet, and the one that stops between phrases +is the voice. **When two tracks are too close to call it refuses and asks you to +name them** rather than carving the wrong one — a bed carved against a bed is +silent and confusing, and typing two ids is cheap. + +Same analysis functions as the panel, so the result is identical. Needs `ffmpeg` +on PATH and `@hyperframes/core` installed in the project (`npm i -D +@hyperframes/core`) — the CLI inlines core rather than shipping it, so it cannot +be borrowed from there. + +**What it writes** is an ordinary chain of peaking filters plus a gain stage, +tagged `fromCarve`. That tagging is the whole trick: a re-run replaces the +previous carve and leaves every effect you built by hand — and every lane you +drew by hand — exactly where it was. So re-carving at a new strength is safe and +repeatable, and `data-fx-carve` exists so the settings can be read back rather +than guessed from the filters. + +## Automation + +A lane is a set of breakpoints on one parameter: `{t, v}` in clip-local seconds +and the parameter's own units. Targets are `volume` for the track's level, or +`fx..` for an effect's knob. + +**Only some parameters can be automated, and a lane on the others is silently +inert.** A knob is automatable when a Web Audio `AudioParam` backs it. The four +worklet-based effects — `compressor`, `limiter`, `gate`, `bitcrush` — expose +none at all, so no lane on any of their parameters will ever move: to make a +compressor's behaviour change over time, automate a `gain` stage before it +instead. `references/fx-registry.md` marks every parameter. + +## Verify + +Almost no static gate covers the mix. The linter reads `data-automation` for +exactly one conflict — `audio_volume_double_automation`, a volume lane on a track +that also has a GSAP tween on `volume`, where the lane wins and the tween is +ignored — and nothing validates the chain or the effect lanes at all. What +enforces those is the render: a chain it cannot parse fails the whole mix rather +than quietly writing the dry signal, because a mix that sounds plausible and is +wrong is worse than a refusal. Preview is the opposite by design: an unreadable +chain plays dry so the composition stays workable. + +A lane pointing at a node the chain does not have is pruned on read, not an +error — so a typo'd `nodeId` costs you the envelope silently. Read the ids back +out of the chain rather than assuming what was minted. + +Effects with a tail (`reverb`, `delay`) make the rendered track **longer** than +its source, and the mix is told how much by the chain. So a bed with reverb no +longer ends exactly at its `data-duration`; that is expected, not a bug. + +Beyond that, a mix is verified by rendering and listening. For a carve: the voice +should be legible without the bed sounding hollowed, and with `dynamic` the bed +should come back up between phrases rather than staying flat. If the bed sounds +notched rather than simply quieter under the voice, the strength is too high — +that is the one failure mode with an obvious sound. diff --git a/skills/hyperframes-audio/references/attributes.md b/skills/hyperframes-audio/references/attributes.md new file mode 100644 index 0000000000..cb44a1bfd3 --- /dev/null +++ b/skills/hyperframes-audio/references/attributes.md @@ -0,0 +1,88 @@ +# The three audio attributes + +All three go on the `