From a39810c6c1f5cc81605bbd3f79cbf184b46480cf Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Mon, 12 Oct 2026 01:07:29 +0200 Subject: [PATCH] feat(ai-edition): let the agent set a zoom's 3D camera addZoom, addZooms and setZoom take rotationPreset (left, right, orbit, or none for a flat screen), the editor's 3D camera control. It is stored as the zoom's rotationPreset, which getCurrentDocument already reported but no tool could set. Served to the in-app agent and to every MCP client alike. --- electron/ai-edition/agent-tools.test.ts | 46 +++++++++++++++++++++ electron/ai-edition/agent-tools.ts | 49 ++++++++++++++++++----- electron/ai-edition/deep-agent/service.ts | 8 +++- 3 files changed, 90 insertions(+), 13 deletions(-) diff --git a/electron/ai-edition/agent-tools.test.ts b/electron/ai-edition/agent-tools.test.ts index c900ee24be..712fb03979 100644 --- a/electron/ai-edition/agent-tools.test.ts +++ b/electron/ai-edition/agent-tools.test.ts @@ -648,6 +648,52 @@ describe("executeAgentTool", () => { ); }); + it("sets a zoom's 3D camera, reports it, and takes it back off with none", () => { + const added = executeAgentTool( + fixtureDocument(), + "addZoom", + JSON.stringify({ startSec: 2, endSec: 4, rotationPreset: "orbit" }), + ); + expect(added.ok).toBe(true); + const withZoom = added.document as AxcutDocument; + expect(withZoom.zoomRanges[0]).toMatchObject({ rotationPreset: "orbit" }); + expect(JSON.parse(added.resultJson)).toMatchObject({ rotationPreset: "orbit" }); + expect(() => documentSchema.parse(withZoom)).not.toThrow(); + + const id = withZoom.zoomRanges[0].id; + const turned = executeAgentTool( + withZoom, + "setZoom", + JSON.stringify({ zoomId: id, rotationPreset: "left" }), + ).document as AxcutDocument; + expect(turned.zoomRanges[0]).toMatchObject({ rotationPreset: "left", depth: 3 }); + + const flat = executeAgentTool( + turned, + "setZoom", + JSON.stringify({ zoomId: id, rotationPreset: "none" }), + ).document as AxcutDocument; + expect(flat.zoomRanges[0]).not.toHaveProperty("rotationPreset"); + + // Left out, it stays as it was; an unknown camera is refused. + const deeper = executeAgentTool(turned, "setZoom", JSON.stringify({ zoomId: id, depth: 5 })); + expect(deeper.document?.zoomRanges[0]).toMatchObject({ rotationPreset: "left", depth: 5 }); + expect( + executeAgentTool(withZoom, "setZoom", JSON.stringify({ zoomId: id, rotationPreset: "up" })) + .ok, + ).toBe(false); + }); + + it("addZoom without a camera stays flat", () => { + const result = executeAgentTool( + fixtureDocument(), + "addZoom", + JSON.stringify({ startSec: 2, endSec: 4 }), + ); + expect(result.document?.zoomRanges[0]).not.toHaveProperty("rotationPreset"); + expect(JSON.parse(result.resultJson)).not.toHaveProperty("rotationPreset"); + }); + it("addSpeed writes a legacyEditor speed region the snapshot exposes", () => { const result = executeAgentTool( fixtureDocument(), diff --git a/electron/ai-edition/agent-tools.ts b/electron/ai-edition/agent-tools.ts index e39022dad1..87a42e8f3c 100644 --- a/electron/ai-edition/agent-tools.ts +++ b/electron/ai-edition/agent-tools.ts @@ -485,13 +485,28 @@ export const getCursorTrackArgs = z.object({ const depthSchema = z.number().int().min(1).max(6); const focusSchema = z.object({ cx: z.number().min(0).max(1), cy: z.number().min(0).max(1) }); +/** The zoom's 3D camera, as the editor's "3D camera" control offers it: the screen turned left or + * right, the camera orbiting the screen, or `none` for a flat screen. Stored and read back + * (getCurrentDocument) as `rotationPreset`, absent when flat. */ +const rotationPresetSchema = z.enum(["none", "left", "right", "orbit"]); + export const addZoomArgs = z.object({ startSec: secondsSchema, endSec: secondsSchema, depth: depthSchema.default(3), focus: focusSchema.default({ cx: 0.5, cy: 0.5 }), + rotationPreset: rotationPresetSchema.default("none"), }); +/** A zoom with its 3D camera set: `none` drops the key, as a flat zoom never carries it. */ +function withRotationPreset( + zoom: Z, + preset: z.infer, +): Z { + const { rotationPreset: _previous, ...flat } = zoom; + return (preset === "none" ? flat : { ...flat, rotationPreset: preset }) as Z; +} + /** Same contract as `addTrimsArgs`: the element schema IS the unitary one, and * it is advertised rather than enforced so a bad region is refused by itself. */ export const addZoomsArgs = z.object({ @@ -504,6 +519,7 @@ export const setZoomArgs = z.object({ endSec: secondsSchema.optional(), depth: depthSchema.optional(), focus: focusSchema.optional(), + rotationPreset: rotationPresetSchema.optional(), }); // The bound every speed reader applies (`readSpeedRegions`): refused here, so the agent hears @@ -2010,15 +2026,18 @@ export function executeAgentTool( if (!parsed.success) return failure(parsed.error.message); const startMs = toMs(Math.min(parsed.data.startSec, parsed.data.endSec)); const endMs = toMs(Math.max(parsed.data.startSec, parsed.data.endSec)); - const zoom = { - id: createId("zoom"), - startMs, - endMs, - depth: parsed.data.depth as 1 | 2 | 3 | 4 | 5 | 6, - focus: parsed.data.focus, - focusMode: "manual" as const, - source: "manual" as const, - }; + const zoom = withRotationPreset( + { + id: createId("zoom"), + startMs, + endMs, + depth: parsed.data.depth as 1 | 2 | 3 | 4 | 5 | 6, + focus: parsed.data.focus, + focusMode: "manual" as const, + source: "manual" as const, + } as AxcutDocument["zoomRanges"][number], + parsed.data.rotationPreset, + ); const placed = anchorForAgent(zoom, document, "zoom"); const landing = landingOf(placed, document); if (!landing.anchored) return coversNoClip("zoom", startMs / 1000, endMs / 1000, document); @@ -2036,6 +2055,7 @@ export function executeAgentTool( resultJson: JSON.stringify({ zoomId: landing.ids[0], depth: zoom.depth, + ...(zoom.rotationPreset ? { rotationPreset: zoom.rotationPreset } : {}), // The depth alone is an ordinal; reported on its own it is what the // model turns into "3×" for a frame that renders 1.80×. renderedScale: effectiveZoomScale(zoom), @@ -2081,13 +2101,16 @@ export function executeAgentTool( document.zoomRanges.map((z) => { if (!zoomPill.has(z.id)) return z; const { customScale, ...rest } = z; - return { + const edited = { ...(clearsCustomScale ? rest : z), ...(parsed.data.depth !== undefined ? { depth: parsed.data.depth as 1 | 2 | 3 | 4 | 5 | 6 } : {}), ...(parsed.data.focus ? { focus: parsed.data.focus } : {}), }; + return parsed.data.rotationPreset === undefined + ? edited + : withRotationPreset(edited, parsed.data.rotationPreset); }), // …then the span: clamped against different-property pills, then re-ventilated. zoomId, @@ -2121,7 +2144,11 @@ export function executeAgentTool( resultJson: JSON.stringify({ zoomId: landing.ids[0] ?? zoomId, ...(strength - ? { depth: strength.depth, renderedScale: effectiveZoomScale(strength) } + ? { + depth: strength.depth, + renderedScale: effectiveZoomScale(strength), + ...(strength.rotationPreset ? { rotationPreset: strength.rotationPreset } : {}), + } : {}), ...(clearsCustomScale ? { clearedCustomScale: true } : {}), ...landingReport(landing, startMs / 1000, endMs / 1000), diff --git a/electron/ai-edition/deep-agent/service.ts b/electron/ai-edition/deep-agent/service.ts index a6fe68089f..5b2df760cd 100644 --- a/electron/ai-edition/deep-agent/service.ts +++ b/electron/ai-edition/deep-agent/service.ts @@ -147,6 +147,10 @@ export function buildSystemPrompt(options: { editsAllowed: boolean }): string { export const SYSTEM_PROMPT = buildSystemPrompt({ editsAllowed: true }); // #1028: a zoom's camera moves sit OUTSIDE its span, where nothing on the ruler shows them. +// The editor's "3D camera" control, in the agent's words. +const ZOOM_CAMERA_NOTE = + "rotationPreset is the zoom's 3D camera: `left` or `right` turns the screen in 3D to that side, `orbit` moves the camera around the screen while the zoom holds, `none` (the default) keeps it flat. It is a strong effect: keep it for a reveal or a key moment, not every zoom. getCurrentDocument reports it as rotationPreset, absent when flat."; + const ZOOM_TRANSITIONS_NOTE = `A zoom HOLDS over its span: it animates in during the window just before startSec and out during the window just after endSec. Each move lasts 0.6 s + 0.55 s × ln(renderedScale) of SCREEN time (${ZOOM_TRANSITION_LEGEND}), so a speed region scales the timeline it covers by its speed: longer above 1×, shorter below. getCurrentDocument gives each zoom's zoomInFromSec and zoomOutUntilSec, in virtual seconds like the zoom. A trim inside either window cuts that move and the export jumps at the cut: leave the windows untrimmed. Two zooms at most 1.5 s apart on screen, on the same clip with no trim between them, are chained: the camera pans from one to the next over 1 s of screen time from the first one's end (panUntilSec), with no zoom-out or zoom-in between them.`; const CUT_TRANSITIONS_RESULT = `When the result cuts a zoom's move in or out, it carries \`cutTransitions\`: which zoom, which side, cutSec, the timeline seconds of the move lost, and by: trim, or junction when the move runs past its clip's edge into the next clip. It is a warning, nothing is refused: move the cut or the zoom unless a hard cut is what you want.`; @@ -173,9 +177,9 @@ export const TOOL_DESCRIPTIONS: Record = { "Reorder a placed clip: move `clipId` so it plays just before `beforeClipId` (pass null, or omit it, to move it last). Ids come from getCurrentDocument, where each clip carries its `index` and its label in `reason`. This preserves every clip id, every source range, every trim, and the zooms / speed regions / annotations anchored to each clip. This is the tool for 'swap these clips', 'put X first' and 'change the clip order' — replaceTimeline cannot reorder anything.", replaceTimeline: "Replace the whole timeline with the given kept intervals of the primary asset's source time. Everything outside the intervals becomes a trim. The intervals are SORTED, so this can never reorder clips — use moveClip for that. DO NOT use this for 'cut silences' or 'remove pauses' — the user has likely placed clips on the timeline that you'd be discarding. Use this ONLY when the user explicitly asks you to rebuild the timeline from scratch (e.g. 'start over with the kept intervals from the transcript'). It is refused when it would merge away, shorten or drop an existing clip; the refusal names them and the tool to use instead.", - addZoom: `Add a zoom-in over a span of the edited timeline (virtual seconds). depth is an ORDINAL 1–6, not a factor: it selects a magnification from a fixed table (${ZOOM_DEPTH_LEGEND}), so the default depth 3 renders at 1.80×. The result reports renderedScale — quote that, never the depth, when telling the user how strong the zoom is. focus is the zoom centre in 0–1 fractions of the frame (default centre). When the recording's pointer telemetry can be read for the footage under the span, the result also carries \`cursorAnchor\`: \`focus\` echoes the value this call used (including the default, if you left it out), \`cursor\` is where the pointer ACTUALLY was over the span the zoom landed on — the median of the recorded samples, \`spread\` being how far the farthest one strays from it — and \`offset\` is the distance between the two, in frame fractions. It is a measurement, not a correction: nothing is moved and no call is refused over it, and a zoom framing a slide, a face, or a region the pointer never enters is a legitimate choice. \`available:false\` names what it found instead (\`no-samples\`, \`trimmed-out\`). Its ABSENCE means no telemetry was read for that footage — never that the recording has none; assets[].hasCursorTelemetry and getCursorTrack are what answer that. Use for 'zoom in on …' and the smart-zoom pass. ${ZOOM_TRANSITIONS_NOTE} ${CUT_TRANSITIONS_RESULT}`, + addZoom: `Add a zoom-in over a span of the edited timeline (virtual seconds). depth is an ORDINAL 1–6, not a factor: it selects a magnification from a fixed table (${ZOOM_DEPTH_LEGEND}), so the default depth 3 renders at 1.80×. The result reports renderedScale — quote that, never the depth, when telling the user how strong the zoom is. focus is the zoom centre in 0–1 fractions of the frame (default centre). When the recording's pointer telemetry can be read for the footage under the span, the result also carries \`cursorAnchor\`: \`focus\` echoes the value this call used (including the default, if you left it out), \`cursor\` is where the pointer ACTUALLY was over the span the zoom landed on — the median of the recorded samples, \`spread\` being how far the farthest one strays from it — and \`offset\` is the distance between the two, in frame fractions. It is a measurement, not a correction: nothing is moved and no call is refused over it, and a zoom framing a slide, a face, or a region the pointer never enters is a legitimate choice. \`available:false\` names what it found instead (\`no-samples\`, \`trimmed-out\`). Its ABSENCE means no telemetry was read for that footage — never that the recording has none; assets[].hasCursorTelemetry and getCursorTrack are what answer that. Use for 'zoom in on …' and the smart-zoom pass. ${ZOOM_CAMERA_NOTE} ${ZOOM_TRANSITIONS_NOTE} ${CUT_TRANSITIONS_RESULT}`, addZooms: `Add MANY zooms in one call: \`regions\` is a list, each entry taking exactly the fields addZoom takes (same depth table, ${ZOOM_DEPTH_LEGEND}). Use this for the smart-zoom pass, where you have decided every zoom before emitting the first one — sending them one at a time costs one round trip each. Each region stands or falls ALONE: one that covers no clip is refused by itself and listed in \`refused\` with its index and the reason, while the others are still applied. The result leads with requested / appliedCount / refusedCount, and each applied entry carries its renderedScale — quote that, never the depth — plus the same \`cursorAnchor\` addZoom reports, whenever the footage under that region has readable pointer telemetry, and the same \`cutTransitions\` warning. ${ZOOM_TRANSITIONS_NOTE}`, - setZoom: `Move, resize, or restyle an existing zoom by id (virtual-timeline seconds). Only the fields you pass are changed. depth selects from the same table (${ZOOM_DEPTH_LEGEND}); if the zoom carries a customScale (getCurrentDocument shows it as depthIsOverridden), that custom value is what renders, and passing depth clears it so the depth takes effect — the result says so. The result reports the resulting renderedScale, and — when the footage under the span has readable pointer telemetry — the same \`cursorAnchor\` addZoom reports, measured against the zoom's EFFECTIVE focus, so a call that moved only the span still learns what its unchanged focus is now looking at. ${ZOOM_TRANSITIONS_NOTE} ${CUT_TRANSITIONS_RESULT}`, + setZoom: `Move, resize, or restyle an existing zoom by id (virtual-timeline seconds). Only the fields you pass are changed. depth selects from the same table (${ZOOM_DEPTH_LEGEND}); if the zoom carries a customScale (getCurrentDocument shows it as depthIsOverridden), that custom value is what renders, and passing depth clears it so the depth takes effect — the result says so. The result reports the resulting renderedScale, and — when the footage under the span has readable pointer telemetry — the same \`cursorAnchor\` addZoom reports, measured against the zoom's EFFECTIVE focus, so a call that moved only the span still learns what its unchanged focus is now looking at. ${ZOOM_CAMERA_NOTE} ${ZOOM_TRANSITIONS_NOTE} ${CUT_TRANSITIONS_RESULT}`, addSpeed: "Add a speed-change region over a span of the edited timeline (virtual seconds). speed > 1 fast-forwards, < 1 slows down, from 0.25× to 16× (default 1.5×). Use to speed through slow stretches without cutting them.", setSpeed: