From 9d7ef54b0e2e293e27ce095d62830fd0f009fd8c Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Mon, 12 Oct 2026 01:06:56 +0200 Subject: [PATCH 1/4] fix(ai-edition): text like the editor's, sets that land where asked, and a format tool From an agent's end-to-end run on macOS, fixed where the cause is so the in-app agent and MCP clients both get it: - addAnnotation places text on the output frame with its box fitted to the text, as the editor's add does, and takes the inspector's look (size, colour, plate, weight, entrance animation); setAnnotation can now move and restyle too. - A set* whose new span would run into another region with other settings is refused, naming it. The editor's drag rule stopped it part of the way back instead, landing it where the caller never asked. - setAspectRatio: the video's shape, written through the editor's own patch so frame text keeps its place. - The time-base guidance said trims shorten the ruler; they leave a hole in it, and the export's clock differs once there are trims or speed regions. - The main-process build resolves @/, so code shared with the renderer no longer needs relative imports. --- electron/ai-edition/agent-tools.test.ts | 135 ++++++++- electron/ai-edition/agent-tools.ts | 270 ++++++++++++++---- .../ai-edition/deep-agent/service.test.ts | 1 + electron/ai-edition/deep-agent/service.ts | 11 +- src/lib/ai-edition/store/settingsPatch.ts | 26 ++ src/lib/ai-edition/store/useEditorSettings.ts | 23 +- .../ai-edition/timeline/zoom-suggestions.ts | 4 +- vite.config.ts | 8 + 8 files changed, 393 insertions(+), 85 deletions(-) create mode 100644 src/lib/ai-edition/store/settingsPatch.ts diff --git a/electron/ai-edition/agent-tools.test.ts b/electron/ai-edition/agent-tools.test.ts index c900ee24be..d0f17a2f63 100644 --- a/electron/ai-edition/agent-tools.test.ts +++ b/electron/ai-edition/agent-tools.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; import { + type AxcutAnnotationRegion, type AxcutDocument, createEmptyDocument, documentSchema, @@ -16,6 +17,15 @@ import { executeAgentTool, isMutatingTool, MUTATING_TOOL_NAMES } from "./agent-t * so pinning it costs nothing and makes every document in the file comparable. */ const FIXTURE_CREATED_AT = "2026-01-01T00:00:00.000Z"; +/** Where an annotation's box is centred, in frame percentages. */ +function centreOf(region: Pick | undefined) { + if (!region) return undefined; + return { + x: region.position.x + region.size.width / 2, + y: region.position.y + region.size.height / 2, + }; +} + function fixtureDocument(): AxcutDocument { const base = createEmptyDocument({ title: "Test", @@ -186,6 +196,7 @@ describe("the mutating-tool table", () => { "removeWords", "replaceTimeline", "setAnnotation", + "setAspectRatio", "setAudio", "setCameraFullscreen", "setClipRange", @@ -685,9 +696,11 @@ describe("executeAgentTool", () => { endMs: 3000, type: "text", textContent: "Look here", - // x/y are the text's centre: a 30x20 box centred on (20, 80). - position: { x: 5, y: 70 }, + // On the output frame, as the editor places text: padding and zooms never move it. + space: "frame", }); + // x/y are the text's centre, and the box is fitted to the text around it. + expect(centreOf(ann)).toEqual({ x: expect.closeTo(20, 5), y: expect.closeTo(80, 5) }); expect(() => documentSchema.parse(result.document)).not.toThrow(); }); @@ -697,16 +710,124 @@ describe("executeAgentTool", () => { "addAnnotation", JSON.stringify({ startSec: 1, endSec: 3, text: "Hi" }), ).document?.annotations.at(-1); - expect(centred).toMatchObject({ - position: { x: 35, y: 40 }, - style: { backgroundColor: "rgba(0, 0, 0, 0.7)" }, - }); + expect(centred).toMatchObject({ style: { backgroundColor: "rgba(0, 0, 0, 0.7)" } }); + expect(centreOf(centred)).toEqual({ x: expect.closeTo(50, 5), y: expect.closeTo(50, 5) }); const corner = executeAgentTool( fixtureDocument(), "addAnnotation", JSON.stringify({ startSec: 1, endSec: 3, text: "Hi", x: 100, y: 0 }), ).document?.annotations.at(-1); - expect(corner).toMatchObject({ position: { x: 70, y: 0 } }); + expect(corner?.position.y).toBe(0); + expect((corner?.position.x ?? 0) + (corner?.size.width ?? 0)).toBeCloseTo(100, 5); + }); + + it("addAnnotation takes the look the editor's inspector offers", () => { + const add = (look: Record) => + executeAgentTool( + fixtureDocument(), + "addAnnotation", + JSON.stringify({ startSec: 1, endSec: 3, text: "Palettes in one click", ...look }), + ).document?.annotations.at(-1); + const plain = add({}); + const title = add({ + fontSize: 72, + color: "#ffcc00", + background: "none", + bold: false, + animation: "typewriter", + }); + expect(title?.style).toMatchObject({ + fontSize: 72, + color: "#ffcc00", + backgroundColor: "transparent", + fontWeight: "normal", + textAnimation: "typewriter", + }); + // Bigger text, bigger box: the box is the text's. + expect(title?.size.width).toBeGreaterThan(plain?.size.width ?? 0); + }); + + it("setAnnotation moves the text, restyles it and refits its box around its centre", () => { + const added = executeAgentTool( + fixtureDocument(), + "addAnnotation", + JSON.stringify({ startSec: 1, endSec: 3, text: "Hi", x: 30, y: 30 }), + ); + const id = JSON.parse(added.resultJson).annotationId; + const moved = executeAgentTool( + added.document as AxcutDocument, + "setAnnotation", + JSON.stringify({ annotationId: id, x: 70, y: 85, animation: "pop" }), + ).document?.annotations.find((a) => a.id === id); + expect(centreOf(moved)).toEqual({ x: expect.closeTo(70, 5), y: expect.closeTo(85, 5) }); + expect(moved?.style.textAnimation).toBe("pop"); + + const retexted = executeAgentTool( + added.document as AxcutDocument, + "setAnnotation", + JSON.stringify({ annotationId: id, text: "A much longer callout than before" }), + ).document?.annotations.find((a) => a.id === id); + expect(retexted?.size.width).toBeGreaterThan(moved?.size.width ?? 0); + expect(centreOf(retexted)).toEqual({ x: expect.closeTo(30, 5), y: expect.closeTo(30, 5) }); + }); + + it("setAspectRatio sets the video's shape, keeping frame text where it was", () => { + const added = executeAgentTool( + fixtureDocument(), + "addAnnotation", + JSON.stringify({ startSec: 1, endSec: 3, text: "Centred", x: 50, y: 50 }), + ); + const result = executeAgentTool( + added.document as AxcutDocument, + "setAspectRatio", + JSON.stringify({ aspectRatio: "9:16" }), + ); + expect(result.ok).toBe(true); + expect(JSON.parse(result.resultJson)).toMatchObject({ aspectRatio: "9:16" }); + expect((result.document?.legacyEditor as { aspectRatio?: string }).aspectRatio).toBe("9:16"); + expect(centreOf(result.document?.annotations.at(-1))).toEqual({ + x: expect.closeTo(50, 5), + y: expect.closeTo(50, 5), + }); + expect( + executeAgentTool(fixtureDocument(), "setAspectRatio", JSON.stringify({ aspectRatio: "7:3" })) + .ok, + ).toBe(false); + }); + + it("refuses a set that would run into another region, instead of landing it elsewhere", () => { + let document = fixtureDocument(); + const add = (name: string, args: Record) => { + const result = executeAgentTool(document, name, JSON.stringify(args)); + document = result.document as AxcutDocument; + return result; + }; + const first = add("addAnnotation", { startSec: 1, endSec: 2, text: "One" }); + add("addAnnotation", { startSec: 6, endSec: 8, text: "Two" }); + const firstId = JSON.parse(first.resultJson).annotationId; + const blocked = executeAgentTool( + document, + "setAnnotation", + JSON.stringify({ annotationId: firstId, startSec: 7, endSec: 9 }), + ); + expect(blocked.ok).toBe(false); + expect(blocked.document).toBeUndefined(); + expect(blocked.resultJson).toMatch(/NOT changed.*6\.0|NOT changed/); + + // The same rule for every kind of effect. + const zoom = add("addZoom", { startSec: 1, endSec: 2, depth: 2 }); + add("addZoom", { startSec: 6, endSec: 8, depth: 4 }); + const zoomId = JSON.parse(zoom.resultJson).zoomId; + const zoomBlocked = executeAgentTool( + document, + "setZoom", + JSON.stringify({ zoomId, startSec: 7, endSec: 9 }), + ); + expect(zoomBlocked.ok).toBe(false); + // A span clear of it still moves. + expect( + executeAgentTool(document, "setZoom", JSON.stringify({ zoomId, startSec: 3, endSec: 4 })).ok, + ).toBe(true); }); it("snapshot exposes clips/trims/effects as virtual-time groups with a time-base note", () => { diff --git a/electron/ai-edition/agent-tools.ts b/electron/ai-edition/agent-tools.ts index e39022dad1..3610270e6b 100644 --- a/electron/ai-edition/agent-tools.ts +++ b/electron/ai-edition/agent-tools.ts @@ -17,6 +17,7 @@ import { z } from "zod"; import { DEFAULT_TEXT_PLATE } from "../../src/lib/ai-edition/annotations/background"; +import { fitTextBox } from "../../src/lib/ai-edition/annotations/placement"; import { collapseTracksToPills, patchAudioTrack, @@ -24,6 +25,7 @@ import { trackGroupId, } from "../../src/lib/ai-edition/document/audioTracks"; import { createId } from "../../src/lib/ai-edition/document/ids"; +import { resolveAspectRatioValue } from "../../src/lib/ai-edition/document/outputFormat"; import { moveClip, planTimelineReplacement, @@ -34,7 +36,9 @@ import { setClipSourceRange, } from "../../src/lib/ai-edition/document/timeline"; import { setDocumentWordText } from "../../src/lib/ai-edition/document/transcript"; -import type { AxcutDocument } from "../../src/lib/ai-edition/schema"; +import type { AxcutAnnotationRegion, AxcutDocument } from "../../src/lib/ai-edition/schema"; +import { getEditorSettings } from "../../src/lib/ai-edition/store/editorSettings"; +import { applySettingsPatch } from "../../src/lib/ai-edition/store/settingsPatch"; import { hasAnyClipWithCamera } from "../../src/lib/ai-edition/timeline/camera"; import { isGeneratedAssetId } from "../../src/lib/ai-edition/timeline/clip-parts"; import { @@ -52,10 +56,6 @@ import { trimAppliesToClip, trimToTimelineSpan, } from "../../src/lib/ai-edition/timeline/trim-mapping"; -// ponytail: relative, and it has to stay that way — `electron/` never resolves -// the `@/` alias (the main-process build does not declare it), which is why the -// scale table was moved out of `components/video-editor/types.ts` to be -// reachable from here at all. import { effectiveZoomScale, ZOOM_DEPTH_LEGEND, @@ -66,6 +66,7 @@ import { zoomTransitions, } from "../../src/lib/ai-edition/timeline/zoom-transitions"; import { SETTING_BOUNDS } from "../../src/lib/projectDefaults"; +import { ASPECT_RATIO_PRESETS } from "../../src/utils/aspectRatioUtils"; export interface AgentToolExecution { ok: boolean; @@ -129,6 +130,93 @@ function resolveSpanMs( return { startMs: toMs(Math.min(s, e)), endMs: toMs(Math.max(s, e)) }; } +/** + * For the effect set* tools: the region another span of the same kind would have to cross. + * + * `replacePillSpan` follows the editor's drag rule: a region dragged into a neighbour with other + * settings stops against it, part of the way back to where it started. That is right under a + * mouse, which shows the stop as it happens. For a caller who named an exact span it put the + * region somewhere it never asked for (asked 7.6 – 9.5 s, landed 1.5 – 3.3 s), so a set that + * would be stopped is refused instead, naming what is in the way. The neighbours are the same + * walls the drag rule sees: other pills with other settings, not ones the region already overlaps. + */ +function refuseIfBlocked( + kind: string, + regions: T[], + id: string, + span: { startMs: number; endMs: number }, +): AgentToolExecution | null { + const pills = coalesceRegionsForRuler(regions); + const pill = pills.find((p) => p.ids.includes(id)); + if (!pill) return null; + const start = span.startMs / 1000; + const end = span.endMs / 1000; + const wall = pills.find( + (other) => + other !== pill && + other.identity !== pill.identity && + (other.end <= pill.start || other.start >= pill.end) && + other.start < end && + other.end > start, + ); + if (!wall) return null; + return failure( + `The ${kind} was NOT changed: ${formatSec(start)} – ${formatSec(end)} would run into ${kind} ` + + `${wall.ids[0]} at ${formatSec(wall.start)} – ${formatSec(wall.end)}, and two ${kind}s with ` + + "different settings cannot overlap. Move or shorten that one first, or pick a span clear of it.", + ); +} + +type AnnotationLook = { + fontSize?: number; + color?: string; + background?: string; + bold?: boolean; + animation?: NonNullable; +}; + +/** The look arguments, written onto a style the way the editor's inspector writes them. */ +function applyAnnotationLook( + style: AxcutAnnotationRegion["style"], + look: AnnotationLook, +): AxcutAnnotationRegion["style"] { + const next = { ...style }; + if (look.fontSize !== undefined) next.fontSize = look.fontSize; + if (look.color !== undefined) next.color = look.color; + if (look.bold !== undefined) next.fontWeight = look.bold ? "bold" : "normal"; + if (look.animation !== undefined) next.textAnimation = look.animation; + if (look.background !== undefined) { + if (look.background === "none") { + // As the inspector's switch does: keep the colour, so turning it back on restores it. + if (style.backgroundColor !== "transparent") next.lastBackgroundColor = style.backgroundColor; + next.backgroundColor = "transparent"; + } else { + next.backgroundColor = look.background; + } + } + return next; +} + +/** + * A text annotation on the output frame, its box sized to its text around `center` (frame + * percentages): what the editor's "add text" does, so the agent's text sits where the editor's + * would, is measured the same, and stays put when the picture zooms. + */ +function placeTextOnFrame( + region: T, + center: { x: number; y: number }, + document: AxcutDocument, +): T { + const frameAspect = resolveAspectRatioValue(document, getEditorSettings(document).aspectRatio); + const centred = { + ...region, + space: "frame" as const, + position: center, + size: { width: 0, height: 0 }, + }; + return { ...centred, ...fitTextBox(centred, frameAspect) }; +} + // v5: a modifier is stored as clip-anchored fragment(s), and adjacent regions with the // same properties read as ONE pill (timelineMap merge rule). The agent reasons in VIRTUAL // seconds over whole regions, so we present exactly the pills the user sees, keyed by the @@ -442,6 +530,10 @@ export const setClipRangeArgs = z.object({ sourceEndSec: secondsSchema, }); +export const setAspectRatioArgs = z.object({ + aspectRatio: z.enum(["auto", ...ASPECT_RATIO_PRESETS]), +}); + export const replaceTimelineArgs = z.object({ intervals: z.array(z.object({ startSec: secondsSchema, endSec: secondsSchema })).min(1), reason: z.string().default(""), @@ -526,8 +618,25 @@ export const setSpeedArgs = z.object({ speed: speedSchema.optional(), }); -/** A new text annotation's box, in frame percentages: the editor's own size for one. */ -const ANNOTATION_BOX = { width: 30, height: 20 } as const; +/** How a text annotation looks: the settings the editor's inspector offers for one. */ +const annotationLookArgs = { + fontSize: z + .number() + .min(8) + .max(200) + .optional() + .describe("Text size; 32 is the editor's default, a title reads well at 56 to 80."), + color: z.string().optional().describe("Text colour, a CSS colour such as #ffffff."), + background: z + .string() + .optional() + .describe('Colour of the plate behind the text, or "none" for no plate.'), + bold: z.boolean().optional(), + animation: z + .enum(["none", "fade", "rise", "pop", "slide-left", "typewriter", "pulse"]) + .optional() + .describe("How the text enters: typewriter types it out, pop springs it in, and so on."), +}; export const addAnnotationArgs = z.object({ startSec: secondsSchema, @@ -535,6 +644,7 @@ export const addAnnotationArgs = z.object({ text: z.string().default(""), x: z.number().min(0).max(100).default(50), y: z.number().min(0).max(100).default(50), + ...annotationLookArgs, }); export const setAnnotationArgs = z.object({ @@ -542,6 +652,9 @@ export const setAnnotationArgs = z.object({ startSec: secondsSchema.optional(), endSec: secondsSchema.optional(), text: z.string().optional(), + x: z.number().min(0).max(100).optional(), + y: z.number().min(0).max(100).optional(), + ...annotationLookArgs, }); export const addAudioArgs = z.object({ @@ -646,6 +759,7 @@ export const OPENSCREEN_TOOL_NAMES = [ "removeTrim", "removeModifier", "removeClip", + "setAspectRatio", ] as const; /** @@ -718,6 +832,7 @@ export const MUTATING_TOOL_NAMES: ReadonlySet = new Set([ "removeTrim", "removeModifier", "removeClip", + "setAspectRatio", ]); export function isMutatingTool(name: string): boolean { @@ -851,7 +966,7 @@ export function documentSnapshotForModel( const autoFocusAll = legacy?.autoFocusAll === true; return { timeBaseNote: - "clips and trims are in source-time seconds; zooms, speedRegions, annotations, cameraFullscreenRegions and audioTracks are in virtual (edited-timeline) seconds.", + "clips and trims are in source-time seconds; zooms, speedRegions, annotations, cameraFullscreenRegions and audioTracks are in virtual (edited-timeline) seconds: the editor's ruler, where a trim leaves a hole instead of shortening it and a speed region stretches nothing. The exported video's clock skips those holes and plays sped-up spans faster, so it differs from the ruler once there are any.", audioNote: "audioTracks are imported voiceover / music files laid over the recording. They are clip-anchored like every other region, so they travel with their clip through reorder and trim, and they play at 1x whatever a speed region does to the picture under them. addAudio places an EXISTING asset of kind 'audio'; nothing here can import a file from disk or record one, so if the project has no audio asset, say so rather than inventing an id.", zoomNote: @@ -1326,6 +1441,20 @@ export function executeAgentTool( } switch (name) { + case "setAspectRatio": { + const parsed = setAspectRatioArgs.safeParse(args); + if (!parsed.success) return failure(parsed.error.message); + const previous = getEditorSettings(document).aspectRatio; + // The editor's own write: the frame-placed annotations keep their shape and centre. + const next = applySettingsPatch(document, { aspectRatio: parsed.data.aspectRatio }); + return { + ok: true, + document: next, + resultJson: JSON.stringify({ aspectRatio: parsed.data.aspectRatio, previous }), + summary: `set the format to ${parsed.data.aspectRatio}`, + }; + } + case "getCurrentDocument": { return { ok: true, @@ -2063,6 +2192,8 @@ export function executeAgentTool( const existing = document.zoomRanges.find((z) => z.id === zoomId); if (!existing) return failure(`Unknown zoom: ${zoomId}`); const { startMs, endMs } = resolveSpanMs(existing, parsed.data.startSec, parsed.data.endSec); + const blocked = refuseIfBlocked("zoom", document.zoomRanges, zoomId, { startMs, endMs }); + if (blocked) return blocked; const zoomPill = new Set(resolvePillIds(document.zoomRanges, zoomId)); // ponytail: a depth write CLEARS customScale, and that is the whole // point. `effectiveZoomScale` returns customScale when it is set, so on a @@ -2179,6 +2310,11 @@ export function executeAgentTool( const speedPill = new Set(resolvePillIds(prev, parsed.data.speedId)); if (!existing) return failure(`Unknown speed region: ${parsed.data.speedId}`); const { startMs, endMs } = resolveSpanMs(existing, parsed.data.startSec, parsed.data.endSec); + const blocked = refuseIfBlocked("speed region", prev, parsed.data.speedId, { + startMs, + endMs, + }); + if (blocked) return blocked; const speed = parsed.data.speed ?? existing.speed; const rebuiltSpeeds = replacePillSpan( prev.map((s) => (speedPill.has(s.id) ? { ...s, speed } : s)), @@ -2215,39 +2351,37 @@ export function executeAgentTool( if (!parsed.success) return failure(parsed.error.message); const startMs = toMs(Math.min(parsed.data.startSec, parsed.data.endSec)); const endMs = toMs(Math.max(parsed.data.startSec, parsed.data.endSec)); - const ann = { - id: createId("ann"), - startMs, - endMs, - type: "text" as const, - content: parsed.data.text, - textContent: parsed.data.text, - // x/y name the text's centre, which is what "default centre" always promised: the - // position is the box's top-left corner, so {50, 50} dropped the text into the - // bottom-right quarter. Kept whole inside the frame. - position: { - x: Math.min( - 100 - ANNOTATION_BOX.width, - Math.max(0, parsed.data.x - ANNOTATION_BOX.width / 2), - ), - y: Math.min( - 100 - ANNOTATION_BOX.height, - Math.max(0, parsed.data.y - ANNOTATION_BOX.height / 2), + // The editor's new text: its defaults, then whatever look the caller asked for, on the + // output frame with the box fitted to the text around x/y (the text's centre). + const ann = placeTextOnFrame( + { + id: createId("ann"), + startMs, + endMs, + type: "text" as const, + content: parsed.data.text, + textContent: parsed.data.text, + position: { x: 0, y: 0 }, + size: { width: 0, height: 0 }, + style: applyAnnotationLook( + { + color: "#ffffff", + backgroundColor: DEFAULT_TEXT_PLATE, + fontSize: 32, + fontFamily: "Inter", + fontWeight: "bold", + fontStyle: "normal", + textDecoration: "none", + textAlign: "center", + textAnimation: "none", + }, + parsed.data, ), - }, - size: { ...ANNOTATION_BOX }, - style: { - color: "#ffffff", - backgroundColor: DEFAULT_TEXT_PLATE, - fontSize: 32, - fontFamily: "Inter", - fontWeight: "bold" as const, - fontStyle: "normal" as const, - textDecoration: "none" as const, - textAlign: "center" as const, - }, - zIndex: document.annotations.length + 1, - }; + zIndex: document.annotations.length + 1, + } as AxcutAnnotationRegion, + { x: parsed.data.x, y: parsed.data.y }, + document, + ); const placed = anchorForAgent(ann, document, "ann"); const landing = landingOf(placed, document); if (!landing.anchored) { @@ -2278,17 +2412,48 @@ export function executeAgentTool( const annPill = new Set(resolvePillIds(document.annotations, annotationId)); if (!existing) return failure(`Unknown annotation: ${annotationId}`); const { startMs, endMs } = resolveSpanMs(existing, parsed.data.startSec, parsed.data.endSec); + const blocked = refuseIfBlocked("annotation", document.annotations, annotationId, { + startMs, + endMs, + }); + if (blocked) return blocked; + const { x, y, text } = parsed.data; + const edit = (a: AxcutAnnotationRegion): AxcutAnnotationRegion => { + const next: AxcutAnnotationRegion = { + ...a, + ...(text !== undefined ? { content: text, textContent: text } : {}), + style: applyAnnotationLook(a.style, parsed.data), + }; + const centre = { + x: x ?? a.position.x + a.size.width / 2, + y: y ?? a.position.y + a.size.height / 2, + }; + const moved = x !== undefined || y !== undefined; + if (a.type === "text") { + // A text box is the size of its text: refit it whenever the text, its size or its + // weight changes, around its centre or the new one. One saved before frame placement + // moves onto the frame the first time it is moved, as the editor does. + const resized = + text !== undefined || + parsed.data.fontSize !== undefined || + parsed.data.bold !== undefined; + return moved || (resized && a.space === "frame") + ? placeTextOnFrame(next, centre, document) + : next; + } + if (!moved) return next; + // An image or an arrow keeps its size and moves its centre, inside its own box. + const { width, height } = a.size; + return { + ...next, + position: { + x: Math.min(Math.max(centre.x - width / 2, 0), Math.max(0, 100 - width)), + y: Math.min(Math.max(centre.y - height / 2, 0), Math.max(0, 100 - height)), + }, + }; + }; const rebuiltAnnotations = replacePillSpan( - document.annotations.map((a) => - annPill.has(a.id) - ? { - ...a, - ...(parsed.data.text !== undefined - ? { content: parsed.data.text, textContent: parsed.data.text } - : {}), - } - : a, - ), + document.annotations.map((a) => (annPill.has(a.id) ? edit(a) : a)), annotationId, startMs, endMs, @@ -2362,6 +2527,11 @@ export function executeAgentTool( if (!existing) return failure(`Unknown full-camera region: ${parsed.data.cameraFullscreenId}`); const { startMs, endMs } = resolveSpanMs(existing, parsed.data.startSec, parsed.data.endSec); + const blocked = refuseIfBlocked("full-camera region", prev, parsed.data.cameraFullscreenId, { + startMs, + endMs, + }); + if (blocked) return blocked; const camPill = new Set(resolvePillIds(prev, parsed.data.cameraFullscreenId)); const rebuiltCamera = replacePillSpan( prev, diff --git a/electron/ai-edition/deep-agent/service.test.ts b/electron/ai-edition/deep-agent/service.test.ts index aa174e6eec..104c64abc0 100644 --- a/electron/ai-edition/deep-agent/service.test.ts +++ b/electron/ai-edition/deep-agent/service.test.ts @@ -83,6 +83,7 @@ const ARGS: Record = { removeTrim: { trimRangeId: "trim_1" }, removeModifier: { id: "nope" }, removeClip: { clipId: "clip_1" }, + setAspectRatio: { aspectRatio: "16:9" }, }; /** ponytail: the fixture starts from a FIXED instant. `createEmptyDocument` reads diff --git a/electron/ai-edition/deep-agent/service.ts b/electron/ai-edition/deep-agent/service.ts index a6fe68089f..d11ff757f8 100644 --- a/electron/ai-edition/deep-agent/service.ts +++ b/electron/ai-edition/deep-agent/service.ts @@ -50,6 +50,7 @@ import { replaceTimelineArgs, resolveCursorAssetId, setAnnotationArgs, + setAspectRatioArgs, setAudioArgs, setCameraFullscreenArgs, setClipRangeArgs, @@ -111,7 +112,7 @@ const BASE_SYSTEM_PROMPT = [ "You can call the tools below against the live document snapshot; the runtime executes each edit and feeds the result back into the loop.", "The AxcutDocument is the single source of truth. The timeline, the transcript editor, and the chat panel are all direct editors of the same document — when the user places a clip on the timeline, the document updates immediately, and when the timeline is empty, the document has no clips. Your edits operate on the live document, so preserve the user's placed clips.", "", - "Time-bases (do not mix them up): clips and trims are in SOURCE-time seconds of an asset; zooms, speed regions, annotations and camera-fullscreen regions are in VIRTUAL (edited-timeline) seconds — the position on the ruler after clips + trims are applied. getCurrentDocument returns all of them, clearly labelled.", + "Time-bases (do not mix them up): clips and trims are in SOURCE-time seconds of an asset; zooms, speed regions, annotations and camera-fullscreen regions are in VIRTUAL (edited-timeline) seconds — the position on the editor's ruler, where the clips sit end to end. A trim leaves a hole in that ruler rather than shortening it, and a speed region does not stretch it, so the exported video's own clock differs from it as soon as there is a trim or a speed region: always place effects in ruler seconds. getCurrentDocument returns all of them, clearly labelled.", "", // ponytail: these describe what each tool is FOR, deliberately without quoting // user phrasings. A phrase→tool table reads as helpful and is not: it swaps the @@ -123,6 +124,7 @@ const BASE_SYSTEM_PROMPT = [ "- Cutting specific spoken words (fillers, a repeated or flubbed phrase, a false start, a duplicate take) is removeWords: read getTranscriptWords, choose the occurrences from their surrounding speech, then pass that assetId and ONLY those word IDs. Word IDs restart for each asset, so keep the read's assetId. Consecutive words become one cut. Do not cut every occurrence of a word just because its text matches one you were asked to remove.", "- Changing where a clip starts or ends within its source is setClipRange — the clip's in/out, distinct from a trim.", `- addZoom takes a virtual-timeline span (depth is an ordinal 1–6 selecting from a fixed table — ${ZOOM_DEPTH_LEGEND} — never a multiplier; focus in 0–1 frame fractions). addSpeed changes pacing over a span. addAnnotation puts text on screen. addCameraFullscreen enlarges the webcam, and only does something where assets[].hasCameraTrack is true.`, + "- setAspectRatio picks the shape of the whole video, for where it will be shown; auto follows the recording.", "- A zoom animates in just before its span and out just after it (zoomInFromSec, zoomOutUntilSec in getCurrentDocument). Leave those windows untrimmed, or the export jumps at the cut.", "- addAudio lays an imported voiceover or music file over a span. It plays an asset the project already has (kind 'audio'); importing or recording one is the editor's job, not a tool you have — so when the project has none, say so rather than naming an id that does not exist.", "- moveClip changes the order of placed clips, one call per clip that moves, preserving ids, source ranges, trims and anchored effects. replaceTimeline rebuilds the timeline from kept intervals and sorts them, so it cannot reorder anything.", @@ -181,9 +183,11 @@ export const TOOL_DESCRIPTIONS: Record = { setSpeed: "Move, resize, or change the multiplier of an existing speed region by id (virtual-timeline seconds). Only the fields you pass are changed.", addAnnotation: - "Add a text annotation over a span of the edited timeline (virtual seconds). x/y place the text's centre, in frame percentages (0–100, default the frame's centre); the box stays inside the frame. It is drawn on a dark plate so it reads on any footage. Use for callouts and labels.", + "Add a text annotation over a span of the edited timeline (virtual seconds). It sits on the output frame, the whole picture padding included, as text added in the editor does: zooms and padding never move it. x/y place the text's centre in frame percentages (0–100, default the centre); the box is sized to the text and kept inside the frame. By default it is drawn on a dark plate so it reads on any footage; fontSize, color, background, bold and animation set its look. Use for callouts, labels and titles.", setAnnotation: - "Move, resize, or edit the text of an existing annotation by id (virtual-timeline seconds). Only the fields you pass are changed.", + "Change an existing annotation by id: its span (virtual-timeline seconds), its text, its centre (x/y, frame percentages) or its look (fontSize, color, background, bold, animation). Only the fields you pass are changed; a text box is refitted to its text around its centre. A new span that would run into another annotation is refused, naming it.", + setAspectRatio: + "Set the shape of the exported video: 16:9 for X, YouTube or a slide, 9:16 for a vertical story or short, 1:1 or 4:5 for a square feed, auto to follow the recording. Annotations on the frame keep their place and shape.", addCameraFullscreen: "Add a camera-fullscreen region over a span of the edited timeline (virtual seconds): the webcam fills the frame for that span. This only does something when the footage under that span comes from an asset with a linked webcam — check assets[].hasCameraTrack (or hasAnyCamera) in getCurrentDocument first. On footage with no camera the call is refused rather than storing a region that would render nothing; say so instead of retrying.", setCameraFullscreen: @@ -393,6 +397,7 @@ export const TOOL_ARG_SCHEMAS: ReadonlyArray = [ ["removeTrim", removeTrimArgs], ["removeModifier", removeModifierArgs], ["removeClip", removeClipArgs], + ["setAspectRatio", setAspectRatioArgs], ]; /** diff --git a/src/lib/ai-edition/store/settingsPatch.ts b/src/lib/ai-edition/store/settingsPatch.ts new file mode 100644 index 0000000000..89d00dc6b2 --- /dev/null +++ b/src/lib/ai-edition/store/settingsPatch.ts @@ -0,0 +1,26 @@ +// A settings patch applied to a document, with what it implies for the rest of it. Pure, so the +// editor's hook and the agent's tools apply a setting through the same code. + +import { refitFrameAnnotations } from "../annotations/placement"; +import { resolveAspectRatioValue } from "../document/outputFormat"; +import type { AxcutDocument } from "../schema"; +import { type EditorSettingsPatch, getEditorSettings, patchEditorSettings } from "./editorSettings"; + +/** + * `patchEditorSettings`, plus what a new format does to the annotations placed on the frame: + * each keeps its shape and its centre (`refitFrameAnnotations`), in the same write, so one undo + * takes both back. + */ +export function applySettingsPatch(doc: AxcutDocument, patch: EditorSettingsPatch): AxcutDocument { + const next = patchEditorSettings(doc, patch); + // ponytail: only a format picked in the settings refits. Under Auto the frame can also + // reshape when clips change, and those annotations keep their shares of the frame; refit + // there too if that shows up in practice. + if (patch.aspectRatio === undefined) return next; + const annotations = refitFrameAnnotations( + next.annotations, + resolveAspectRatioValue(doc, getEditorSettings(doc).aspectRatio), + resolveAspectRatioValue(next, getEditorSettings(next).aspectRatio), + ); + return annotations === next.annotations ? next : { ...next, annotations }; +} diff --git a/src/lib/ai-edition/store/useEditorSettings.ts b/src/lib/ai-edition/store/useEditorSettings.ts index d104f54548..11f7698c9e 100644 --- a/src/lib/ai-edition/store/useEditorSettings.ts +++ b/src/lib/ai-edition/store/useEditorSettings.ts @@ -13,35 +13,14 @@ // extra state, no caches — the document is the single source of truth. import { useCallback, useEffect, useMemo, useRef } from "react"; -import { refitFrameAnnotations } from "../annotations/placement"; -import { resolveAspectRatioValue } from "../document/outputFormat"; import type { AxcutDocument } from "../schema"; import { type EditorSettingsPatch, type EditorSettingsSnapshot, getEditorSettings, - patchEditorSettings, } from "./editorSettings"; import { useProjectStore } from "./projectStore"; - -/** - * `patchEditorSettings`, plus what a new format does to the annotations placed on the frame: - * each keeps its shape and its centre (`refitFrameAnnotations`), in the same write, so one undo - * takes both back. - */ -export function applySettingsPatch(doc: AxcutDocument, patch: EditorSettingsPatch): AxcutDocument { - const next = patchEditorSettings(doc, patch); - // ponytail: only a format picked in the settings refits. Under Auto the frame can also - // reshape when clips change, and those annotations keep their shares of the frame; refit - // there too if that shows up in practice. - if (patch.aspectRatio === undefined) return next; - const annotations = refitFrameAnnotations( - next.annotations, - resolveAspectRatioValue(doc, getEditorSettings(doc).aspectRatio), - resolveAspectRatioValue(next, getEditorSettings(next).aspectRatio), - ); - return annotations === next.annotations ? next : { ...next, annotations }; -} +import { applySettingsPatch } from "./settingsPatch"; export interface UseEditorSettingsResult { settings: EditorSettingsSnapshot; diff --git a/src/lib/ai-edition/timeline/zoom-suggestions.ts b/src/lib/ai-edition/timeline/zoom-suggestions.ts index 9ca9709b53..8df686f48d 100644 --- a/src/lib/ai-edition/timeline/zoom-suggestions.ts +++ b/src/lib/ai-edition/timeline/zoom-suggestions.ts @@ -12,9 +12,7 @@ // automatic zoom. import type { CursorTelemetryPoint, ZoomFocus } from "@/components/video-editor/types"; -// Relative, not `@/`: the main process plans a fresh MCP take's zooms with this file, -// and its build does not resolve the alias. -import { zoomTransitionMs } from "../../zoomMath/constants"; +import { zoomTransitionMs } from "@/lib/zoomMath/constants"; import type { AxcutClip } from "../schema"; import { DEFAULT_ZOOM_DEPTH, ZOOM_DEPTH_SCALES, type ZoomDepth } from "./zoom-scale"; diff --git a/vite.config.ts b/vite.config.ts index 7261f2197d..cb1bffcbf4 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -33,6 +33,14 @@ export default defineConfig({ }, vite: { build: {}, + // The main process runs code from `src/lib` too (the agent's tools, the MCP + // server): it resolves `@/` as the renderer does, or every such import fails + // the build. + resolve: { + alias: { + "@": path.resolve(__dirname, "src"), + }, + }, }, }, preload: { From 4d8aa2b8cdde37985a83a1a278bb796bd0fd3d7d Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Mon, 12 Oct 2026 01:06:59 +0200 Subject: [PATCH 2/4] feat(mcp): getFrames reads a project's export at ruler seconds getFrames with a projectId looks at the project's last export, mapping the ruler seconds the document tools use through trims and speed regions with the exporter's own projection, and says when the project changed after that export. The skill teaches one clock, the format first, and text styles, and that nothing but the app belongs on screen. --- electron/mcp/media-tools.test.ts | 30 ++++++++++ electron/mcp/media-tools.ts | 94 +++++++++++++++++++++++++++----- skills/openscreen-demo/SKILL.md | 15 +++-- 3 files changed, 120 insertions(+), 19 deletions(-) diff --git a/electron/mcp/media-tools.test.ts b/electron/mcp/media-tools.test.ts index d0745748d0..0470c15a67 100644 --- a/electron/mcp/media-tools.test.ts +++ b/electron/mcp/media-tools.test.ts @@ -399,6 +399,36 @@ describe("exportVideo", () => { }); }); +describe("getFrames on a project", () => { + it("reads the last export at the ruler seconds the document tools use", async () => { + const { projectId } = await recordTake(); + expect(textOf(await call("getFrames", { projectId, atSec: [1] }))).toContain("no export yet"); + + // A cut of 2 s early in the take: the ruler keeps a hole there, the export does not. + expect((await call("addTrim", { projectId, startSec: 2, endSec: 4 })).isError).toBeFalsy(); + runners.results.export = { success: true, outputPath: path.join(dir, "demo.mp4") }; + await call("exportVideo", { projectId, path: path.join(dir, "demo.mp4") }); + + const frames = await call("getFrames", { projectId, atSec: [1, 6] }); + const labels = frames.content.flatMap((part) => (part.type === "text" ? [part.text] : [])); + expect(labels.slice(1)).toEqual([ + "1.00s on the ruler, 1.00s into the video", + "6.00s on the ruler, 4.00s into the video", + ]); + expect(labels[0]).not.toContain("changed after this export"); + + // An edit after the export is flagged, so a stale video is never taken for the edit. + await call("addAnnotation", { projectId, startSec: 5, endSec: 7, text: "New" }); + const stale = await call("getFrames", { projectId, atSec: [6] }); + expect(textOf(stale)).toContain("changed after this export"); + }); + + it("takes a project or a file, not both", async () => { + expect((await call("getFrames", {})).isError).toBe(true); + expect((await call("getFrames", { projectId: "p", path: "x.mp4" })).isError).toBe(true); + }); +}); + describe("getFrames", () => { it("returns each frame as an image, labelled with its time", async () => { const result = await call("getFrames", { path: "take.mp4", atSec: [1, 5] }); diff --git a/electron/mcp/media-tools.ts b/electron/mcp/media-tools.ts index 463cd86580..8472b81d61 100644 --- a/electron/mcp/media-tools.ts +++ b/electron/mcp/media-tools.ts @@ -12,6 +12,10 @@ import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; import { z } from "zod"; import type { CursorTelemetryPoint } from "../../src/components/video-editor/types"; +import { + projectRawTimelineSecToPlayback, + readSpeedRegions, +} from "../../src/lib/ai-edition/document/timeline"; import type { AxcutDocument } from "../../src/lib/ai-edition/schema"; import { appendAutoZoomSuggestions, @@ -41,6 +45,7 @@ export interface MediaToolDeps { | "addAsset" | "saveProject" | "filePathFor" + | "getProject" | "getProjectForUpdate" | "saveProjectIfUnchanged" >; @@ -139,7 +144,7 @@ const READ_ONLY = { readOnlyHint: true, destructiveHint: false, openWorldHint: f export const MEDIA_PREAMBLE = [ "OpenScreen records the screen, turns each take into a project you edit with the document tools, and exports it as a video.", - "A demo goes: listSources, then startRecording, then drive the app, then stopRecording. It returns the projectId and a checkpointId. Edit with the document tools, always passing that projectId: every edit is saved to the project straight away, and restoreCheckpoint with the checkpointId takes the project back to the raw take in one step. Then exportVideo, and getFrames on the export to check it before you hand it over.", + "A demo goes: listSources, then startRecording, then drive the app, then stopRecording. It returns the projectId and a checkpointId. Edit with the document tools, always passing that projectId: every edit is saved to the project straight away, and restoreCheckpoint with the checkpointId takes the project back to the raw take in one step. Then exportVideo, and getFrames with the projectId to check the export before you hand it over: it takes the same ruler seconds as the document tools, so a trim or a speed region never shifts what you look at.", "Drive the app with the real mouse when you can: OpenScreen records the pointer, draws a clean cursor and zooms on each click. If you drive it from a script instead (Playwright, CDP), the screen pointer never moves: log each action as the script runs it, then hand the log to setPointerTrack after stopRecording.", "These projects are in the OpenScreen app's project list, so the user can open one in the editor. Do not edit a project while the user has it open there: the editor would keep its own copy.", "", @@ -159,6 +164,8 @@ export function registerMediaTools(server: McpServer, deps: MediaToolDeps): void let exporting = false; /** When each take of this session started, by its video: setPointerTrack's clock. */ const startedAtByVideo = new Map(); + /** Each project's last export of this session, and when it was made: what getFrames reads. */ + const lastExports = new Map(); /** A runner's result, its hidden window closed: this server outlives every runner. */ async function finish(run: ReturnType) { @@ -450,7 +457,7 @@ export function registerMediaTools(server: McpServer, deps: MediaToolDeps): void "exportVideo", { description: - "Render a project to an MP4 (H.264, 60 fps) or a GIF with OpenScreen's compositor: the background, cursor, zooms, cuts, speed changes and annotations as edited. Returns the file's path; call getFrames on it to check the result.", + "Render a project to an MP4 (H.264, 60 fps) or a GIF with OpenScreen's compositor: the background, cursor, zooms, cuts, speed changes and annotations as edited. Returns the file's path; call getFrames with the projectId to check it at the ruler seconds you edited with. The video's shape follows the project's format (setAspectRatio).", inputSchema: z.object({ projectId: z.string(), path: z @@ -461,7 +468,9 @@ export function registerMediaTools(server: McpServer, deps: MediaToolDeps): void quality: z .enum(["medium", "good", "source"]) .optional() - .describe("medium is 720p, good is 1080p (default), source keeps the recording's size."), + .describe( + "The height: medium is 720 pixels, good is 1080 (default), source keeps the recording's. The width follows the format.", + ), }), annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: false }, }, @@ -507,6 +516,10 @@ export function registerMediaTools(server: McpServer, deps: MediaToolDeps): void if (!result.success) { return text(`The export failed: ${result.error ?? "unknown error"}`, true); } + lastExports.set(args.projectId, { + path: result.outputPath ?? outPath, + at: new Date().toISOString(), + }); return text( JSON.stringify({ path: result.outputPath, @@ -524,37 +537,79 @@ export function registerMediaTools(server: McpServer, deps: MediaToolDeps): void server.registerTool( "getFrames", { - description: `Look at a video: still frames of a take (videoPath from stopRecording) or of an export, as images. Ask for the moments you need to check, such as each zoom and each callout. At most ${MAX_FRAMES} frames per call.`, + description: [ + `Look at a video as still images, at most ${MAX_FRAMES} per call. Ask for the moments you need to check, such as each zoom and each callout.`, + "To check an edit, pass projectId: you get the project's last export, and atSec are the ruler seconds the document tools use, mapped through the trims and speed regions to the moment the export plays them. With path instead, you get any video file (a take's videoPath, say) and atSec are that file's own seconds.", + ].join("\n"), inputSchema: z.object({ - path: z.string().describe("The video file."), + projectId: z.string().optional().describe("Look at this project's last export."), + path: z.string().optional().describe("Or look at this video file."), atSec: z .array(z.number().nonnegative()) .max(MAX_FRAMES) .optional() - .describe("Times in seconds of that video. A time past the end gives the last frame."), + .describe("Times in seconds: ruler seconds with projectId, the file's own with path."), count: z .number() .int() .min(1) .max(MAX_FRAMES) .optional() - .describe("Frames spread evenly across the video, when atSec is not given. Default 6."), + .describe("Frames spread evenly, when atSec is not given. Default 6."), }), annotations: READ_ONLY, }, async (args) => { - const video = await deps.openVideo(path.resolve(args.path)); + if ((args.projectId === undefined) === (args.path === undefined)) { + return text("Pass either projectId or path.", true); + } + let videoPath: string; + /** Ruler seconds → seconds of the exported file, when looking at a project. */ + let toVideoSec = (sec: number) => sec; + let rulerEndSec: number | null = null; + let note = ""; + if (args.projectId !== undefined) { + const exported = lastExports.get(args.projectId); + if (!exported) { + return text("This project has no export yet: call exportVideo first.", true); + } + const document = await deps.documents.getProject(args.projectId); + videoPath = exported.path; + toVideoSec = (sec) => programmeSecOf(document, sec); + rulerEndSec = Math.max(0, ...document.timeline.clips.map((clip) => clip.timelineEndSec)); + if (document.project.updatedAt > exported.at) { + note = " The project changed after this export: export it again to see the changes."; + } + } else { + videoPath = path.resolve(args.path ?? ""); + } + + const video = await deps.openVideo(videoPath); try { - const times = frameTimes(video.durationSec, args.atSec ?? null, args.count ?? 6); + const asked = + args.atSec ?? + (rulerEndSec === null ? null : frameTimes(rulerEndSec, null, args.count ?? 6)); + const times = asked + ? asked.map((sec) => ({ + asked: sec, + at: frameTimes(video.durationSec, [toVideoSec(sec)], 1)[0], + })) + : frameTimes(video.durationSec, null, args.count ?? 6).map((at) => ({ asked: at, at })); const content: CallToolResult["content"] = [ { type: "text", - text: `${times.length} frame(s) of a ${video.width}x${video.height} video lasting ${video.durationSec.toFixed(2)}s.`, + text: `${times.length} frame(s) of a ${video.width}x${video.height} video lasting ${video.durationSec.toFixed(2)}s.${note}`, }, ]; - for (const timeSec of times) { - const frame = await video.capture(timeSec, FRAME_WIDTH); - content.push({ type: "text", text: `${timeSec.toFixed(2)}s` }); + for (const { asked: askedSec, at } of times) { + const frame = await video.capture(at, FRAME_WIDTH); + content.push({ + type: "text", + text: + rulerEndSec === null + ? `${at.toFixed(2)}s` + : `${askedSec.toFixed(2)}s on the ruler, ${at.toFixed(2)}s into the video`, + }); content.push({ type: "image", data: frame.data, mimeType: frame.mimeType }); } return { content }; @@ -564,3 +619,16 @@ export function registerMediaTools(server: McpServer, deps: MediaToolDeps): void }, ); } + +/** + * Where a ruler second plays in the export: the exporter's own projection, through the trims + * (holes in the ruler that the export skips) and the speed regions (spans it plays faster). + */ +function programmeSecOf(document: AxcutDocument, rulerSec: number): number { + return projectRawTimelineSecToPlayback( + document.timeline.clips, + document.timeline.trimRanges, + rulerSec, + readSpeedRegions(document), + ); +} diff --git a/skills/openscreen-demo/SKILL.md b/skills/openscreen-demo/SKILL.md index 48e142a610..40e2491ecc 100644 --- a/skills/openscreen-demo/SKILL.md +++ b/skills/openscreen-demo/SKILL.md @@ -40,7 +40,7 @@ Write a storyboard and show it to the user in a few lines, then carry on unless - Start the app and open it on the starting state of the first beat. - Give it one clean window of about 1600×1000. For a web app, open it in an app window without browser chrome: `chrome --app=http://localhost:3000 --window-size=1600,1000 --user-data-dir=`. Its title is the page's ``. - Call `listSources` and pick a unique part of the window's title. -- Close or hide anything personal. Recording one window keeps the rest of the desktop out. +- **Only the app on screen.** No browser toolbar, address bar, local file path, user name, bookmark bar, translation prompt or notification in the recorded window. Recording one window keeps the rest of the desktop out. ## 4. Record @@ -49,7 +49,7 @@ Write a storyboard and show it to the user in a few lines, then carry on unless 3. `stopRecording` with a `title`. Keep what it returns: `projectId`, `checkpointId`, `videoPath`, `recordingStartedAt`. **How you drive the app decides the cursor.** -- **Real mouse** (computer-use, or any tool that moves the actual pointer): OpenScreen records the pointer, draws a clean cursor and already zooms on each click. +- **Real mouse** (computer-use, or any tool that moves the actual pointer): OpenScreen records the pointer, draws a clean cursor and already zooms on each click. If your mouse tool cannot click in the app's window (some only read browsers, for one), drive it from a script instead. - **A script** (Playwright, CDP): the screen pointer never moves, so the take has no cursor and no zoom. In the script, log every action as it runs: `at` is `Date.now()`, `x` and `y` are the action's position as fractions of the window, `click` is true for a click. Hand the log to `setPointerTrack` after `stopRecording`; its description gives the formula from a page's coordinates. A script also gives the cleanest take: same timing on every run, no hesitation. ## 5. Check the take @@ -58,11 +58,14 @@ Write a storyboard and show it to the user in a few lines, then carry on unless ## 6. Edit -Use the document tools, always with the `projectId`. Read the project first with `getCurrentDocument`: its notes say which times are in the recording's seconds and which in the edited video's. +Use the document tools, always with the `projectId`. Read the project first with `getCurrentDocument`: its notes say which times are in the recording's seconds and which on the editor's ruler. +**One clock.** Place every effect in ruler seconds, and look at the result with `getFrames` and the `projectId`, which takes the same seconds. Never read times off the exported file: a cut or a speed-up shifts its clock away from the ruler. + +- **Format first:** `setAspectRatio` for where the video will be shown, `16:9` for X. Do it before placing text. - **Cut dead time:** the setup at the start, the stop at the end, any wait. Use `addTrims`, or `addSpeed` to fast-forward a wait instead of cutting it. -- **Zoom on the action:** the clicks already have zooms. Check them, then add or move zooms with `addZooms` and `setZoom` where a result appears without a click. Hold a zoom at least 1.5 s. -- **Callouts:** one `addAnnotation` per beat, short text placed clear of the action. +- **Zoom on the action:** clicks close together share one zoom, and the first 2.5 s stay wide. Check them, then add or move zooms with `addZooms` and `setZoom` where a result deserves its own. Hold a zoom at least 1.5 s. +- **Callouts:** one `addAnnotation` per beat, short, placed clear of the action. Text sits on the output frame and stays put when the picture zooms. Use its look for hierarchy: a large title with `animation: "typewriter"`, step callouts with `animation: "pop"`. `setAnnotation` moves or restyles one. - **Undo:** `restoreCheckpoint` with the `checkpointId` takes the project back to the raw take in one step. Keep the rest as it is: the background, padding, shadow and cursor style are chosen to look good. @@ -70,7 +73,7 @@ Keep the rest as it is: the background, padding, shadow and cursor style are cho ## 7. Export and review 1. `exportVideo` with the `projectId` and a path such as `demo/demo.mp4`. -2. `getFrames` on the export, at the moments of your beats. +2. `getFrames` with the `projectId`, at the ruler seconds of your beats. 3. Check every frame: - Is each callout readable and clear of the action? - Is each zoom on the thing that changes? From df4cd105f3ecd72613eb90f08a94017debe9911d Mon Sep 17 00:00:00 2001 From: EtienneLescot <etiennelescot@gmail.com> Date: Mon, 12 Oct 2026 01:21:26 +0200 Subject: [PATCH 3/4] fix(mcp): read an export through the document it rendered From review: getFrames mapped ruler seconds through the current project, so a cut made after an export shifted which frame of that unchanged video it showed. exportVideo now keeps the document it renders and the moment it started; getFrames maps through that document and flags any save after that moment, compared as instants. An unknown project is refused before the runner starts. --- electron/ai-edition/agent-tools.test.ts | 4 +++- electron/mcp/media-tools.test.ts | 18 +++++++++++------- electron/mcp/media-tools.ts | 24 +++++++++++++++--------- 3 files changed, 29 insertions(+), 17 deletions(-) diff --git a/electron/ai-edition/agent-tools.test.ts b/electron/ai-edition/agent-tools.test.ts index d0f17a2f63..aa1801f38e 100644 --- a/electron/ai-edition/agent-tools.test.ts +++ b/electron/ai-edition/agent-tools.test.ts @@ -784,7 +784,9 @@ describe("executeAgentTool", () => { ); expect(result.ok).toBe(true); expect(JSON.parse(result.resultJson)).toMatchObject({ aspectRatio: "9:16" }); - expect((result.document?.legacyEditor as { aspectRatio?: string }).aspectRatio).toBe("9:16"); + expect( + (result.document?.legacyEditor as { aspectRatio?: string } | undefined)?.aspectRatio, + ).toBe("9:16"); expect(centreOf(result.document?.annotations.at(-1))).toEqual({ x: expect.closeTo(50, 5), y: expect.closeTo(50, 5), diff --git a/electron/mcp/media-tools.test.ts b/electron/mcp/media-tools.test.ts index 0470c15a67..1ce39955f8 100644 --- a/electron/mcp/media-tools.test.ts +++ b/electron/mcp/media-tools.test.ts @@ -387,12 +387,13 @@ describe("exportVideo", () => { expect(runners.open).toBe(0); }); - it("refuses a path that is not a video, and reports a failed render", async () => { - expect((await call("exportVideo", { projectId: "proj_x", path: "out.mov" })).isError).toBe( - true, - ); + it("refuses a path that is not a video or an unknown project, and reports a failed render", async () => { + const { projectId } = await recordTake(); + expect((await call("exportVideo", { projectId, path: "out.mov" })).isError).toBe(true); + const unknown = await call("exportVideo", { projectId: "proj_x", path: "out.mp4" }); + expect(textOf(unknown)).toContain('No project has the id "proj_x"'); runners.results.export = { success: false, error: "compositor exploded" }; - const failed = await call("exportVideo", { projectId: "proj_x", path: "out.gif" }); + const failed = await call("exportVideo", { projectId, path: "out.gif" }); expect(failed.isError).toBe(true); expect(textOf(failed)).toContain("compositor exploded"); expect(runners.started.at(-1)).toMatchObject({ format: "gif" }); @@ -417,10 +418,13 @@ describe("getFrames on a project", () => { ]); expect(labels[0]).not.toContain("changed after this export"); - // An edit after the export is flagged, so a stale video is never taken for the edit. - await call("addAnnotation", { projectId, startSec: 5, endSec: 7, text: "New" }); + // A cut made after the export is flagged, and does not move which frame of THAT video + // a ruler second shows: the mapping is the exported document's. + await call("addTrim", { projectId, startSec: 4.5, endSec: 5.5 }); const stale = await call("getFrames", { projectId, atSec: [6] }); expect(textOf(stale)).toContain("changed after this export"); + const staleLabels = stale.content.flatMap((part) => (part.type === "text" ? [part.text] : [])); + expect(staleLabels[1]).toBe("6.00s on the ruler, 4.00s into the video"); }); it("takes a project or a file, not both", async () => { diff --git a/electron/mcp/media-tools.ts b/electron/mcp/media-tools.ts index 8472b81d61..53ef61306f 100644 --- a/electron/mcp/media-tools.ts +++ b/electron/mcp/media-tools.ts @@ -165,7 +165,7 @@ export function registerMediaTools(server: McpServer, deps: MediaToolDeps): void /** When each take of this session started, by its video: setPointerTrack's clock. */ const startedAtByVideo = new Map<string, number>(); /** Each project's last export of this session, and when it was made: what getFrames reads. */ - const lastExports = new Map<string, { path: string; at: string }>(); + const lastExports = new Map<string, { path: string; at: Date; rendered: AxcutDocument }>(); /** A runner's result, its hidden window closed: this server outlives every runner. */ async function finish(run: ReturnType<MediaToolDeps["runners"]["start"]>) { @@ -486,6 +486,12 @@ export function registerMediaTools(server: McpServer, deps: MediaToolDeps): void const progressToken = extra._meta?.progressToken; exporting = true; try { + // The project as the export reads it, and when: getFrames maps ruler seconds through + // THIS document's cuts and speeds, and calls anything saved after `at` a change the + // video does not have. + const at = new Date(); + const rendered = await deps.documents.getProject(args.projectId).catch(() => null); + if (!rendered) return text(`No project has the id "${args.projectId}".`, true); const run = deps.runners.start( { kind: "export", @@ -516,10 +522,7 @@ export function registerMediaTools(server: McpServer, deps: MediaToolDeps): void if (!result.success) { return text(`The export failed: ${result.error ?? "unknown error"}`, true); } - lastExports.set(args.projectId, { - path: result.outputPath ?? outPath, - at: new Date().toISOString(), - }); + lastExports.set(args.projectId, { path: result.outputPath ?? outPath, at, rendered }); return text( JSON.stringify({ path: result.outputPath, @@ -573,11 +576,14 @@ export function registerMediaTools(server: McpServer, deps: MediaToolDeps): void if (!exported) { return text("This project has no export yet: call exportVideo first.", true); } - const document = await deps.documents.getProject(args.projectId); + // Mapped through the document the export rendered, so a cut made since cannot shift + // which frame of this video is shown; the current one only says whether it is stale. + const { rendered } = exported; videoPath = exported.path; - toVideoSec = (sec) => programmeSecOf(document, sec); - rulerEndSec = Math.max(0, ...document.timeline.clips.map((clip) => clip.timelineEndSec)); - if (document.project.updatedAt > exported.at) { + toVideoSec = (sec) => programmeSecOf(rendered, sec); + rulerEndSec = Math.max(0, ...rendered.timeline.clips.map((clip) => clip.timelineEndSec)); + const current = await deps.documents.getProject(args.projectId); + if (Date.parse(current.project.updatedAt) > exported.at.getTime()) { note = " The project changed after this export: export it again to see the changes."; } } else { From 564f1ae0fff711963936ef1288c4ce8e3e70b929 Mon Sep 17 00:00:00 2001 From: EtienneLescot <etiennelescot@gmail.com> Date: Mon, 12 Oct 2026 01:26:57 +0200 Subject: [PATCH 4/4] fix(mcp): keep an export record only when it matches its video From review: the export runner reads the project file itself, so an edit saved during an export could render a version other than the one recorded for getFrames; such an export now gets no record and the result says to export again. A successful export also drops any other project's record that pointed at the same file. --- electron/mcp/media-tools.test.ts | 29 +++++++++++++++++++++++++++++ electron/mcp/media-tools.ts | 16 +++++++++++++++- 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/electron/mcp/media-tools.test.ts b/electron/mcp/media-tools.test.ts index 1ce39955f8..2943f1b05a 100644 --- a/electron/mcp/media-tools.test.ts +++ b/electron/mcp/media-tools.test.ts @@ -427,6 +427,35 @@ describe("getFrames on a project", () => { expect(staleLabels[1]).toBe("6.00s on the ruler, 4.00s into the video"); }); + it("keeps no record of an export the project changed during, or whose file was overwritten", async () => { + const first = await recordTake("First"); + const out = path.join(dir, "demo.mp4"); + runners.results.export = { success: true, outputPath: out }; + + runners.holdExport = true; + const exporting = call("exportVideo", { projectId: first.projectId, path: out }); + await new Promise((resolve) => setTimeout(resolve, 20)); + await call("addTrim", { projectId: first.projectId, startSec: 2, endSec: 4 }); + runners.releaseExport(); + expect(textOf(await exporting)).toContain("changed while it exported"); + expect(textOf(await call("getFrames", { projectId: first.projectId }))).toContain( + "no export yet", + ); + + runners.holdExport = false; + await call("exportVideo", { projectId: first.projectId, path: out }); + expect( + (await call("getFrames", { projectId: first.projectId, atSec: [1] })).isError, + ).toBeFalsy(); + // Another project's export to the same file replaces that video. + const second = await documents.createProject("Second"); + await documents.addAsset(second.project.id, { path: videoPath }); + await call("exportVideo", { projectId: second.project.id, path: out }); + expect(textOf(await call("getFrames", { projectId: first.projectId }))).toContain( + "no export yet", + ); + }); + it("takes a project or a file, not both", async () => { expect((await call("getFrames", {})).isError).toBe(true); expect((await call("getFrames", { projectId: "p", path: "x.mp4" })).isError).toBe(true); diff --git a/electron/mcp/media-tools.ts b/electron/mcp/media-tools.ts index 53ef61306f..f049019784 100644 --- a/electron/mcp/media-tools.ts +++ b/electron/mcp/media-tools.ts @@ -522,7 +522,21 @@ export function registerMediaTools(server: McpServer, deps: MediaToolDeps): void if (!result.success) { return text(`The export failed: ${result.error ?? "unknown error"}`, true); } - lastExports.set(args.projectId, { path: result.outputPath ?? outPath, at, rendered }); + const written = result.outputPath ?? outPath; + // The file now holds this video: no other project's record may still point at it. + for (const [id, record] of lastExports) { + if (id !== args.projectId && record.path === written) lastExports.delete(id); + } + // The runner read the project file itself. Unless nothing was saved since `rendered` + // was read, it may have rendered another version, so this video gets no record. + const after = await deps.documents.getProject(args.projectId).catch(() => null); + if (after?.project.updatedAt !== rendered.project.updatedAt) { + lastExports.delete(args.projectId); + return text( + `${JSON.stringify({ path: result.outputPath, format: result.format, width: result.width, height: result.height })}\nThe project changed while it exported, so this video may not match it: export again before checking it with getFrames.`, + ); + } + lastExports.set(args.projectId, { path: written, at, rendered }); return text( JSON.stringify({ path: result.outputPath,