diff --git a/docs/qa/ux7144-pr5-captions.md b/docs/qa/ux7144-pr5-captions.md new file mode 100644 index 000000000..345a2cf1f --- /dev/null +++ b/docs/qa/ux7144-pr5-captions.md @@ -0,0 +1,70 @@ +# Captions Library follow-up + +Tracking: quantfive/codepress#7144. This is a dependent draft; its prerequisite +branch includes PR38, PR39, PR40 and PR4's word-occurrence contract checkpoints. +The source release target remains `codepress-main`; no package is published here. + +## Behavior and limits + +Library → More → Captions opens the existing host caption editor alongside a +non-mutating generation preview. This edit / Selected clip explicitly maps +measured transcript words into remaining clip occurrences. Missing/incomplete +word timing, mixed source scopes, old hosts, oversized ranges and locked target +tracks fail with explanations. This UI cannot align a cached legacy transcript. +Standalone points to its existing Transcript, clip context menu Generate Captions, +and temporary Clip settings workflow; it does not add a new local generator. + +Apply submits one revision-bound caption-only batch. An unchecked, explicit +Replace all cues choice is required before replacing the Transcript captions +track, including its manual corrections. Other tracks are not touched. New +tracks go above video. Existing generated tracks below video require explicit +removal and recreation; the tool explains that removal loses manual corrections. +Text corrections in the caption editor never cut footage. + +Track defaults reach the real preview/render projection, cue overrides take +precedence (including zero opacity), and unchanged effective styles round-trip +back to their original provenance. Explicit overrides equal to a default remain +overrides after changing that default. A subsequent clip trim still derives and +applies as a trim-only command. + +## Reproducible fixture verification + +- `npm run dev -- --port 4195 --strictPort` +- `node_modules/.bin/playwright test --config playwright.caption-refresh.config.ts` +- `npm run test:run -- src/features/editor/host/caption-generation.test.ts src/features/editor/host/caption-render-contract.test.ts src/features/editor/codepress/caption-editor.test.tsx src/features/editor/host/caption-editor-context.test.tsx` + +The browser test generates a three-second test-pattern video with sine audio. +Its source transcript is a controlled test fixture, not provider output from real +footage. The first occurrence retains source 0–1s and 2–3s; the repeat retains +0–3s. Expected caption output is HELLO 0–1s, WORLD 1–2s, HELLO UM WORLD 2–5s. +The fixture adapter submits real surface command batches and renders the host's +native projection through `window.freecut.renderTimeline`. It is not an +authenticated CodePress backend test or a pinned consumer adoption test. + +Artifacts retained under `artifacts/qa/captions/`: + +- `caption-export.webm`: actual five-second rendered AV export. +- `export-hello.png`: output 0.5s / source 0.5s, HELLO. +- `export-world.png`: output 1.5s / source 2.5s, WORLD in yellow. +- `export-repeat.png`: output 3.5s / source 1.5s, HELLO UM WORLD in yellow. +- `library-1280.png`, `library-1440.png`: Library controls with independent chat. + +Browser scenarios cover discovery, preview without mutation, one apply, explicit +replacement without duplicate cues, selected occurrence + one undo, live style +preview, navigation retention, and unsupported/pending/missing/error recovery. +No unprompted human usability session, authenticated provider/backend flow, or +final CodePress pin/patch/static-asset adoption is claimed. Parent owns those +integration gates. + +## Backend augmentation contract evidence + +Read-only inspection of CodePress `apps/backend/apps/video_editor/api.py` +`apply_video_operation` and `services.py` `apply_command_batch` establishes that +normal editor submission accepts `VideoCommandBatchRequest`, validates the +submitted batch, binds the current revision, and hashes the submitted operation +for idempotency. No transcript preview receipt ID or signed exact-preview binding +is part of this apply request. Replacement augmentation is checked again by the +surface command validator before submission. This is source evidence, not a +backend execution claim; the parent handoff records the inspected checkpoint. + +Full exact-head gate results are posted separately on the draft PR. diff --git a/playwright.caption-refresh.config.ts b/playwright.caption-refresh.config.ts new file mode 100644 index 000000000..95343ba3c --- /dev/null +++ b/playwright.caption-refresh.config.ts @@ -0,0 +1,17 @@ +// fallow-ignore-file unused-file +// Invoked explicitly by the caption browser QA command. +import { defineConfig } from 'playwright/test' +export default defineConfig({ + testDir: './tests/browser', + testMatch: 'caption-refresh.spec.ts', + workers: 1, + reporter: 'line', + use: { + baseURL: 'http://127.0.0.1:4195', + channel: 'chrome', + headless: true, + viewport: { width: 1440, height: 900 }, + trace: 'retain-on-failure', + video: 'retain-on-failure', + }, +}) diff --git a/src/config/editor-workspaces.ts b/src/config/editor-workspaces.ts index fc579fa90..bad4bf5e4 100644 --- a/src/config/editor-workspaces.ts +++ b/src/config/editor-workspaces.ts @@ -16,6 +16,7 @@ export type EditorSidebarTab = | 'effects' | 'transitions' | 'lottie' + | 'captions' | 'transcript' | 'ai' export type EditorClipInspectorTab = 'video' | 'motion' | 'audio' | 'effects' @@ -81,6 +82,7 @@ const SIDEBAR_TABS: readonly EditorSidebarTab[] = [ 'effects', 'transitions', 'lottie', + 'captions', 'transcript', 'ai', ] diff --git a/src/features/docs/pages/11-text-captions-subtitles.ts b/src/features/docs/pages/11-text-captions-subtitles.ts index 1cff8e5e9..ec025bdb5 100644 --- a/src/features/docs/pages/11-text-captions-subtitles.ts +++ b/src/features/docs/pages/11-text-captions-subtitles.ts @@ -82,7 +82,10 @@ const page = { { kind: 'steps', items: [ - 'Generate a transcript for the clip from the Media library or the Transcript panel.', + 'Open **Library → More → Captions**. In a host-backed editor, choose **This edit** or **Selected clip**, then **Preview captions** using the current transcript.', + 'Review the cue text and output times, then **Apply captions**. Replacing an existing generated track requires **Replace all cues**, which also replaces manual corrections in that track. Other caption tracks are preserved.', + 'Caption text corrections change displayed text only. Transcript footage deletion remains a separate action. Hosts without occurrence-bound caption support explain why generation is unavailable.', + 'For local media, generate a transcript for the clip from the Media library or the Transcript panel.', 'Use **Generate Captions** (or let FreeCut enable transcript captions automatically) from the clip context menu.', 'Edit cue timing and text, and pick a style preset (Netflix, YouTube, Bold Yellow, Outlined, TikTok) in the **Subtitle** section.', 'Adjust caption color, size, vertical position, and an optional background.', diff --git a/src/features/editor/codepress/caption-editor-view.tsx b/src/features/editor/codepress/caption-editor-view.tsx index f7d4b6c73..e52d8d880 100644 --- a/src/features/editor/codepress/caption-editor-view.tsx +++ b/src/features/editor/codepress/caption-editor-view.tsx @@ -74,6 +74,7 @@ interface CaptionStyleControlsProps { } interface CaptionPreviewProps { + styleDraft: CaptionStyle activeCue?: FreeCutFrameCaptionCue activeTrack: FreeCutFrameTrack currentFrame: number @@ -111,7 +112,7 @@ function CaptionStyleControls({

Caption style

- Apply a default or cue-specific style through the command contract. + Preview a style, then apply it to this track or one cue.

{ + requestGeneration.current += 1 + setScope(event.target.value) + }} + disabled={busy} + > + + + + + {!available && ( +

+ {mode === 'local' + ? 'For local media, open Transcript to generate timing. Select a clip and use Generate Captions in its context menu; caption styling is available in Clip settings.' + : !port + ? 'This host does not provide a transcript. Caption tracks can still be edited below when supported.' + : 'This host needs support for captions bound to edited clip occurrences. Caption creation is unavailable until the host is updated.'} +

+ )} + {hasGeneratedTrack && ( + + )} +
+ + +
+

+ Uses the host’s current transcript asset. Existing captions follow supported footage cuts. + Preview again to replace generated captions from the current transcript. +

+ + {message && ( +

+ {message} +

+ )} + {preview && ( +
+

{cues.length} cues ready · footage unchanged

+ + +
+ )} + +

+ Visible caption tracks appear in the video preview and rendered video. Check captions across + each cut before exporting. +

+ + ) +} diff --git a/src/features/editor/host/caption-render-contract.test.ts b/src/features/editor/host/caption-render-contract.test.ts new file mode 100644 index 000000000..dc183a2b5 --- /dev/null +++ b/src/features/editor/host/caption-render-contract.test.ts @@ -0,0 +1,160 @@ +import { describe, expect, it } from 'vite-plus/test' +import { deriveSupportedHostEdit } from './controller' +import { createCodePressCommandAdapter, freeCutDocumentToControlledDocument } from '../codepress' +import type { EmbeddedEditorSnapshot } from './contract' +import { hostSnapshotToNativeTimeline, nativeTimelineToFrameDocument } from './document' + +const snapshot: EmbeddedEditorSnapshot = { + project: { id: 'project', name: 'Captions', width: 640, height: 360, fps: 30 }, + assets: [], + timeline: { + timelineId: 'timeline', + revision: 2, + fps: 30, + width: 640, + height: 360, + durationInFrames: 60, + media: [], + tracks: [ + { + id: 'captions', + name: 'Captions', + kind: 'caption', + locked: false, + muted: false, + defaultStyle: { color: '#ffff00', font_size: 48 }, + items: [ + { + id: 'cue', + type: 'caption_cue', + trackId: 'captions', + from: 30, + durationInFrames: 30, + text: 'WORLD', + }, + { + id: 'override', + type: 'caption_cue', + trackId: 'captions', + from: 0, + durationInFrames: 30, + text: 'HELLO', + style: { color: '#ff0000', background_opacity: 0 }, + }, + ], + }, + ], + }, +} +describe('caption render contract', () => { + it('projects track defaults and cue overrides into real preview/export text', () => { + const native = hostSnapshotToNativeTimeline(snapshot) + expect(native.items[0]).toMatchObject({ + type: 'text', + text: 'WORLD', + color: '#ffff00', + fontSize: 48, + from: 30, + durationInFrames: 30, + }) + expect(native.items[1]).toMatchObject({ color: '#ff0000', fontSize: 48 }) + expect(native.items[1]).toMatchObject({ backgroundOpacity: 0 }) + const restored = nativeTimelineToFrameDocument(native, snapshot.timeline) + expect(restored.ok).toBe(true) + if (restored.ok) + expect(restored.document.tracks[0]!.items).toEqual(snapshot.timeline.tracks[0]!.items) + }) + it.each(['#ffff00', '#ffffff'])( + 'keeps explicit %s override equal to the default when the default later changes', + (color) => { + const track = snapshot.timeline.tracks[0]! + const original = { + ...snapshot, + timeline: { + ...snapshot.timeline, + tracks: [ + { + ...track, + defaultStyle: { color, background_opacity: 0 }, + items: [{ ...track.items[0]!, style: { color, background_opacity: 0 } }], + }, + ], + }, + } + const native = hostSnapshotToNativeTimeline(original) + const restored = nativeTimelineToFrameDocument(native, original.timeline) + expect(restored.ok).toBe(true) + if (!restored.ok) return + const changed = hostSnapshotToNativeTimeline({ + ...original, + timeline: { + ...restored.document, + tracks: restored.document.tracks.map((candidate) => ({ + ...candidate, + defaultStyle: { color: '#ff0000', background_opacity: 1 }, + })), + }, + }) + expect(changed.items[0]).toMatchObject({ color, backgroundOpacity: 0 }) + }, + ) + it('derives and applies a subsequent clip trim without turning inherited styles into edits', () => { + const original: EmbeddedEditorSnapshot = { + ...snapshot, + timeline: { + ...snapshot.timeline, + media: [ + { + media_id: 'media', + media_kind: 'video', + content_hash: 'hash', + duration_us: 3_000_000, + availability: { mode: 'cloud', cloud: { object_id: 'media' } }, + }, + ], + tracks: [ + ...snapshot.timeline.tracks, + { + id: 'video', + kind: 'video', + name: 'Video', + locked: false, + muted: false, + items: [ + { + type: 'video', + id: 'clip', + trackId: 'video', + mediaId: 'media', + from: 0, + durationInFrames: 60, + sourceStart: 0, + sourceEnd: 60, + }, + ], + }, + ], + }, + } + const native = hostSnapshotToNativeTimeline(original) + const edited = nativeTimelineToFrameDocument( + { + ...native, + items: native.items.map((item) => + item.id === 'clip' ? { ...item, from: 5, durationInFrames: 55, sourceStart: 5 } : item, + ), + }, + original.timeline, + ) + expect(edited.ok).toBe(true) + if (!edited.ok) return + const derived = deriveSupportedHostEdit(original.timeline, edited.document) + expect(derived.batch?.commands.map((command) => command.type)).toEqual(['trim_item']) + const adapter = createCodePressCommandAdapter({ + document: freeCutDocumentToControlledDocument(original.timeline), + }) + const captionsBeforeTrim = adapter.getDocument().timeline.tracks[0] + expect(adapter.apply(derived.batch).status).toBe('applied') + expect(adapter.getDocument().timeline.tracks[0]).toEqual(captionsBeforeTrim) + }) +}) diff --git a/src/features/editor/host/document.ts b/src/features/editor/host/document.ts index 6faea0b97..0f4f96a1b 100644 --- a/src/features/editor/host/document.ts +++ b/src/features/editor/host/document.ts @@ -251,7 +251,14 @@ export function hostSnapshotToNativeTimeline(snapshot: EmbeddedEditorSnapshot): const assets = assetById(snapshot.assets) const tracks = snapshot.timeline.tracks.map(nativeTrackFromHostTrack) const items = snapshot.timeline.tracks.flatMap((track) => - track.items.map((item) => nativeItemFromHostItem(item, assets)), + track.items.map((item) => + nativeItemFromHostItem( + item.type === 'caption_cue' + ? { ...item, style: { ...track.defaultStyle, ...item.style } } + : item, + assets, + ), + ), ) return { tracks, items, fps: snapshot.project.fps } } @@ -381,6 +388,26 @@ function authoritativeTrackMetadata( * document. Shapes, compositions, subtitles, Lottie, effects, and animation * edits fail closed so a host action can be visibly rejected and restored. */ +function restoreCaptionStyleProvenance( + converted: Extract, + authoritative: FreeCutFrameDocument, +) { + const track = authoritative.tracks.find((candidate) => candidate.id === converted.trackId) + const original = track?.items.find((candidate) => candidate.id === converted.id) + if (original?.type !== 'caption_cue') return + const effective = frameItemToNativeComparable( + nativeItemFromHostItem( + { ...original, style: { ...track?.defaultStyle, ...original.style } }, + new Map(), + ), + ) + if ('reason' in effective || effective.type !== 'caption_cue') return + if (JSON.stringify(converted.style) !== JSON.stringify(effective.style)) return + // Restore original provenance, including explicit overrides equal to defaults. + if (original.style === undefined) delete converted.style + else converted.style = original.style +} + export function nativeTimelineToFrameDocument( state: Pick, authoritative: FreeCutFrameDocument, @@ -399,6 +426,7 @@ export function nativeTimelineToFrameDocument( } const converted = frameItemToNativeComparable(item) if ('reason' in converted) return { ok: false, failure: converted } + if (converted.type === 'caption_cue') restoreCaptionStyleProvenance(converted, authoritative) const list = itemsByTrack.get(item.trackId) ?? [] list.push(converted) itemsByTrack.set(item.trackId, list) diff --git a/src/features/editor/host/runtime.ts b/src/features/editor/host/runtime.ts index b153d96f1..30d7c7bb7 100644 --- a/src/features/editor/host/runtime.ts +++ b/src/features/editor/host/runtime.ts @@ -312,7 +312,7 @@ export class EmbeddedEditorHostRuntime implements EmbeddedEditorHostRuntimeContr // applied/conflict state before the user can see it. const currentTab = useEditorStore.getState().activeTab const currentTabVisibleInHostMode = - currentTab === 'media' || + ['media', 'captions'].includes(currentTab) || (currentTab === 'text' && isHostCapabilityEnabled(this.host.capabilities, 'timeline.add')) || (currentTab === 'transcript' && diff --git a/tests/browser/caption-refresh.html b/tests/browser/caption-refresh.html new file mode 100644 index 000000000..8c7a15bd8 --- /dev/null +++ b/tests/browser/caption-refresh.html @@ -0,0 +1 @@ +
diff --git a/tests/browser/caption-refresh.spec.ts b/tests/browser/caption-refresh.spec.ts new file mode 100644 index 000000000..54eee748d --- /dev/null +++ b/tests/browser/caption-refresh.spec.ts @@ -0,0 +1,218 @@ +import { execFileSync } from 'node:child_process' +import { unlinkSync, mkdirSync } from 'node:fs' +import { expect, test } from 'playwright/test' +const media = 'tests/browser/.caption-generated.webm' +test.beforeAll(() => { + mkdirSync('artifacts/qa/captions', { recursive: true }) + execFileSync( + 'ffmpeg', + [ + '-y', + '-f', + 'lavfi', + '-i', + 'testsrc2=size=640x360:rate=30:duration=3', + '-f', + 'lavfi', + '-i', + 'sine=frequency=440:duration=3', + '-c:v', + 'libvpx-vp9', + '-c:a', + 'libopus', + media, + ], + { stdio: 'ignore' }, + ) +}) +test.afterAll(() => unlinkSync(media)) +test.setTimeout(120_000) +test('More discovers captions; edited occurrence timing, style and one apply survive Library navigation', async ({ + page, +}) => { + await page.goto('/tests/browser/caption-refresh.html') + await page.locator('[data-item-id]').first().waitFor() + await page.getByRole('combobox', { name: 'More library tools' }).selectOption('captions') + await expect(page.getByRole('heading', { name: 'Create captions', exact: true })).toBeVisible() + await page.getByRole('textbox', { name: 'Chat draft' }).fill('Keep this draft') + await page.getByRole('button', { name: 'Preview captions', exact: true }).click() + await expect(page.getByRole('button', { name: 'Apply captions', exact: true })).toBeVisible() + expect(await page.evaluate(() => (window as any).captionFixture.applies)).toBe(0) + await expect(page.getByText('1.00–2.00s')).toBeVisible() + await page.getByRole('button', { name: 'Apply captions', exact: true }).click() + await expect(page.getByTestId('caption-editor')).toBeVisible() + expect(await page.evaluate(() => (window as any).captionFixture.applies)).toBe(1) + const cues = await page.evaluate( + () => + (window as any).captionFixture + .snapshot() + .timeline.tracks.find((track: any) => track.kind === 'caption').items, + ) + expect(cues.map((cue: any) => [cue.text, cue.from, cue.durationInFrames])).toEqual([ + ['HELLO', 0, 30], + ['WORLD', 30, 30], + ['HELLO UM WORLD', 60, 90], + ]) + await page.getByRole('button', { name: /Seek to cue 2 at frame 30/ }).click() + await page.getByLabel('Color', { exact: true }).fill('#ffff00') + await expect(page.getByTestId('caption-preview').getByText('WORLD', { exact: true })).toHaveCSS( + 'color', + 'rgb(255, 255, 0)', + ) + await page.getByRole('button', { name: 'Apply style', exact: true }).click() + await page.getByRole('button', { name: 'Media', exact: true }).click() + await page.getByRole('combobox', { name: 'More library tools' }).selectOption('captions') + await expect(page.getByLabel('Color', { exact: true })).toHaveValue('#ffff00') + await expect(page.getByRole('textbox', { name: 'Chat draft' })).toHaveValue('Keep this draft') + const native = await page.evaluate(() => (window as any).captionFixture.native()) + expect(native.items.find((item: any) => item.text === 'WORLD').color).toBe('#ffff00') + const renderPage = await page.context().newPage() + await renderPage.goto('/headless.html') + await renderPage.waitForFunction(() => Boolean((window as any).freecut?.ready)) + const downloadPromise = renderPage.waitForEvent('download', { timeout: 120_000 }) + const summary = await renderPage.evaluate( + (native) => + (window as any).freecut.renderTimeline({ + ...native, + width: 640, + height: 360, + media: [ + { + mediaId: 'fixture', + url: 'http://127.0.0.1:4195/tests/browser/.caption-generated.webm', + }, + ], + settings: { + mode: 'video', + codec: 'vp9', + audioCodec: 'opus', + container: 'webm', + quality: 'high', + resolution: { width: 640, height: 360 }, + fps: 30, + audioBitrate: 128000, + }, + outputFileName: 'caption-export.webm', + }), + native, + ) + await (await downloadPromise).saveAs('artifacts/qa/captions/caption-export.webm') + expect(summary).toBeTruthy() + for (const [name, time] of [ + ['hello', '0.5'], + ['world', '1.5'], + ['repeat', '3.5'], + ]) + execFileSync( + 'ffmpeg', + [ + '-y', + '-ss', + time!, + '-i', + 'artifacts/qa/captions/caption-export.webm', + '-frames:v', + '1', + `artifacts/qa/captions/export-${name}.png`, + ], + { stdio: 'ignore' }, + ) + await renderPage.close() + for (const width of [1440, 1280]) { + await page.setViewportSize({ width, height: 900 }) + await page.screenshot({ path: `artifacts/qa/captions/library-${width}.png` }) + } + await page.getByRole('checkbox', { name: /Replace all cues/ }).check() + await page.getByRole('button', { name: 'Preview captions', exact: true }).click() + await page.getByRole('button', { name: 'Apply captions', exact: true }).click() + await expect(page.getByTestId('caption-editor')).toBeVisible() + expect( + await page.evaluate( + () => + (window as any).captionFixture + .snapshot() + .timeline.tracks.find((track: any) => track.kind === 'caption').items.length, + ), + ).toBe(3) +}) + +for (const [scenario, message] of [ + ['unsupported', 'This host needs support for captions bound to edited clip occurrences.'], + ['pending', 'Transcription is still processing.'], + ['missing', 'A completed transcript is needed.'], + ['error', 'Fixture preview unavailable.'], +]) { + test(`caption recovery: ${scenario}`, async ({ page }) => { + await page.goto(`/tests/browser/caption-refresh.html?scenario=${scenario}`) + await page.locator('[data-item-id]').first().waitFor() + await page.getByRole('combobox', { name: 'More library tools' }).selectOption('captions') + const preview = page.getByRole('button', { name: 'Preview captions', exact: true }) + if (scenario === 'unsupported') await expect(preview).toBeDisabled() + else await preview.click() + await expect(page.getByText(message, { exact: false })).toBeVisible() + expect(await page.evaluate(() => (window as any).captionFixture.applies)).toBe(0) + }) +} +test('selected occurrence captions undo in one transaction', async ({ page }) => { + await page.goto('/tests/browser/caption-refresh.html') + await page.locator('[data-item-id="after-cut"][role="button"]').click() + await page.getByRole('combobox', { name: 'More library tools' }).selectOption('captions') + await page.getByLabel('Caption scope').selectOption('selected') + await page.getByRole('button', { name: 'Preview captions', exact: true }).click() + await page.getByRole('button', { name: 'Apply captions', exact: true }).click() + await expect(page.getByTestId('caption-editor')).toBeVisible() + const cues = await page.evaluate( + () => + (window as any).captionFixture + .snapshot() + .timeline.tracks.find((track: any) => track.kind === 'caption').items, + ) + expect(cues.map((cue: any) => [cue.text, cue.from, cue.durationInFrames])).toEqual([ + ['WORLD', 30, 30], + ]) + await page.evaluate(() => (window as any).captionFixture.undo()) + await expect(page.getByTestId('caption-editor-empty')).toBeVisible() + expect( + await page.evaluate( + () => + (window as any).captionFixture + .snapshot() + .timeline.tracks.flatMap((track: any) => track.items).length, + ), + ).toBe(3) +}) + +for (const returnToFirst of [false, true]) { + test(`pending selected caption preview rejects A→B${returnToFirst ? '→A' : ''}`, async ({ + page, + }) => { + await page.goto('/tests/browser/caption-refresh.html?scenario=deferred') + const first = page.locator('[data-item-id="first"][role="button"]') + const second = page.locator('[data-item-id="after-cut"][role="button"]') + await first.click() + await page.getByRole('combobox', { name: 'More library tools' }).selectOption('captions') + await page.getByLabel('Caption scope').selectOption('selected') + await page.getByRole('button', { name: 'Preview captions', exact: true }).click() + await expect + .poll(() => page.evaluate(() => (window as any).captionFixture.requests.length)) + .toBe(1) + await second.click() + if (returnToFirst) await first.click() + await page.evaluate(() => (window as any).captionFixture.resolvePreview()) + await expect(page.getByRole('alert')).toContainText('selection changed') + await expect(page.getByRole('button', { name: 'Apply captions', exact: true })).toHaveCount(0) + expect(await page.evaluate(() => (window as any).captionFixture.applies)).toBe(0) + await page.getByRole('button', { name: 'Preview captions', exact: true }).click() + await expect + .poll(() => page.evaluate(() => (window as any).captionFixture.requests.length)) + .toBe(2) + expect( + await page.evaluate(() => + (window as any).captionFixture.requests[1].ranges.map((range: any) => range.itemId), + ), + ).toEqual([returnToFirst ? 'first' : 'after-cut']) + await page.evaluate(() => (window as any).captionFixture.resolvePreview()) + await page.getByRole('button', { name: 'Apply captions', exact: true }).click() + expect(await page.evaluate(() => (window as any).captionFixture.applies)).toBe(1) + }) +} diff --git a/tests/browser/caption-refresh.tsx b/tests/browser/caption-refresh.tsx new file mode 100644 index 000000000..6d24beabb --- /dev/null +++ b/tests/browser/caption-refresh.tsx @@ -0,0 +1,248 @@ +// fallow-ignore-file unused-file +import { createRoot } from 'react-dom/client' +import { FreeCutEditorSurface } from '../../src/features/editor/host/editor-surface' +import { + createCodePressCommandAdapter, + controlledDocumentToFreeCutDocument, + freeCutDocumentToControlledDocument, +} from '../../src/features/editor/codepress' +import type { EditorHost, EmbeddedEditorSnapshot } from '../../src/features/editor/host/contract' +import { hostSnapshotToNativeTimeline } from '../../src/features/editor/host/document' + +const scenario = new URLSearchParams(location.search).get('scenario') +const clip = { + id: 'first', + type: 'video' as const, + trackId: 'video', + mediaId: 'fixture', + from: 0, + durationInFrames: 30, + sourceStart: 0, + sourceEnd: 30, +} +let snapshot: EmbeddedEditorSnapshot = { + project: { + id: 'caption-fixture', + name: 'Caption fixture · edited sequence', + width: 640, + height: 360, + fps: 30, + }, + timeline: { + timelineId: 'caption-timeline', + revision: 0, + fps: 30, + width: 640, + height: 360, + durationInFrames: 150, + media: [ + { + media_id: 'fixture', + media_kind: 'video', + content_hash: 'fixture-hash', + duration_us: 3_000_000, + availability: { mode: 'cloud', cloud: { object_id: 'fixture' } }, + }, + ], + tracks: [ + { + id: 'video', + kind: 'video', + name: 'V1', + locked: false, + muted: false, + items: [ + clip, + { ...clip, id: 'after-cut', from: 30, sourceStart: 60, sourceEnd: 90 }, + { ...clip, id: 'repeat', from: 60, durationInFrames: 90, sourceEnd: 90 }, + ], + }, + ], + }, + assets: [ + { + id: 'fixture', + kind: 'video', + fileName: 'generated-av.webm', + mimeType: 'video/webm', + durationSeconds: 3, + width: 640, + height: 360, + fps: 30, + contentHash: 'fixture-hash', + }, + ], +} +let adapter = createCodePressCommandAdapter({ + document: freeCutDocumentToControlledDocument(snapshot.timeline), +}) +const listeners = new Set<(value: EmbeddedEditorSnapshot) => void>() +const history: EmbeddedEditorSnapshot[] = [] +const fixtureState = { + applies: 0, + resolvePreview: () => {}, + undo: () => host.history!.undo(), + requests: [] as unknown[], + native: () => hostSnapshotToNativeTimeline(snapshot), + snapshot: () => snapshot, +} +Object.assign(window, { captionFixture: fixtureState }) +const host: EditorHost = { + capabilities: { 'media.resolve': true, 'media.transcription': true, 'timeline.caption': true }, + load: () => snapshot, + resolveMedia: () => ({ source: '/tests/browser/.caption-generated.webm' }), + subscribe: (listener) => { + listeners.add(listener) + return () => { + listeners.delete(listener) + } + }, + history: { + undo: () => { + const previous = history.pop() + if (previous) { + snapshot = { + ...previous, + timeline: { ...previous.timeline, revision: snapshot.timeline.revision + 1 }, + } + adapter = createCodePressCommandAdapter({ + document: freeCutDocumentToControlledDocument(snapshot.timeline), + }) + listeners.forEach((listener) => listener(snapshot)) + } + }, + redo: () => {}, + }, + submitEdit: (batch) => { + const previous = snapshot + const result = adapter.apply(batch) + if (result.status !== 'rejected') { + history.push(previous) + fixtureState.applies++ + snapshot = { + ...snapshot, + timeline: controlledDocumentToFreeCutDocument(adapter.getDocument()), + } + listeners.forEach((listener) => listener(snapshot)) + } + return { status: result.status, result, snapshot } + }, + transcript: { + occurrenceSelection: scenario !== 'unsupported', + getStatus: () => ({ + transcriptId: 'transcript', + assetId: 'fixture', + sourceAssetHash: 'fixture-hash', + status: scenario === 'pending' ? 'running' : scenario === 'missing' ? 'failed' : 'succeeded', + durationUs: 3_000_000, + sectionCount: 1, + }), + getSections: () => ({ + transcriptId: 'transcript', + hasMore: false, + sections: [ + { + id: 'section', + transcriptId: 'transcript', + ordinal: 0, + startUs: 0, + endUs: 3_000_000, + text: 'HELLO UM WORLD', + timingSource: 'provider', + words: [ + { text: 'HELLO', startUs: 0, endUs: 1_000_000 }, + { text: 'UM', startUs: 1_000_000, endUs: 2_000_000 }, + { text: 'WORLD', startUs: 2_000_000, endUs: 3_000_000 }, + ], + }, + ], + }), + previewCommands: async (request) => { + if (scenario === 'error') throw new Error('Fixture preview unavailable. Retry preview.') + fixtureState.requests.push(request) + if (scenario === 'deferred') + await new Promise((resolve) => { + fixtureState.resolvePreview = resolve + }) + const trackId = request.captionTrackId! + const existing = snapshot.timeline.tracks.find((track) => track.id === trackId) + const cues = request.ranges!.map((range, index) => { + const item = snapshot.timeline.tracks + .flatMap((track) => track.items) + .find((item) => item.id === range.itemId)! + if (item.type !== 'video') throw new Error('Unsupported fixture') + return { + item_type: 'caption_cue' as const, + cue_id: `cue-${index}`, + track_id: trackId, + start_us: Math.round( + (item.from / 30) * 1_000_000 + range.startUs - (item.sourceStart! / 30) * 1_000_000, + ), + end_us: Math.round( + (item.from / 30) * 1_000_000 + range.endUs - (item.sourceStart! / 30) * 1_000_000, + ), + text: range.text!, + } + }) + const commandBatch = { + contract_version: 1 as const, + timeline_id: snapshot.timeline.timelineId, + operation_id: request.operationId, + idempotency_key: request.idempotencyKey, + base_revision: request.baseRevision, + preconditions: [], + commands: [ + ...(existing + ? [] + : [ + { + type: 'add_caption_track' as const, + command_id: 'track', + track_id: trackId, + name: 'Captions', + language: 'en', + index: 1, + }, + ]), + { type: 'upsert_caption_cues' as const, command_id: 'cues', track_id: trackId, cues }, + ], + } + return { + status: 'preview', + receiptId: 'receipt', + transcriptId: request.transcriptId, + assetId: request.assetId, + sourceAssetHash: request.sourceAssetHash, + timestampCapability: 'section', + timelineId: snapshot.timeline.timelineId, + operationId: request.operationId, + idempotencyKey: request.idempotencyKey, + baseRevision: request.baseRevision, + commandBatch, + preview: { action: 'captions', captionCount: cues.length, willMutateTimeline: false }, + } + }, + }, +} +createRoot(document.getElementById('root')!).render( +
+