feat(captions): discover, preview and style edited-sequence captions

docs/qa/ux7144-pr5-captions.md

@@ -0,0 +1,70 @@
+# Captions Library follow-up
+
+Tracking: quantfive/codepress#7144. This is a dependent draft; its prerequisite
+branch includes PR38, PR39, PR40 and PR4's word-occurrence contract checkpoints.
+The source release target remains `codepress-main`; no package is published here.
+
+## Behavior and limits
+
+Library → More → Captions opens the existing host caption editor alongside a
+non-mutating generation preview. This edit / Selected clip explicitly maps
+measured transcript words into remaining clip occurrences. Missing/incomplete
+word timing, mixed source scopes, old hosts, oversized ranges and locked target
+tracks fail with explanations. This UI cannot align a cached legacy transcript.
+Standalone points to its existing Transcript, clip context menu Generate Captions,
+and temporary Clip settings workflow; it does not add a new local generator.
+
+Apply submits one revision-bound caption-only batch. An unchecked, explicit
+Replace all cues choice is required before replacing the Transcript captions
+track, including its manual corrections. Other tracks are not touched. New
+tracks go above video. Existing generated tracks below video require explicit
+removal and recreation; the tool explains that removal loses manual corrections.
+Text corrections in the caption editor never cut footage.
+
+Track defaults reach the real preview/render projection, cue overrides take
+precedence (including zero opacity), and unchanged effective styles round-trip
+back to their original provenance. Explicit overrides equal to a default remain
+overrides after changing that default. A subsequent clip trim still derives and
+applies as a trim-only command.
+
+## Reproducible fixture verification
+
+- `npm run dev -- --port 4195 --strictPort`
+- `node_modules/.bin/playwright test --config playwright.caption-refresh.config.ts`
+- `npm run test:run -- src/features/editor/host/caption-generation.test.ts src/features/editor/host/caption-render-contract.test.ts src/features/editor/codepress/caption-editor.test.tsx src/features/editor/host/caption-editor-context.test.tsx`
+
+The browser test generates a three-second test-pattern video with sine audio.
+Its source transcript is a controlled test fixture, not provider output from real
+footage. The first occurrence retains source 0–1s and 2–3s; the repeat retains
+0–3s. Expected caption output is HELLO 0–1s, WORLD 1–2s, HELLO UM WORLD 2–5s.
+The fixture adapter submits real surface command batches and renders the host's
+native projection through `window.freecut.renderTimeline`. It is not an
+authenticated CodePress backend test or a pinned consumer adoption test.
+
+Artifacts retained under `artifacts/qa/captions/`:
+
+- `caption-export.webm`: actual five-second rendered AV export.
+- `export-hello.png`: output 0.5s / source 0.5s, HELLO.
+- `export-world.png`: output 1.5s / source 2.5s, WORLD in yellow.
+- `export-repeat.png`: output 3.5s / source 1.5s, HELLO UM WORLD in yellow.
+- `library-1280.png`, `library-1440.png`: Library controls with independent chat.
+
+Browser scenarios cover discovery, preview without mutation, one apply, explicit
+replacement without duplicate cues, selected occurrence + one undo, live style
+preview, navigation retention, and unsupported/pending/missing/error recovery.
+No unprompted human usability session, authenticated provider/backend flow, or
+final CodePress pin/patch/static-asset adoption is claimed. Parent owns those
+integration gates.
+
+## Backend augmentation contract evidence
+
+Read-only inspection of CodePress `apps/backend/apps/video_editor/api.py`
+`apply_video_operation` and `services.py` `apply_command_batch` establishes that
+normal editor submission accepts `VideoCommandBatchRequest`, validates the
+submitted batch, binds the current revision, and hashes the submitted operation
+for idempotency. No transcript preview receipt ID or signed exact-preview binding
+is part of this apply request. Replacement augmentation is checked again by the
+surface command validator before submission. This is source evidence, not a
+backend execution claim; the parent handoff records the inspected checkpoint.
+
+Full exact-head gate results are posted separately on the draft PR.

playwright.caption-refresh.config.ts

@@ -0,0 +1,17 @@
+// fallow-ignore-file unused-file
+// Invoked explicitly by the caption browser QA command.
+import { defineConfig } from 'playwright/test'
+export default defineConfig({
+  testDir: './tests/browser',
+  testMatch: 'caption-refresh.spec.ts',
+  workers: 1,
+  reporter: 'line',
+  use: {
+    baseURL: 'http://127.0.0.1:4195',
+    channel: 'chrome',
+    headless: true,
+    viewport: { width: 1440, height: 900 },
+    trace: 'retain-on-failure',
+    video: 'retain-on-failure',
+  },
+})

src/config/editor-workspaces.ts

@@ -16,6 +16,7 @@ export type EditorSidebarTab =
   | 'effects'
   | 'transitions'
   | 'lottie'
+  | 'captions'
   | 'transcript'
   | 'ai'
 export type EditorClipInspectorTab = 'video' | 'motion' | 'audio' | 'effects'
@@ -81,6 +82,7 @@ const SIDEBAR_TABS: readonly EditorSidebarTab[] = [
   'effects',
   'transitions',
   'lottie',
+  'captions',
   'transcript',
   'ai',
 ]

src/features/docs/pages/11-text-captions-subtitles.ts

@@ -82,7 +82,10 @@ const page = {
         {
           kind: 'steps',
           items: [
-            'Generate a transcript for the clip from the Media library or the Transcript panel.',
+            'Open **Library → More → Captions**. In a host-backed editor, choose **This edit** or **Selected clip**, then **Preview captions** using the current transcript.',
+            'Review the cue text and output times, then **Apply captions**. Replacing an existing generated track requires **Replace all cues**, which also replaces manual corrections in that track. Other caption tracks are preserved.',
+            'Caption text corrections change displayed text only. Transcript footage deletion remains a separate action. Hosts without occurrence-bound caption support explain why generation is unavailable.',
+            'For local media, generate a transcript for the clip from the Media library or the Transcript panel.',
             'Use **Generate Captions** (or let FreeCut enable transcript captions automatically) from the clip context menu.',
             'Edit cue timing and text, and pick a style preset (Netflix, YouTube, Bold Yellow, Outlined, TikTok) in the **Subtitle** section.',
             'Adjust caption color, size, vertical position, and an optional background.',

src/features/editor/codepress/caption-editor-view.tsx

@@ -74,6 +74,7 @@ interface CaptionStyleControlsProps {
 }
 
 interface CaptionPreviewProps {
+  styleDraft: CaptionStyle
   activeCue?: FreeCutFrameCaptionCue
   activeTrack: FreeCutFrameTrack
   currentFrame: number
@@ -111,7 +112,7 @@ function CaptionStyleControls({
         <div>
           <h3 className="text-xs font-semibold">Caption style</h3>
           <p className="text-[11px] text-muted-foreground">
-            Apply a default or cue-specific style through the command contract.
+            Preview a style, then apply it to this track or one cue.
           </p>
         </div>
         <select
@@ -128,7 +129,7 @@ function CaptionStyleControls({
           ))}
         </select>
       </div>
-      <div className="mt-3 grid gap-2 sm:grid-cols-2 lg:grid-cols-4">
+      <div className="mt-3 grid grid-cols-2 gap-2">
         <div>
           <Label htmlFor="caption-font-family" className="text-[11px]">
             Font
@@ -202,16 +203,22 @@ function CaptionStyleControls({
   )
 }
 
-function CaptionPreview({ activeCue, activeTrack, currentFrame, fps }: CaptionPreviewProps) {
-  const previewStyle = captionStyleOrDefault(activeCue?.style ?? activeTrack.defaultStyle)
+function CaptionPreview({
+  activeCue,
+  activeTrack,
+  currentFrame,
+  fps,
+  styleDraft,
+}: CaptionPreviewProps) {
+  const previewStyle = captionStyleOrDefault(styleDraft)
 
   return (
     <div
       className="rounded-md border border-border/70 bg-background p-3"
       data-testid="caption-preview"
     >
       <p className="text-[11px] font-medium uppercase tracking-wide text-muted-foreground">
-        Preview · {formatFrame(currentFrame, fps)}
+        Style preview · {formatFrame(currentFrame, fps)}
       </p>
       <div className="mt-2 flex min-h-14 items-center justify-center rounded bg-black px-4 py-3">
         {activeTrack.muted ? (
@@ -607,6 +614,7 @@ function CaptionTrackEditor({
       />
 
       <CaptionPreview
+        styleDraft={styleDraft}
         activeCue={activeCue}
         activeTrack={activeTrack}
         currentFrame={currentFrame}
@@ -724,7 +732,7 @@ export function CaptionEditorView({
             <h2 className="text-sm font-semibold">Captions</h2>
           </div>
           <p className="mt-1 text-xs text-muted-foreground">
-            Frame-aligned cues are saved through the controlled command contract.
+            Edit the text displayed in your video. Text corrections never remove footage.
           </p>
         </div>
         <Button

src/features/editor/codepress/caption-editor.tsx

@@ -223,11 +223,11 @@ export function CaptionEditor({
         track_id: trackId,
         name: `Captions ${tracks.length + 1}`,
         language: 'en',
-        index: document?.tracks.length ?? 0,
+        index: 0,
       },
     ])
     if (applied) setActiveTrackId(trackId)
-  }, [document?.tracks.length, runCommands, tracks])
+  }, [runCommands, tracks])
 
   const removeTrack = useCallback(async () => {
     if (!activeTrack) return

src/features/editor/components/media-sidebar.tsx

@@ -62,6 +62,7 @@ import { createLogger } from '@/shared/logging/logger'
 import { useSettingsStore } from '@/features/editor/deps/settings'
 import { resolveGeneratedLayerCanvasSize } from '../utils/generated-layer-canvas-size'
 import { useEditorCapability, useEditorHostContext, useEditorHostMode } from '../host/context'
+import { HostCaptionLibrary } from '../host/caption-library'
 import { HostTranscriptEditor } from '../host/transcript-editor'
 const LazyAiPanel = lazy(() => import('./ai-tab').then((m) => ({ default: m.AiTab })))
 const LazyTranscriptEditorPanel = lazy(() =>
@@ -547,6 +548,10 @@ export const MediaSidebar = memo(function MediaSidebar({
     toggleLeftSidebar,
   })
 
+  const [captionsActivated, setCaptionsActivated] = useState(activeTab === 'captions')
+  useEffect(() => {
+    if (activeTab === 'captions') setCaptionsActivated(true)
+  }, [activeTab])
   const [transcriptActivated, setTranscriptActivated] = useState(activeTab === 'transcript')
   useEffect(() => {
     if (activeTab === 'transcript') setTranscriptActivated(true)
@@ -795,6 +800,7 @@ export const MediaSidebar = memo(function MediaSidebar({
   // Category items for the vertical nav
   const categories = [
     { id: 'media' as const, icon: Film, label: t('editor.mediaSidebar.media') },
+    { id: 'captions' as const, icon: Captions, label: 'Captions' },
     { id: 'text' as const, icon: Type, label: t('editor.mediaSidebar.text') },
     { id: 'shapes' as const, icon: Pentagon, label: t('editor.mediaSidebar.shapes') },
     { id: 'effects' as const, icon: Layers, label: t('editor.mediaSidebar.effects') },
@@ -807,6 +813,7 @@ export const MediaSidebar = memo(function MediaSidebar({
     ? categories.filter(
         ({ id }) =>
           id === 'media' ||
+          id === 'captions' ||
           (id === 'text' && canAddTimeline) ||
           (id === 'transcript' && canTranscribe && !!host?.transcript),
       )
@@ -1408,6 +1415,12 @@ export const MediaSidebar = memo(function MediaSidebar({
               {lottieTabActivated && <LottieBrowserPanel />}
             </div>
 
+            <div
+              className={`min-h-0 flex-1 overflow-y-auto ${activeTab === 'captions' ? 'block' : 'hidden'}`}
+            >
+              {captionsActivated && <HostCaptionLibrary />}
+            </div>
+
             {/* Transcript Tab */}
             <TranscriptSidebarContent
               activeTab={activeTab}

src/features/editor/host/caption-generation.test.ts

@@ -0,0 +1,151 @@
+import { describe, expect, it, vi } from 'vite-plus/test'
+import type { FreeCutFrameClip, FreeCutFrameDocument } from '../codepress/document'
+import type { EditorTranscriptPort, HostTranscriptSection } from './contract'
+import {
+  captionRangesForEdit,
+  assertCaptionTargetAvailable,
+  loadCaptionSections,
+  validateCaptionPreview,
+} from './caption-generation'
+
+const section: HostTranscriptSection = {
+  id: 's',
+  transcriptId: 't',
+  ordinal: 0,
+  startUs: 0,
+  endUs: 3_000_000,
+  text: 'hello um world',
+  timingSource: 'provider',
+  words: [
+    { text: 'hello', startUs: 0, endUs: 1_000_000 },
+    { text: 'um', startUs: 1_000_000, endUs: 2_000_000 },
+    { text: 'world', startUs: 2_000_000, endUs: 3_000_000 },
+  ],
+}
+const clip: FreeCutFrameClip = {
+  type: 'video',
+  id: 'a',
+  trackId: 'v',
+  mediaId: 'm',
+  from: 0,
+  durationInFrames: 30,
+  sourceStart: 0,
+  sourceEnd: 30,
+}
+const document: FreeCutFrameDocument = {
+  timelineId: 'timeline',
+  revision: 4,
+  fps: 30,
+  width: 1280,
+  height: 720,
+  durationInFrames: 150,
+  media: [],
+  tracks: [
+    {
+      id: 'v',
+      kind: 'video',
+      name: 'Video',
+      locked: false,
+      muted: false,
+      items: [
+        clip,
+        { ...clip, id: 'after-cut', from: 30, sourceStart: 60, sourceEnd: 90 },
+        { ...clip, id: 'repeat', from: 60, durationInFrames: 90, sourceEnd: 90 },
+      ],
+    },
+  ],
+}
+
+describe('captions from the edited sequence', () => {
+  it('excludes removed words while retaining an untouched repeated occurrence', () => {
+    expect(captionRangesForEdit(document, 'm', [section])).toEqual([
+      { itemId: 'a', startUs: 0, endUs: 1_000_000, text: 'hello' },
+      { itemId: 'after-cut', startUs: 2_000_000, endUs: 3_000_000, text: 'world' },
+      { itemId: 'repeat', startUs: 0, endUs: 3_000_000, text: 'hello um world' },
+    ])
+    expect(captionRangesForEdit(document, 'm', [section], new Set(['after-cut']))).toEqual([
+      { itemId: 'after-cut', startUs: 2_000_000, endUs: 3_000_000, text: 'world' },
+    ])
+  })
+  it('scopes individually selected linked audio before occurrence deduplication', () => {
+    const linked = {
+      ...document,
+      tracks: [
+        {
+          ...document.tracks[0]!,
+          items: [
+            { ...clip, linkedGroupId: 'linked' },
+            { ...clip, type: 'audio' as const, id: 'audio', linkedGroupId: 'linked' },
+          ],
+        },
+      ],
+    }
+    expect(captionRangesForEdit(linked, 'm', [section], new Set(['audio']))).toEqual([
+      { itemId: 'audio', startUs: 0, endUs: 1_000_000, text: 'hello' },
+    ])
+    expect(captionRangesForEdit(linked, 'm', [section])).toEqual([
+      { itemId: 'a', startUs: 0, endUs: 1_000_000, text: 'hello' },
+    ])
+  })
+  it('clamps measured word boundaries to trimmed source handles for caption containment', () => {
+    const trimmed = {
+      ...document,
+      tracks: [
+        { ...document.tracks[0]!, items: [{ ...clip, sourceStart: 15, sourceEnd: 75, speed: 2 }] },
+      ],
+    }
+    expect(captionRangesForEdit(trimmed, 'm', [section])).toEqual([
+      { itemId: 'a', startUs: 500_000, endUs: 2_500_000, text: 'hello um world' },
+    ])
+  })
+  it('fails honestly without real timing or selected clips', () => {
+    expect(() =>
+      captionRangesForEdit(document, 'm', [{ ...section, timingSource: 'synthetic' }]),
+    ).toThrow('No timed transcript')
+    expect(() => captionRangesForEdit(document, 'm', [section], new Set())).toThrow(
+      'No timed transcript',
+    )
+  })
+  it('loads all pages and rejects a repeated cursor instead of silently captioning a partial edit', async () => {
+    const getSections = vi
+      .fn()
+      .mockResolvedValueOnce({
+        transcriptId: 't',
+        sections: [section],
+        hasMore: true,
+        nextCursor: 'next',
+      })
+      .mockResolvedValue({ transcriptId: 't', sections: [], hasMore: true, nextCursor: 'next' })
+    await expect(
+      loadCaptionSections({ getSections } as unknown as EditorTranscriptPort, 't'),
+    ).rejects.toThrow('completely')
+  })
+  it('rejects a footage-changing command masquerading as captions', () => {
+    expect(() =>
+      validateCaptionPreview(
+        { commandBatch: { commands: [{ type: 'ripple_delete' }] } } as never,
+        {} as never,
+        'timeline',
+      ),
+    ).toThrow()
+  })
+  it('rejects an existing generated caption track hidden below video or locked', () => {
+    const track = {
+      id: 'host-transcript-captions',
+      name: 'Transcript captions',
+      kind: 'caption' as const,
+      locked: false,
+      muted: false,
+      items: [],
+    }
+    expect(() =>
+      assertCaptionTargetAvailable({ ...document, tracks: [...document.tracks, track] }, true),
+    ).toThrow('below video')
+    expect(() =>
+      assertCaptionTargetAvailable({ ...document, tracks: [{ ...track, locked: true }] }, true),
+    ).toThrow('locked')
+    expect(() => assertCaptionTargetAvailable({ ...document, tracks: [track] }, false)).toThrow(
+      'Replace all cues',
+    )
+  })
+})