diff --git a/docs/qa/ux7144-word-editing.md b/docs/qa/ux7144-word-editing.md new file mode 100644 index 000000000..296c9cb2f --- /dev/null +++ b/docs/qa/ux7144-word-editing.md @@ -0,0 +1,60 @@ +# Word editing — #7144 PR4 + +This source change is stacked on FreeCut PR38 (`ux7144-03-reliability`, +6c11e173656b7b00b2c418784959dbc4e0a71803). Its consumer companion is stacked on +CodePress PR7147 (a69438dc447c913f47f1af28aeb05ca54630f58f). The parent owns +package adoption, the 0.3.12 vendor patch, asset guards, and aggregate QA. +No package version was changed or published for this work. + +Provider words remain immutable source data. The host contract adds measured +`words`, `timingSource`, per-range `itemId`, and the `occurrenceSelection` opt-in. +The source mapper rejects synthetic, malformed, or partial word coverage. It +projects each occurrence through source trims and speed into unrounded sequence +frames; the backend quantizes once to the nearest rational project-frame +boundary (half up) and rejects zero-frame cuts. Only proven linked same-source, +same-time representations are deduplicated. Sparse word selections do not merge +across unselected words. + +The host view seeks/selects words, extends by drag/Shift or keyboard, cuts +immediately through preview + one controller submission, and derives remaining +words from authoritative occurrences. Selection binds to document revision and +transcript/source identity. Older hosts can seek but cannot cut words. Playback +highlight is separate from selection; gaps clear it, and manual reading +suspends following until Resume following. Standalone defaults to the whole +edit and uses its existing single undoable split/ripple operation with ranges +keyed by occurrence. Linked media cut together even if clip linked-selection is +disabled. Lock and transition/frame preflight run before mutation. + +Validation performed in this isolated worktree: + +- Focused host UI/mapping, standalone mapping/range removal tests (including + repeated sources, linked audio, undo/redo, disjoint source ranges, and gaps). +- Browser fixtures for host: click seek/select, Backspace cuts one repeat, + local fixture persistence after reload, undo restoration, active-word gap, + follow suspend/resume, Shift+arrow selection, adjacent textarea Backspace, + section-only fallback, and old-host capability denial; no page errors. +- Browser fixture for standalone: whole-edit default, real store Backspace + footage cut, repeated source preserved, one undo restores words/footage; + no page errors. +- Source boundary/dependency/edge/unused-export checks, package build and + installed-package consumer smoke tests. Changed-health passes against the + actual PR38 base. Full source check encounters PR38's two inherited readonly + fixture `.push` errors in controller.test.ts; parent owns their test-only fix. + +Browser fixture artifacts and scripts remain in this worker's ignored +`tmp/ux7144-pr4/` directory: host-selection.png, host-legacy.png, +host-narrow.png, standalone.png, browser.mjs, standalone-browser.mjs and TSX +fixtures. These use manually specified timing fixtures, not claimed provider +transcription runs. The host persistence/undo transport is a fixture; it is not +an authenticated CodePress session or durable backend-history browser proof. +Backend companion tests separately exercise real atomic preview/apply in a +worker-owned disposable Postgres instance. No live-project transcription or +media upload request was made. + +Explicit follow-ups agreed with the parent: mixed-source host transcript +batching (current source is labeled), durable text correction, and a real +regenerate/align/cache-bypass path. Existing Generate transcript can reuse a +cached transcript; Start over does not supply missing words. Audio listening, +authenticated aggregate CodePress playback/reload, and the unprompted user +usability session remain integration evidence gaps. Caption design/export is +PR5's scope. diff --git a/packages/freecut-editor/src/index.d.ts b/packages/freecut-editor/src/index.d.ts index f1f7363b9..4e4d145fc 100644 --- a/packages/freecut-editor/src/index.d.ts +++ b/packages/freecut-editor/src/index.d.ts @@ -304,7 +304,16 @@ export interface HostTranscriptStatusReceipt { error?: HostTranscriptError | null } +/** Only measured provider timings are eligible for word editing. */ +export interface HostTranscriptWord { + startUs: number + endUs: number + text: string +} + export interface HostTranscriptSection { + timingSource?: 'provider' | 'synthetic' + words?: readonly HostTranscriptWord[] | null id: string transcriptId: string ordinal: number @@ -345,6 +354,8 @@ export interface HostTranscriptSearchPage { } export interface HostTranscriptRange { + /** Exact sequence occurrence; omitted retains legacy all-occurrences behavior. */ + itemId?: string startUs: number endUs: number text?: string @@ -391,6 +402,8 @@ export interface HostTranscriptCommandPreview { } export interface EditorTranscriptPort { + /** Explicit opt-in: this host validates range.itemId and never broadens its scope. */ + occurrenceSelection?: boolean getStatus(): Promise | HostTranscriptStatusReceipt | null requestTranscription?(input: { assetId: string diff --git a/packages/freecut-editor/src/index.ts b/packages/freecut-editor/src/index.ts index 30133f3ab..d94cf24f7 100644 --- a/packages/freecut-editor/src/index.ts +++ b/packages/freecut-editor/src/index.ts @@ -48,6 +48,7 @@ export type { HostTranscriptSearchPage, HostTranscriptSearchRequest, HostTranscriptSection, + HostTranscriptWord, HostTranscriptSectionsPage, HostTranscriptSectionsRequest, HostTranscriptStatus, diff --git a/src/features/editor/host/contract.ts b/src/features/editor/host/contract.ts index d291e3d65..7ce8f8783 100644 --- a/src/features/editor/host/contract.ts +++ b/src/features/editor/host/contract.ts @@ -198,7 +198,16 @@ export interface HostTranscriptStatusReceipt { } /** One bounded, source-addressable transcript section. */ +/** Only measured provider timings are eligible for word editing. */ +export interface HostTranscriptWord { + startUs: Microseconds + endUs: Microseconds + text: string +} + export interface HostTranscriptSection { + timingSource?: 'provider' | 'synthetic' + words?: readonly HostTranscriptWord[] | null id: string transcriptId: string ordinal: number @@ -240,6 +249,8 @@ export interface HostTranscriptSearchPage { /** A positive integer-microsecond source range selected for preview. */ export interface HostTranscriptRange { + /** Exact sequence occurrence; omitted retains legacy all-occurrences behavior. */ + itemId?: string startUs: Microseconds endUs: Microseconds text?: string @@ -292,6 +303,8 @@ export interface HostTranscriptCommandPreview { * those details never enter the FreeCut surface. */ export interface EditorTranscriptPort { + /** Explicit opt-in: this host validates range.itemId and never broadens its scope. */ + occurrenceSelection?: boolean getStatus(): Promise | HostTranscriptStatusReceipt | null /** * Optional host-owned transcription start. The host performs the work and diff --git a/src/features/editor/host/controller.test.ts b/src/features/editor/host/controller.test.ts index f75d73205..aa0bd9be7 100644 --- a/src/features/editor/host/controller.test.ts +++ b/src/features/editor/host/controller.test.ts @@ -1907,10 +1907,13 @@ describe('gesture authority and recovery regressions', () => { const initial = snapshot() const anchor = initial.timeline.tracks[0]!.items[0]! Object.assign(anchor, { from: 10, sourceStart: 20, sourceEnd: 140, speed: 2 }) - initial.timeline.tracks[0]!.items.push( - { ...anchor, id: 'next', from: 70, sourceStart: 0, sourceEnd: 120 }, - { ...anchor, id: 'detached', from: 130, rippleLinked: false }, - ) + Object.assign(initial.timeline.tracks[0]!, { + items: [ + ...initial.timeline.tracks[0]!.items, + { ...anchor, id: 'next', from: 70, sourceStart: 0, sourceEnd: 120 }, + { ...anchor, id: 'detached', from: 130, rippleLinked: false }, + ], + }) const batch = trimIntentBatch(initial.timeline, 'clip-1', { handle: 'start', deltaFrames: 5, @@ -1944,10 +1947,11 @@ describe('gesture authority and recovery regressions', () => { it('never interprets a new delete selection as a retry of an unknown earlier delete', async () => { const initial = snapshot() - initial.timeline.tracks[0]!.items.push({ - ...initial.timeline.tracks[0]!.items[0]!, - id: 'clip-2', - from: 60, + Object.assign(initial.timeline.tracks[0]!, { + items: [ + ...initial.timeline.tracks[0]!.items, + { ...initial.timeline.tracks[0]!.items[0]!, id: 'clip-2', from: 60 }, + ], }) const submitEdit = vi.fn(async () => { throw new Error('unknown outcome') diff --git a/src/features/editor/host/index.ts b/src/features/editor/host/index.ts index ad2645826..4a03b45a4 100644 --- a/src/features/editor/host/index.ts +++ b/src/features/editor/host/index.ts @@ -49,6 +49,7 @@ export type { HostTranscriptSearchPage, HostTranscriptSearchRequest, HostTranscriptSection, + HostTranscriptWord, HostTranscriptSectionsPage, HostTranscriptSectionsRequest, HostTranscriptStatus, diff --git a/src/features/editor/host/transcript-editor.test.tsx b/src/features/editor/host/transcript-editor.test.tsx index 4f42c53c9..5d870535a 100644 --- a/src/features/editor/host/transcript-editor.test.tsx +++ b/src/features/editor/host/transcript-editor.test.tsx @@ -21,6 +21,7 @@ import { type EditCommandBatch, } from '@/features/editor/codepress' import { useEditorStore } from '@/shared/state/editor' +import { usePlaybackStore } from '@/shared/state/playback' import { MediaSidebar } from '../components/media-sidebar' import { EditorHostProvider } from './context-provider' import { @@ -31,6 +32,7 @@ import { type HostTranscriptCommandPreview, type HostTranscriptCommandPreviewRequest, type HostTranscriptSection, + type HostTranscriptSectionsPage, type HostTranscriptStatusReceipt, } from './contract' import { EmbeddedEditorHostRuntime } from './runtime' @@ -68,6 +70,7 @@ const sections: HostTranscriptSection[] = [ startUs: 1_000_000, endUs: 2_000_000, text: 'First bounded caption.', + timingSource: 'provider', speaker: 'Speaker 1', }, { @@ -77,6 +80,7 @@ const sections: HostTranscriptSection[] = [ startUs: 4_000_000, endUs: 5_000_000, text: 'Second bounded caption.', + timingSource: 'provider', speaker: 'Speaker 1', }, ] @@ -202,6 +206,34 @@ function createHarness( const previewCommands = vi.fn( async (request: HostTranscriptCommandPreviewRequest): Promise => { const isCut = request.action === 'cut' || request.action === 'ripple_cut' + // Model the backend's occurrence projection, then execute through the real adapter. + const targetedCutBatch = request.ranges?.every((range) => range.itemId) + ? { + ...cutBatch, + base_revision: request.baseRevision, + commands: request.ranges + .map((range, index) => { + const clip = remoteSnapshot.timeline.tracks + .flatMap((track) => track.items) + .find((item) => item.id === range.itemId) + if (!clip || (clip.type !== 'video' && clip.type !== 'audio')) + throw new Error('Unknown fixture occurrence') + const frameUs = 1_000_000 / remoteSnapshot.project.fps + const sourceStartUs = (clip.sourceStart ?? 0) * frameUs + const speed = clip.speed ?? 1 + return { + command_id: `targeted-cut-${index}`, + type: 'ripple_delete' as const, + start_us: Math.round( + clip.from * frameUs + (range.startUs - sourceStartUs) / speed, + ), + end_us: Math.round(clip.from * frameUs + (range.endUs - sourceStartUs) / speed), + track_ids: null, + } + }) + .sort((a, b) => b.start_us - a.start_us), + } + : cutBatch return { status: previewStatus, receiptId: 'transcript-receipt-1', @@ -214,7 +246,7 @@ function createHarness( operationId: previewBatch.operation_id, idempotencyKey: previewBatch.idempotency_key, baseRevision: 0, - commandBatch: isCut ? cutBatch : previewBatch, + commandBatch: isCut ? targetedCutBatch : previewBatch, preview: isCut ? { action: 'cut', selectionCount: 1, willMutateTimeline: false } : { action: 'captions', captionCount: 1, willMutateTimeline: false }, @@ -295,6 +327,16 @@ function renderHostEditor(harness: ReturnType) { ) } +function deferredPage() { + let resolve!: (page: HostTranscriptSectionsPage) => void + let reject!: (error: Error) => void + const promise = new Promise((resolvePage, rejectPage) => { + resolve = resolvePage + reject = rejectPage + }) + return { promise, resolve, reject } +} + function receipt( status: HostTranscriptStatusReceipt['status'], overrides: Partial = {}, @@ -326,6 +368,274 @@ afterEach(() => { }) describe('host-backed transcript consumer', () => { + it('selects provider words, seeks, suspends follow and immediately submits one occurrence cut', async () => { + const initial = snapshot() + initial.timeline.media = [ + { + media_id: 'asset-1', + media_kind: 'video', + content_hash: 'sha256:source-1', + duration_us: 10_000_000, + availability: { mode: 'cloud', cloud: { object_id: 'object-1' } }, + }, + ] + initial.timeline.tracks = [ + { + id: 'video', + name: 'Video', + kind: 'video', + locked: false, + muted: false, + items: [ + { + type: 'video', + id: 'occurrence-a', + trackId: 'video', + mediaId: 'asset-1', + from: 0, + durationInFrames: 300, + sourceStart: 0, + sourceEnd: 300, + }, + ], + }, + ] + const harness = createHarness(initial) + harness.host.transcript!.occurrenceSelection = true + harness.host.transcript!.getSections = () => ({ + transcriptId: 'transcript-1', + hasMore: false, + sections: [ + { + ...sections[0]!, + timingSource: 'provider', + words: [ + { text: 'First', startUs: 1_000_000, endUs: 1_300_000 }, + { text: 'bounded', startUs: 1_400_000, endUs: 1_700_000 }, + { text: 'caption.', startUs: 1_800_000, endUs: 2_000_000 }, + ], + }, + ], + }) + renderHostEditor(harness) + const first = await screen.findByRole('button', { name: 'First' }) + fireEvent.pointerDown(first) + expect(usePlaybackStore.getState().currentFrame).toBe(30) + expect(screen.getByRole('button', { name: 'Resume following' })).toBeInTheDocument() + const region = screen.getByRole('region', { name: 'Timed transcript words' }) + fireEvent.keyDown(region, { key: 'ArrowRight', shiftKey: true }) + expect(screen.getByRole('button', { name: 'bounded' })).toHaveAttribute('aria-pressed', 'true') + fireEvent.keyDown(region, { key: 'Backspace' }) + await waitFor(() => expect(harness.submitEdit).toHaveBeenCalledTimes(1)) + expect(harness.previewCommands).toHaveBeenCalledWith( + expect.objectContaining({ + baseRevision: 0, + ranges: [ + { itemId: 'occurrence-a', startUs: 1_000_000, endUs: 1_700_000, text: 'First bounded' }, + ], + }), + ) + expect(screen.queryByTestId('host-transcript-apply')).not.toBeInTheDocument() + }) + + it('invalidates a repeated-occurrence word selection when Load more reorders the projection', async () => { + const initial = snapshot() + initial.timeline.durationInFrames = 600 + initial.timeline.media = [ + { + media_id: 'asset-1', + media_kind: 'video', + content_hash: 'sha256:source-1', + duration_us: 10_000_000, + availability: { mode: 'cloud', cloud: { object_id: 'object-1' } }, + }, + ] + const occurrenceA = { + type: 'video' as const, + id: 'occurrence-a', + trackId: 'video', + mediaId: 'asset-1', + from: 0, + durationInFrames: 300, + sourceStart: 0, + sourceEnd: 300, + } + initial.timeline.tracks = [ + { + id: 'video', + name: 'Video', + kind: 'video', + locked: false, + muted: false, + items: [occurrenceA, { ...occurrenceA, id: 'occurrence-b', from: 300 }], + }, + ] + const harness = createHarness(initial) + harness.host.transcript!.occurrenceSelection = true + const pages = sections.map((section, index) => ({ + ...section, + text: index === 0 ? 'X' : 'Y', + words: [{ text: index === 0 ? 'X' : 'Y', startUs: section.startUs, endUs: section.endUs }], + })) + harness.host.transcript!.getSections = vi.fn(({ cursor }) => ({ + transcriptId: 'transcript-1', + sections: [pages[cursor ? 1 : 0]!], + hasMore: !cursor, + nextCursor: cursor ? null : 'page-2', + })) + renderHostEditor(harness) + const originalA = structuredClone( + harness.runtime.controller.getSnapshot().timeline.tracks[0]!.items[0], + ) + const words = await screen.findAllByRole('button', { name: 'X' }) + fireEvent.pointerDown(words[1]!) // index1 is B:X before pagination. + fireEvent.pointerUp(window) + fireEvent.click(screen.getByRole('button', { name: 'Load more transcript' })) + await screen.findAllByRole('button', { name: 'Y' }) // index1 is now A:Y. + expect(harness.runtime.controller.getSnapshot().timeline.revision).toBe(0) + const region = screen.getByRole('region', { name: 'Timed transcript words' }) + fireEvent.keyDown(region, { key: 'Delete' }) + expect(harness.previewCommands).not.toHaveBeenCalled() + expect(harness.submitEdit).not.toHaveBeenCalled() + expect( + screen.getByText('The edit or transcript changed. Select the words again.'), + ).toBeInTheDocument() + expect(screen.getByRole('button', { name: 'Delete selection' })).toBeDisabled() + + // A fresh B:X selection derives an actual ripple command from B + X's source range. + fireEvent.pointerDown(screen.getAllByRole('button', { name: 'X' })[1]!) + fireEvent.pointerUp(window) + fireEvent.keyDown(region, { key: 'Delete' }) + await waitFor(() => expect(harness.submitEdit).toHaveBeenCalledTimes(1)) + expect(harness.previewCommands).toHaveBeenCalledWith( + expect.objectContaining({ + baseRevision: 0, + ranges: [{ itemId: 'occurrence-b', startUs: 1_000_000, endUs: 2_000_000, text: 'X' }], + }), + ) + expect(harness.submitEdit.mock.calls[0]![0].commands).toEqual([ + expect.objectContaining({ + type: 'ripple_delete', + start_us: 11_000_000, + end_us: 12_000_000, + }), + ]) + await waitFor(() => expect(harness.runtime.controller.getSnapshot().timeline.revision).toBe(1)) + const edited = harness.runtime.controller.getSnapshot().timeline + expect(edited.tracks[0]!.items.find((item) => item.id === 'occurrence-a')).toEqual(originalA) + expect(edited.durationInFrames).toBe(570) + expect(screen.getAllByRole('button', { name: 'X' })).toHaveLength(1) + expect(screen.getAllByRole('button', { name: 'Y' })).toHaveLength(2) + }) + + it.each(['resolve', 'reject'] as const)( + 'ignores an old transcript page that %ss after refresh while the new source is loading more', + async (outcome) => { + const initial = snapshot() + initial.timeline.durationInFrames = 600 + initial.timeline.media = ['asset-1', 'asset-2'].map((id) => ({ + media_id: id, + media_kind: 'video', + content_hash: `sha256:${id}`, + duration_us: 10_000_000, + availability: { mode: 'cloud', cloud: { object_id: id } }, + })) + initial.timeline.tracks = [ + { + id: 'video', + name: 'Video', + kind: 'video', + locked: false, + muted: false, + items: ['asset-1', 'asset-2'].map((id, index) => ({ + type: 'video', + id: `clip-${id}`, + trackId: 'video', + mediaId: id, + from: index * 300, + durationInFrames: 300, + sourceStart: 0, + sourceEnd: 300, + })), + }, + ] + const harness = createHarness(initial) + harness.host.transcript!.occurrenceSelection = true + let fresh = false + harness.host.transcript!.getStatus = () => + receipt( + 'succeeded', + fresh + ? { + transcriptId: 'transcript-2', + assetId: 'asset-2', + sourceAssetHash: 'sha256:asset-2', + } + : {}, + ) + const oldPage = deferredPage() + const newPage = deferredPage() + const page = ( + transcriptId: string, + text: string, + ordinal: number, + ): HostTranscriptSectionsPage => ({ + transcriptId, + hasMore: ordinal === 0, + nextCursor: ordinal === 0 ? 'next' : null, + sections: [ + { + id: `${transcriptId}-${ordinal}`, + transcriptId, + ordinal, + startUs: ordinal * 2_000_000, + endUs: ordinal * 2_000_000 + 1_000_000, + text, + timingSource: 'provider', + words: [{ text, startUs: ordinal * 2_000_000, endUs: ordinal * 2_000_000 + 1_000_000 }], + }, + ], + }) + const getSections = vi.fn( + ({ transcriptId, cursor }: { transcriptId: string; cursor?: string | null }) => { + if (cursor) return transcriptId === 'transcript-1' ? oldPage.promise : newPage.promise + return page(transcriptId, transcriptId === 'transcript-1' ? 'Original' : 'Fresh', 0) + }, + ) + harness.host.transcript!.getSections = getSections + renderHostEditor(harness) + await screen.findByRole('button', { name: 'Original' }) + fireEvent.click(screen.getByRole('button', { name: 'Load more transcript' })) + expect(getSections).toHaveBeenCalledWith( + expect.objectContaining({ transcriptId: 'transcript-1', cursor: 'next' }), + ) + fresh = true + fireEvent.click(screen.getByRole('button', { name: 'Refresh transcript' })) + await screen.findByRole('button', { name: 'Fresh' }) + expect(screen.queryByRole('button', { name: 'Original' })).not.toBeInTheDocument() + fireEvent.click(screen.getByRole('button', { name: 'Load more transcript' })) + expect(getSections).toHaveBeenCalledWith( + expect.objectContaining({ transcriptId: 'transcript-2', cursor: 'next' }), + ) + await act(async () => { + if (outcome === 'resolve') oldPage.resolve(page('transcript-1', 'Forbidden', 1)) + else oldPage.reject(new Error('Obsolete source page failed')) + await Promise.resolve() + }) + expect(screen.queryByRole('button', { name: 'Forbidden' })).not.toBeInTheDocument() + expect(screen.queryByTestId('host-transcript-error')).not.toBeInTheDocument() + expect(screen.getByRole('button', { name: 'Load more transcript' })).toBeDisabled() + await act(async () => { + newPage.resolve(page('transcript-2', 'Latest', 1)) + }) + await screen.findByRole('button', { name: 'Latest' }) + expect(screen.queryByRole('button', { name: 'Load more transcript' })).not.toBeInTheDocument() + expect(screen.queryByRole('button', { name: 'Forbidden' })).not.toBeInTheDocument() + expect(harness.previewCommands).not.toHaveBeenCalled() + expect(harness.submitEdit).not.toHaveBeenCalled() + }, + ) + it('displays bounded sections, previews without mutation, then applies through submitEdit', async () => { const harness = createHarness(snapshot()) renderHostEditor(harness) @@ -547,7 +857,7 @@ describe('host-backed transcript consumer', () => { ).toBeInTheDocument() expect(requestTranscription).toHaveBeenCalledWith({ assetId: 'asset-1', language: 'en' }) expect(polls).toBeGreaterThanOrEqual(2) - expect(screen.getByTestId('host-transcript-status')).toHaveTextContent('succeeded') + expect(screen.getByTestId('host-transcript-status')).toBeEmptyDOMElement() }) it('hides Transcribe when the transcription capability is off', async () => { diff --git a/src/features/editor/host/transcript-editor.tsx b/src/features/editor/host/transcript-editor.tsx index b8bd4c707..02d9d1c94 100644 --- a/src/features/editor/host/transcript-editor.tsx +++ b/src/features/editor/host/transcript-editor.tsx @@ -36,6 +36,8 @@ import { } from './contract' import { useEditorHostContext } from './context' import { useHostTranscriptEditorRuntime } from './transcript-editor-context' +import { HostTranscriptWordView } from './transcript-word-view' +import { hasUsableHostWordTiming } from './transcript-words' const OPAQUE_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u const SAFE_HASH_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u @@ -249,9 +251,35 @@ function normalizeSection(value: unknown, expectedTranscriptId: string): HostTra endUs: endUs as number, text, speaker: speaker ?? null, + timingSource: value.timingSource === 'provider' ? 'provider' : 'synthetic', + words: normalizeWords(value.words, startUs as number, endUs as number), } } +function normalizeWords( + value: unknown, + sectionStart: number, + sectionEnd: number, +): HostTranscriptSection['words'] { + if (value === undefined || value === null) return null + if (!Array.isArray(value) || value.length > 2000) throw new Error('Invalid transcript words.') + let previousEnd = sectionStart + return value.map((raw) => { + if (!isRecord(raw) || !boundedText(raw.text, MAX_TRANSCRIPT_SECTION_TEXT_BYTES)) + throw new Error('Invalid transcript word text.') + const startUs = raw.startUs + const endUs = raw.endUs + if (typeof startUs !== 'number' || typeof endUs !== 'number') + throw new Error('Invalid transcript word timing.') + if (![startUs, endUs].every(Number.isSafeInteger)) + throw new Error('Invalid transcript word timing.') + if (startUs < previousEnd || endUs <= startUs || endUs > sectionEnd) + throw new Error('Invalid transcript word bounds.') + previousEnd = endUs + return { startUs, endUs, text: raw.text } + }) +} + // fallow-ignore-next-line complexity function normalizePage(value: unknown, expectedTranscriptId: string): NormalizedPage { if (!isRecord(value) || value.transcriptId !== expectedTranscriptId) { @@ -561,14 +589,14 @@ export function HostTranscriptEditor({ active = true }: { active?: boolean }) { replace: boolean, ): Promise => { if (!port) return - const page = normalizePage( - await port.getSections({ - transcriptId: receipt.transcriptId, - cursor, - limit: MAX_TRANSCRIPT_SECTION_PAGE_SIZE, - }), - receipt.transcriptId, - ) + const generation = requestGeneration.current + const response = await port.getSections({ + transcriptId: receipt.transcriptId, + cursor, + limit: MAX_TRANSCRIPT_SECTION_PAGE_SIZE, + }) + if (requestGeneration.current !== generation) return + const page = normalizePage(response, receipt.transcriptId) const durationUs = receipt.durationUs if ( (durationUs !== null && page.sections.some((section) => section.endUs > durationUs)) || @@ -579,6 +607,7 @@ export function HostTranscriptEditor({ active = true }: { active?: boolean }) { throw new Error('The host returned transcript sections outside the bounded transcript.') } setSections((current) => { + if (requestGeneration.current !== generation) return current const next = replace ? [] : [...current] const ids = new Set(next.map((section) => section.id)) for (const section of page.sections) { @@ -600,6 +629,7 @@ export function HostTranscriptEditor({ active = true }: { active?: boolean }) { const generation = requestGeneration.current + 1 requestGeneration.current = generation setLoading(true) + setLoadingMore(false) setError(null) setPreview(null) setSelectedIds(new Set()) @@ -645,17 +675,22 @@ export function HostTranscriptEditor({ active = true }: { active?: boolean }) { useEffect(() => { if (!active || !port || !canTranscribe) return void refresh() + return () => { + requestGeneration.current += 1 + } }, [active, canTranscribe, port, refresh]) const loadMore = useCallback(async () => { if (!status || status.status !== 'succeeded' || !nextCursor || !port || loadingMore) return + const generation = requestGeneration.current setLoadingMore(true) try { await loadSectionsPage(status, nextCursor, false) } catch (caught) { + if (requestGeneration.current !== generation) return setError(errorFromHost(caught, 'More transcript sections could not be loaded.')) } finally { - setLoadingMore(false) + if (requestGeneration.current === generation) setLoadingMore(false) } }, [loadSectionsPage, loadingMore, nextCursor, port, status]) @@ -702,7 +737,10 @@ export function HostTranscriptEditor({ active = true }: { active?: boolean }) { const previewSelection = useCallback( // fallow-ignore-next-line complexity - async (action: HostTranscriptCommandAction) => { + async ( + action: HostTranscriptCommandAction, + wordSelection?: { ranges: HostTranscriptRange[]; revision: number }, + ) => { if (!port || !runtime || !status || status.status !== 'succeeded' || previewing) return if (!isOpaqueId(status.assetId)) { setError({ @@ -718,8 +756,25 @@ export function HostTranscriptEditor({ active = true }: { active?: boolean }) { setError(null) setAnnouncement('Preparing a non-mutating transcript preview…') try { - const ranges = rangesForSelection(selectedSections) + if ( + !wordSelection && + selectedSections.some((section) => section.timingSource !== 'provider') + ) { + throw new Error( + 'Measured section timing is needed before footage can be cut or captioned.', + ) + } + const ranges = wordSelection?.ranges ?? rangesForSelection(selectedSections) const currentSnapshot = runtime.controller.getSnapshot() + if ( + wordSelection && + (wordSelection.revision !== currentSnapshot.timeline.revision || + !port.occurrenceSelection) + ) { + throw new Error( + 'This edit changed or the host cannot target occurrences. Select the words again.', + ) + } const request = { transcriptId: status.transcriptId, assetId: status.assetId, @@ -748,6 +803,19 @@ export function HostTranscriptEditor({ active = true }: { active?: boolean }) { ) { throw new Error('The transcript preview contains an unsupported timeline command.') } + if (wordSelection) { + const applied = await runtime.controller.submitEdit(result.commandBatch) + if (applied.status !== 'applied' && applied.status !== 'replayed') { + throw new Error( + applied.status === 'unsupported' + ? applied.reason + : 'This edit could not be saved. Refresh and select the words again.', + ) + } + setPreview(null) + setAnnouncement('Selected footage deleted and gap closed. Undo restores this cut.') + return + } setPreview(result) setAnnouncement( result.status === 'replayed' @@ -903,6 +971,8 @@ export function HostTranscriptEditor({ active = true }: { active?: boolean }) { } }, [port, refresh, status, transcribeAssetId, transcribing]) + const hasWordTiming = sections.some(hasUsableHostWordTiming) + if (!port || !runtime || !canTranscribe) { return ( + )} + {hasWordTiming && sections.some((section) => !hasUsableHostWordTiming(section)) && ( +

+ Some sections need word timing and cannot be edited as words. +

+ )} + {hasWordTiming && status?.assetId ? ( + previewSelection('cut', { ranges, revision })} + /> + ) : ( + <> +
+

Word timing needed

+

+ Word editing is unavailable for this transcript. These controls select whole sections. + Check Generate transcript for supported transcription options; regeneration may reuse + cached timing. +

- ) : ( -
- {visibleSections.map((section, index) => { - const selected = selectedIds.has(section.id) - return ( - + ) + })} +
+ )} + {hasMore ? ( +
+ - ) - })} + {loadingMore && } + Load more + + +
+ ) : null} - )} - {hasMore ? ( -
- -
- ) : null} - - {preview ? ( -
-
- -
-

- {preview.status === 'replayed' ? 'Preview replayed safely.' : 'Preview ready.'} -

-

+ {preview ? ( +

+
+ +
+

+ {preview.status === 'replayed' ? 'Preview replayed safely.' : 'Preview ready.'} +

+

+ {previewAction === 'captions' || previewAction === 'caption' + ? `${preview.preview.captionCount ?? selectedIds.size} caption(s) · timeline unchanged` + : `${selectedIds.size} range(s) · timeline unchanged`} +

+
+
+
-
- -
- ) : null} + ) : null} -
- - {selectedIds.size > 0 - ? `${selectedIds.size} section${selectedIds.size === 1 ? '' : 's'} selected` - : 'Select sections to cut or caption'} - -
- - -
-
+
+ + {selectedIds.size > 0 + ? `${selectedIds.size} section${selectedIds.size === 1 ? '' : 's'} selected` + : 'Select sections to cut or caption'} + +
+ + +
+
+ + )} {announcement} diff --git a/src/features/editor/host/transcript-word-view.tsx b/src/features/editor/host/transcript-word-view.tsx new file mode 100644 index 000000000..1a6b6ed6a --- /dev/null +++ b/src/features/editor/host/transcript-word-view.tsx @@ -0,0 +1,292 @@ +import { transcriptSelectionIndex } from '@/shared/utils/transcript-selection' +import { useEffect, useMemo, useRef, useState } from 'react' +import { Button } from '@/components/ui/button' +import { usePlaybackStore } from '@/shared/state/playback' +import { useSelectionStore } from '@/shared/state/selection' +import { useEditorStore } from '@/shared/state/editor' +import type { EmbeddedEditorHostRuntime } from './runtime' +import type { HostTranscriptRange, HostTranscriptSection } from './contract' +import { + hostWordSelectionRanges, + mapHostTranscriptWords, + type HostWordOccurrence, +} from './transcript-words' + +function selectionDurationFrames(words: readonly HostWordOccurrence[]): number { + const intervals = hostWordSelectionRanges(words) + .map((range) => { + const selected = words.filter( + (word) => + word.itemId === range.itemId && + word.sourceStartUs >= range.startUs && + word.sourceEndUs <= range.endUs, + ) + return { + start: Math.min(...selected.map((word) => word.startFrame)), + end: Math.max(...selected.map((word) => word.endFrame)), + } + }) + .sort((a, b) => a.start - b.start) + let end = 0 + let duration = 0 + for (const interval of intervals) { + duration += Math.max(0, interval.end - Math.max(end, interval.start)) + end = Math.max(end, interval.end) + } + return duration +} + +export function HostTranscriptWordView({ + runtime, + assetId, + sections, + query, + canCut, + busy, + onDelete, +}: { + runtime: EmbeddedEditorHostRuntime + assetId: string + sections: readonly HostTranscriptSection[] + query: string + canCut: boolean + busy: boolean + onDelete: (ranges: HostTranscriptRange[], revision: number) => Promise +}) { + const [document, setDocument] = useState(() => runtime.controller.getSnapshot().timeline) + const [scope, setScope] = useState('edit') + const selectedClips = useSelectionStore((state) => state.selectedItemIds) + const [selection, setSelection] = useState<{ + anchor: number + focus: number + revision: number + projection: string + } | null>(null) + const [following, setFollowing] = useState(true) + const scroll = useRef(null) + const dragging = useRef(false) + useEffect(() => () => useEditorStore.getState().setTranscriptEditorShortcutScopeActive(false), []) + const inFlight = useRef(false) + const currentFrame = usePlaybackStore((state) => state.currentFrame) + const playing = usePlaybackStore((state) => state.isPlaying) + useEffect( + () => runtime.controller.subscribe((snapshot) => setDocument(snapshot.timeline)), + [runtime], + ) + const words = useMemo( + () => + mapHostTranscriptWords(document, assetId, sections).filter( + (word) => scope === 'edit' || selectedClips.includes(word.itemId), + ), + [document, assetId, sections, scope, selectedClips], + ) + // Pagination can reorder repeated occurrences without changing the timeline revision. + // Bind indices to the complete measured projection, including retranscribed timings/text. + const projection = useMemo(() => JSON.stringify(words), [words]) + const otherSources = document.tracks.some((track) => + track.items.some((item) => 'mediaId' in item && item.mediaId !== assetId), + ) + const active = words.findIndex( + (word) => currentFrame >= word.startFrame && currentFrame < word.endFrame, + ) + const stale = + selection !== null && + (selection.revision !== document.revision || selection.projection !== projection) + const selected = + selection && !stale + ? words.slice( + Math.min(selection.anchor, selection.focus), + Math.max(selection.anchor, selection.focus) + 1, + ) + : [] + const duration = selectionDurationFrames(selected) / runtime.controller.getSnapshot().project.fps + + useEffect(() => { + if (following && playing && active >= 0) + scroll.current + ?.querySelector(`[data-word-index="${active}"]`) + ?.scrollIntoView({ block: 'nearest' }) + }, [following, playing, active]) + useEffect(() => { + const stop = () => { + dragging.current = false + } + window.addEventListener('pointerup', stop) + return () => window.removeEventListener('pointerup', stop) + }, []) + const choose = (index: number, extend: boolean) => { + setFollowing(false) + setSelection((previous) => ({ + anchor: + extend && previous?.revision === document.revision && previous.projection === projection + ? previous.anchor + : index, + focus: index, + revision: document.revision, + projection, + })) + usePlaybackStore.getState().setCurrentFrame(Math.floor(words[index]!.startFrame)) + } + const remove = async () => { + if (!canCut || busy || inFlight.current || selected.length === 0 || !selection || stale) return + inFlight.current = true + try { + await onDelete(hostWordSelectionRanges(selected), selection.revision) + setSelection(null) + } finally { + inFlight.current = false + } + } + return ( +
+
+ + {!following && ( + + )} +
+ {otherSources && ( +

+ Showing this transcript’s source in the current edit. Other source transcripts are not + included. +

+ )} +
useEditorStore.getState().setTranscriptEditorShortcutScopeActive(true)} + onBlur={(event) => { + if (!event.currentTarget.contains(event.relatedTarget as Node | null)) + useEditorStore.getState().setTranscriptEditorShortcutScopeActive(false) + }} + onWheel={() => setFollowing(false)} + onTouchMove={() => setFollowing(false)} + onPointerDown={(event) => { + if (event.target === event.currentTarget) setFollowing(false) + }} + onPointerMove={(event) => { + if (!dragging.current) return + const element = globalThis.document + .elementFromPoint(event.clientX, event.clientY) + ?.closest('[data-word-index]') + if (element && scroll.current?.contains(element)) { + const index = Number(element.dataset.wordIndex) + setSelection((previous) => (previous ? { ...previous, focus: index } : previous)) + } + }} + onKeyDown={(event) => { + if (event.key === 'Backspace' || event.key === 'Delete') { + event.preventDefault() + event.stopPropagation() + void remove() + } else if (event.key === 'Escape') { + event.stopPropagation() + setSelection(null) + } else { + const next = transcriptSelectionIndex( + event.key, + stale ? 0 : (selection?.focus ?? 0), + words.length, + ) + if (next === null) return + event.preventDefault() + event.stopPropagation() + choose(next, event.shiftKey) + } + }} + > + {words.length === 0 && ( +

+ No timed words in {scope === 'selection' ? 'the selected clip' : 'this edit'}. +

+ )} +
+ {words.map((word, index) => { + const previous = words[index - 1] + const paragraph = + !previous || previous.itemId !== word.itemId || previous.sectionId !== word.sectionId + const chosen = selected.some((entry) => entry.key === word.key) + const matches = + query && word.text.toLocaleLowerCase().includes(query.toLocaleLowerCase()) + return ( + + {paragraph && ( + + {Math.floor( + word.startFrame / runtime.controller.getSnapshot().project.fps / 60, + )} + : + {String( + Math.floor(word.startFrame / runtime.controller.getSnapshot().project.fps) % + 60, + ).padStart(2, '0')} + + )} + {' '} + + ) + })} +
+
+ {stale && ( +

+ The edit or transcript changed. Select the words again. +

+ )} + {!canCut && ( +

+ This host needs occurrence-aware transcript editing before words can be cut. +

+ )} +
+ +

+ Select words, then Backspace or Delete to cut footage and close the gap. Undo restores the + cut. +

+
+
+ ) +} diff --git a/src/features/editor/host/transcript-words.test.ts b/src/features/editor/host/transcript-words.test.ts new file mode 100644 index 000000000..fab1df218 --- /dev/null +++ b/src/features/editor/host/transcript-words.test.ts @@ -0,0 +1,133 @@ +import { describe, expect, it } from 'vite-plus/test' +import type { FreeCutFrameDocument, FreeCutFrameClip } from '../codepress/document' +import type { HostTranscriptSection } from './contract' +import { hostWordSelectionRanges, mapHostTranscriptWords } from './transcript-words' + +const section: HostTranscriptSection = { + id: 'section', + transcriptId: 'transcript', + ordinal: 0, + startUs: 0, + endUs: 3_000_000, + text: 'hello um world', + timingSource: 'provider', + words: [ + { text: 'hello', startUs: 0, endUs: 500_000 }, + { text: 'um', startUs: 1_000_000, endUs: 1_200_000 }, + { text: 'world', startUs: 2_000_000, endUs: 3_000_000 }, + ], +} +function document(clips: FreeCutFrameClip[]): FreeCutFrameDocument { + return { + timelineId: 'timeline', + revision: 7, + fps: 30, + width: 1280, + height: 720, + durationInFrames: 300, + media: [], + tracks: [ + { id: 'video', kind: 'video', name: 'Video', locked: false, muted: false, items: clips }, + ], + } +} +const clip: FreeCutFrameClip = { + type: 'video', + id: 'a', + mediaId: 'media', + trackId: 'video', + from: 0, + durationInFrames: 90, + sourceStart: 0, + sourceEnd: 90, +} + +describe('host word occurrence mapping', () => { + it('retains repeated occurrences, clamps trims and maps speed without quantizing provider words', () => { + const words = mapHostTranscriptWords( + document([ + clip, + { + ...clip, + id: 'b', + from: 100, + sourceStart: 33, + sourceEnd: 90, + durationInFrames: 38, + speed: 1.5, + }, + ]), + 'media', + [section], + ) + expect(words.map((word) => word.key)).toEqual([ + 'a:section:0', + 'a:section:1', + 'a:section:2', + 'b:section:1', + 'b:section:2', + ]) + expect(words[3]).toMatchObject({ + revision: 7, + itemId: 'b', + sourceStartUs: 1_000_000, + startFrame: 100, + endFrame: 102, + }) + expect(hostWordSelectionRanges(words.slice(1, 2))).toEqual([ + { itemId: 'a', startUs: 1_000_000, endUs: 1_200_000, text: 'um' }, + ]) + expect(hostWordSelectionRanges(words.slice(2, 4)).map((range) => range.itemId)).toEqual([ + 'a', + 'b', + ]) + }) + it('only deduplicates proven linked same-source/time representations', () => { + const words = mapHostTranscriptWords( + document([ + { ...clip, linkedGroupId: 'group' }, + { ...clip, id: 'audio', type: 'audio', linkedGroupId: 'group' }, + { ...clip, id: 'independent', type: 'audio' }, + { ...clip, id: 'different-time', from: 100, linkedGroupId: 'group' }, + ]), + 'media', + [section], + ) + expect([...new Set(words.map((word) => word.itemId))]).toEqual([ + 'a', + 'independent', + 'different-time', + ]) + }) + it('rejects incomplete or malformed provider word coverage', () => { + expect( + mapHostTranscriptWords(document([clip]), 'media', [ + { ...section, words: section.words!.slice(1) }, + ]), + ).toEqual([]) + expect( + mapHostTranscriptWords(document([clip]), 'media', [ + { ...section, words: [{ text: section.text, startUs: NaN, endUs: 3_000_000 }] }, + ]), + ).toEqual([]) + expect( + mapHostTranscriptWords(document([clip]), 'media', [ + { ...section, words: [{ text: section.text, startUs: 0, endUs: 3_000_001 }] }, + ]), + ).toEqual([]) + }) + + it('never invents word timing from legacy or synthetic sections and leaves gaps inactive', () => { + expect( + mapHostTranscriptWords(document([clip]), 'media', [ + { ...section, timingSource: 'synthetic' }, + ]), + ).toEqual([]) + expect( + mapHostTranscriptWords(document([clip]), 'media', [{ ...section, words: null }]), + ).toEqual([]) + const words = mapHostTranscriptWords(document([clip]), 'media', [section]) + expect(words.find((word) => 20 >= word.startFrame && 20 < word.endFrame)).toBeUndefined() + expect(hostWordSelectionRanges([words[0]!, words[2]!])).toHaveLength(2) + }) +}) diff --git a/src/features/editor/host/transcript-words.ts b/src/features/editor/host/transcript-words.ts new file mode 100644 index 000000000..7b8209206 --- /dev/null +++ b/src/features/editor/host/transcript-words.ts @@ -0,0 +1,129 @@ +import type { FreeCutFrameDocument, FreeCutFrameClip } from '../codepress/document' +import { framesToMicroseconds } from '../codepress/timing' +import type { HostTranscriptRange, HostTranscriptSection } from './contract' + +export interface HostWordOccurrence { + key: string + itemId: string + wordIndex: number + sectionId: string + revision: number + text: string + sourceStartUs: number + sourceEndUs: number + /** Unrounded frame coordinates; quantization belongs to the command boundary. */ + startFrame: number + endFrame: number +} + +/** Fail closed on partial, synthetic or malformed word data, including direct mapper callers. */ +export function hasUsableHostWordTiming(section: HostTranscriptSection): boolean { + if (section.timingSource !== 'provider' || !section.words?.length || section.words.length > 2000) + return false + let previousEnd = section.startUs + for (const word of section.words) { + if ( + !Number.isSafeInteger(word.startUs) || + !Number.isSafeInteger(word.endUs) || + word.startUs < previousEnd || + word.endUs <= word.startUs || + word.endUs > section.endUs || + typeof word.text !== 'string' || + !word.text.trim() + ) + return false + previousEnd = word.endUs + } + const normalize = (text: string) => text.replace(/[\s\p{P}]/gu, '').toLocaleLowerCase() + return normalize(section.words.map((word) => word.text).join(' ')) === normalize(section.text) +} + +/** Project source words into the current edit, retaining each distinct occurrence. */ +export function mapHostTranscriptWords( + document: FreeCutFrameDocument, + assetId: string, + sections: readonly HostTranscriptSection[], +): HostWordOccurrence[] { + const words: HostWordOccurrence[] = [] + const linked = new Set() + const clips = document.tracks + .flatMap((track) => track.items) + .filter((item) => (item.type === 'video' || item.type === 'audio') && item.mediaId === assetId) + .sort((a, b) => Number(a.type !== 'video') - Number(b.type !== 'video')) + for (const clip of clips) { + if (clip.type !== 'video' && clip.type !== 'audio') continue + const cohort = clip.linkedGroupId + ? `${clip.linkedGroupId}:${clip.mediaId}:${clip.from}:${clip.durationInFrames}:${clip.sourceStart}:${clip.sourceEnd}:${clip.speed ?? 1}` + : null + if (cohort && linked.has(cohort)) continue + if (cohort) linked.add(cohort) + words.push(...wordsForClip(clip, document, sections)) + } + return words.sort((a, b) => a.startFrame - b.startFrame) +} + +function wordsForClip( + clip: FreeCutFrameClip, + document: FreeCutFrameDocument, + sections: readonly HostTranscriptSection[], +): HostWordOccurrence[] { + const words: HostWordOccurrence[] = [] + const sourceStart = framesToMicroseconds(clip.sourceStart ?? 0, document.fps) + const sourceEnd = framesToMicroseconds( + clip.sourceEnd ?? (clip.sourceStart ?? 0) + clip.durationInFrames * (clip.speed ?? 1), + document.fps, + ) + if (sourceEnd <= sourceStart) return [] + for (const section of sections) { + if (!hasUsableHostWordTiming(section)) continue + section.words?.forEach((word, index) => { + if (word.endUs <= sourceStart || word.startUs >= sourceEnd) return + const start = Math.max(sourceStart, word.startUs) + const end = Math.min(sourceEnd, word.endUs) + words.push({ + key: `${clip.id}:${section.id}:${index}`, + itemId: clip.id, + wordIndex: index, + sectionId: section.id, + revision: document.revision, + text: word.text, + // Preserve the measured word boundaries. The backend intersects the trim. + sourceStartUs: word.startUs, + sourceEndUs: word.endUs, + startFrame: + clip.from + ((start - sourceStart) / (sourceEnd - sourceStart)) * clip.durationInFrames, + endFrame: + clip.from + ((end - sourceStart) / (sourceEnd - sourceStart)) * clip.durationInFrames, + }) + }) + } + return words +} + +/** Adjacent selected words merge only within the same occurrence. */ +export function hostWordSelectionRanges( + words: readonly HostWordOccurrence[], +): HostTranscriptRange[] { + const ranges: HostTranscriptRange[] = [] + let previousWord: HostWordOccurrence | undefined + for (const word of words) { + const previous = ranges.at(-1) + if ( + previous?.itemId === word.itemId && + previousWord?.sectionId === word.sectionId && + previousWord.wordIndex + 1 === word.wordIndex + ) { + previous.endUs = Math.max(previous.endUs, word.sourceEndUs) + previous.text += ` ${word.text}` + } else { + ranges.push({ + itemId: word.itemId, + startUs: word.sourceStartUs, + endUs: word.sourceEndUs, + text: word.text, + }) + } + previousWord = word + } + return ranges +} diff --git a/src/features/timeline/components/transcript-editor/transcript-editor-panel.tsx b/src/features/timeline/components/transcript-editor/transcript-editor-panel.tsx index a220cce93..6bd2e10a7 100644 --- a/src/features/timeline/components/transcript-editor/transcript-editor-panel.tsx +++ b/src/features/timeline/components/transcript-editor/transcript-editor-panel.tsx @@ -1,3 +1,4 @@ +import { transcriptSelectionIndex } from '@/shared/utils/transcript-selection' import { Fragment, memo, @@ -59,6 +60,7 @@ import { } from '../../deps/media-transcription-service' import { buildRemovalRangesByMediaId, + buildRemovalRangesByItemId, buildTranscriptTokens, findActiveTokenIndex, getSelectedTokenSlice, @@ -275,7 +277,8 @@ export function TranscriptEditorPanel({ active }: TranscriptEditorPanelProps) { const ignoreRanges = useTranscriptIgnoreStore((s) => s.ranges) const setTranscriptShortcutScope = useEditorStore((s) => s.setTranscriptEditorShortcutScopeActive) - const [scope, setScope] = useState('selection') + const [following, setFollowing] = useState(true) + const [scope, setScope] = useState('project') const [mediaState, setMediaState] = useState>({}) const [anchorIndex, setAnchorIndex] = useState(-1) const [focusIndex, setFocusIndex] = useState(-1) @@ -404,7 +407,7 @@ export function TranscriptEditorPanel({ active }: TranscriptEditorPanelProps) { setAnchorIndex(-1) setFocusIndex(-1) setMatchCursor(0) - }, [uniqueMediaIds, scope]) + }, [uniqueMediaIds, scope, allItems, tokens]) useEffect(() => { mountedRef.current = true @@ -478,14 +481,14 @@ export function TranscriptEditorPanel({ active }: TranscriptEditorPanelProps) { // Keep the active word in view during playback. Skip entirely when hidden — a // querySelector + scrollIntoView every frame on an off-screen panel is pure waste. useEffect(() => { - if (!active || !isPlaying || activeIndex < 0) return + if (!active || !following || !isPlaying || activeIndex < 0) return const key = tokens[activeIndex]?.key if (!key) return const el = scrollRef.current?.querySelector( `[data-token-key="${CSS.escape(key)}"]`, ) el?.scrollIntoView({ block: 'nearest' }) - }, [active, isPlaying, activeIndex, tokens]) + }, [active, following, isPlaying, activeIndex, tokens]) useEffect(() => { const stop = () => { @@ -503,6 +506,7 @@ export function TranscriptEditorPanel({ active }: TranscriptEditorPanelProps) { (index: number, event: ReactPointerEvent) => { const token = tokens[index] if (!token) return + setFollowing(false) if (event.shiftKey && anchorIndexRef.current >= 0) { setFocusIndex(index) } else { @@ -536,18 +540,27 @@ export function TranscriptEditorPanel({ active }: TranscriptEditorPanelProps) { if (Number.isInteger(index)) setFocusIndex(index) }, []) - // Non-destructive: striking words stages them as "ignored" (restorable) rather - // than cutting the timeline. Re-striking an already-ignored selection restores it. const handleIgnoreToggle = useCallback(() => { if (selectedSlice.length === 0) return - const ranges = buildRemovalRangesByMediaId(selectedSlice) - const allIgnored = selectedSlice.every((token) => ignoredKeys.has(token.key)) - if (allIgnored) { - useTranscriptIgnoreStore.getState().restore(ranges) - } else { - useTranscriptIgnoreStore.getState().ignore(ranges) + try { + const result = useTimelineStore + .getState() + .removeTranscriptRangesFromItems( + [...new Set(selectedSlice.map((token) => token.itemId))], + buildRemovalRangesByMediaId(selectedSlice), + buildRemovalRangesByItemId(selectedSlice), + ) + if (result.removedItemCount === 0) { + toast.error('The selection cannot be cut. Check locked tracks and transition boundaries.') + return + } + setAnchorIndex(-1) + setFocusIndex(-1) + toast.success('Selected footage deleted. Undo restores this cut.') + } catch { + toast.error(t('transcript.toastRemoveFailed')) } - }, [selectedSlice, ignoredKeys]) + }, [selectedSlice, t]) // Word-level copy/cut that carries the media: each run of selected words // becomes a trimmed clone of its clip, placed on the shared clipboard so the @@ -576,7 +589,13 @@ export function TranscriptEditorPanel({ active }: TranscriptEditorPanelProps) { const rangesByMediaId = buildRemovalRangesByMediaId(selectedSlice) const itemIds = Array.from(new Set(selectedSlice.map((token) => token.itemId))) try { - useTimelineStore.getState().removeTranscriptRangesFromItems(itemIds, rangesByMediaId) + useTimelineStore + .getState() + .removeTranscriptRangesFromItems( + itemIds, + rangesByMediaId, + buildRemovalRangesByItemId(selectedSlice), + ) } catch (error) { logger.warn('Transcript cut failed', error) toast.error(t('transcript.toastRemoveFailed')) @@ -685,23 +704,35 @@ export function TranscriptEditorPanel({ active }: TranscriptEditorPanelProps) { return } + const next = transcriptSelectionIndex(event.key, focusIndex, tokens.length) + if (next !== null) { + event.preventDefault() + event.stopPropagation() + setFollowing(false) + if (!event.shiftKey || anchorIndex < 0) setAnchorIndex(next) + setFocusIndex(next) + seekToToken(tokens[next]!.startFrame) + return + } if (event.key === 'Delete' || event.key === 'Backspace') { // Always own these so they never fall through to the timeline's clip // delete — even with no selection (then it's simply a no-op). event.preventDefault() event.stopPropagation() if (selectedKeys.size > 0) handleIgnoreToggle() - } else if (event.key === 'Enter' && (event.metaKey || event.ctrlKey)) { - if (ignoredSpanCount === 0) return - event.preventDefault() - event.stopPropagation() - handleApply() } else if (event.key === 'Escape') { setAnchorIndex(-1) setFocusIndex(-1) } }, - [selectedKeys.size, handleIgnoreToggle, ignoredSpanCount, handleApply], + [ + selectedKeys.size, + handleIgnoreToggle, + tokens, + focusIndex, + anchorIndex, + seekToToken, + ], ) const needsTranscription = uniqueMediaIds.filter( @@ -712,61 +743,64 @@ export function TranscriptEditorPanel({ active }: TranscriptEditorPanelProps) { return status === 'loading' || status === 'transcribing' }) - const handleTranscribe = useCallback((values: TranscribeDialogValues) => { - const targets = uniqueMediaIds.filter((id) => { - const status = mediaState[id]?.status - return status === 'needs' || status === 'error' - }) - if (targets.length === 0) return + const handleTranscribe = useCallback( + (values: TranscribeDialogValues) => { + const targets = uniqueMediaIds.filter((id) => { + const status = mediaState[id]?.status + return status === 'needs' || status === 'error' + }) + if (targets.length === 0) return - setTranscribeDialogOpen(false) + setTranscribeDialogOpen(false) - for (const id of targets) requestedRef.current.add(id) - setMediaState((prev) => { - const next = { ...prev } - for (const id of targets) next[id] = { status: 'transcribing' } - return next - }) + for (const id of targets) requestedRef.current.add(id) + setMediaState((prev) => { + const next = { ...prev } + for (const id of targets) next[id] = { status: 'transcribing' } + return next + }) - void Promise.all( - targets.map(async (mediaId) => { - try { - const result = await runMediaTranscriptionJob(mediaId, { - ...values, - onModelFallback: () => { - toast.info(t('transcript.largeTurboFallback')) - }, - }) - if (!mountedRef.current) return - if (result.status === 'cancelled') { - setMediaState((prev) => ({ ...prev, [mediaId]: { status: 'needs' } })) - return - } - const { transcript } = result - setMediaState((prev) => ({ - ...prev, - [mediaId]: hasWordTimings(transcript) - ? { status: 'ready', transcript } - : { status: 'needs' }, - })) - } catch (error) { - logger.warn('Transcription failed', { mediaId, error }) - const errorMessage = isTranscriptionOutOfMemoryError(error) - ? TRANSCRIPTION_OOM_HINT - : error instanceof Error && error.message.trim().length > 0 - ? error.message - : t('transcript.toastTranscribeFailed') - if (mountedRef.current) { + void Promise.all( + targets.map(async (mediaId) => { + try { + const result = await runMediaTranscriptionJob(mediaId, { + ...values, + onModelFallback: () => { + toast.info(t('transcript.largeTurboFallback')) + }, + }) + if (!mountedRef.current) return + if (result.status === 'cancelled') { + setMediaState((prev) => ({ ...prev, [mediaId]: { status: 'needs' } })) + return + } + const { transcript } = result setMediaState((prev) => ({ ...prev, - [mediaId]: { status: 'error', errorMessage }, + [mediaId]: hasWordTimings(transcript) + ? { status: 'ready', transcript } + : { status: 'needs' }, })) + } catch (error) { + logger.warn('Transcription failed', { mediaId, error }) + const errorMessage = isTranscriptionOutOfMemoryError(error) + ? TRANSCRIPTION_OOM_HINT + : error instanceof Error && error.message.trim().length > 0 + ? error.message + : t('transcript.toastTranscribeFailed') + if (mountedRef.current) { + setMediaState((prev) => ({ + ...prev, + [mediaId]: { status: 'error', errorMessage }, + })) + } + toast.error(errorMessage) } - toast.error(errorMessage) - } - }), - ) - }, [uniqueMediaIds, mediaState, t]) + }), + ) + }, + [uniqueMediaIds, mediaState, t], + ) const transcriptionError = useMemo( () => @@ -812,6 +846,11 @@ export function TranscriptEditorPanel({ active }: TranscriptEditorPanelProps) { {/* Scope toggle */}
+ {!following && ( + + )}
{/* Search */} @@ -889,6 +928,8 @@ export function TranscriptEditorPanel({ active }: TranscriptEditorPanelProps) { {/* Transcript body */}
setFollowing(false)} + onTouchMove={() => setFollowing(false)} onPointerMove={handlePointerMove} className="min-h-0 flex-1 overflow-x-hidden overflow-y-auto px-3 py-2" > @@ -1021,9 +1062,7 @@ export function TranscriptEditorPanel({ active }: TranscriptEditorPanelProps) { {selectionCount > 0 ? t('transcript.wordsSelected', { count: selectionCount }) - : t('transcript.ignoreHint', { - defaultValue: 'Select words, then Backspace to mark them for deletion', - })} + : 'Select words, then Backspace or Delete to cut footage'}
diff --git a/src/features/timeline/stores/actions/edit/range-removal-actions.ts b/src/features/timeline/stores/actions/edit/range-removal-actions.ts index 81caab759..ef87d86dc 100644 --- a/src/features/timeline/stores/actions/edit/range-removal-actions.ts +++ b/src/features/timeline/stores/actions/edit/range-removal-actions.ts @@ -61,9 +61,12 @@ function isMostlyInsideRanges( return covered / duration >= SILENCE_COVERAGE_REMOVAL_THRESHOLD } -function applyRippleRemoval(ids: string[]): { removedIds: string[]; affectedIds: string[] } { +function applyRippleRemoval( + ids: string[], + forceLinked = false, +): { removedIds: string[]; affectedIds: string[] } { const items = useItemsStore.getState().items - const linkedSelectionEnabled = isLinkedSelectionEnabled() + const linkedSelectionEnabled = forceLinked || isLinkedSelectionEnabled() const expandedIds = expandIdsWithLinkedItems(items, ids, linkedSelectionEnabled) if (expandedIds.length === 0) return { removedIds: [], affectedIds: [] } @@ -206,13 +209,20 @@ export function removeFillerWordsFromItems( export function removeTranscriptRangesFromItems( itemIds: string[], rangesByMediaId: Record, + rangesByItemId?: Record, ): RemoveSilenceResult { - return removeTimelineRangesFromItems('REMOVE_TRANSCRIPT_SELECTION', itemIds, rangesByMediaId) + return removeTimelineRangesFromItems( + 'REMOVE_TRANSCRIPT_SELECTION', + itemIds, + rangesByMediaId, + rangesByItemId, + ) } function getRangeRemovalAnchors( itemIds: string[], rangesByMediaId: Record, + rangesByItemId?: Record, ): TimelineItem[] { const store = useItemsStore.getState() const anchorIds = getUniqueLinkedItemAnchorIds(store.items, itemIds) @@ -223,7 +233,8 @@ function getRangeRemovalAnchors( item !== undefined && (item.type === 'video' || item.type === 'audio') && !!item.mediaId && - (rangesByMediaId[item.mediaId]?.length ?? 0) > 0, + ((rangesByItemId ? rangesByItemId[item.id] : rangesByMediaId[item.mediaId])?.length ?? 0) > + 0, ) } @@ -314,13 +325,14 @@ function addRangeDownstreamPreflight(params: { function buildRangeRemovalPreflight( itemIds: string[], rangesByMediaId: Record, + rangesByItemId?: Record, ): { analyzedItemCount: number; mutationIds: string[] } { const store = useItemsStore.getState() const timelineFps = useTimelineSettingsStore.getState().fps - const anchors = getRangeRemovalAnchors(itemIds, rangesByMediaId) + const anchors = getRangeRemovalAnchors(itemIds, rangesByMediaId, rangesByItemId) if (anchors.length === 0) return { analyzedItemCount: 0, mutationIds: [] } - const linkedSelectionEnabled = isLinkedSelectionEnabled() + const linkedSelectionEnabled = !!rangesByItemId || isLinkedSelectionEnabled() const accumulator: RangeRemovalPreflightAccumulator = { mutationIds: new Set(), editedTrackIds: new Set(), @@ -331,7 +343,7 @@ function buildRangeRemovalPreflight( for (const anchor of anchors) { addRangeAnchorPreflight({ anchor, - ranges: rangesByMediaId[anchor.mediaId!] ?? [], + ranges: (rangesByItemId ? rangesByItemId[anchor.id] : rangesByMediaId[anchor.mediaId!]) ?? [], timelineFps, linkedSelectionEnabled, accumulator, @@ -352,16 +364,69 @@ function buildRangeRemovalPreflight( return { analyzedItemCount: anchors.length, mutationIds: Array.from(accumulator.mutationIds) } } +function assertTranscriptSplitOutsideTransition(item: TimelineItem, frame: number): void { + if (frame <= item.from || frame >= item.from + item.durationInFrames) return + if (isInTransitionOverlap(item.id, frame - item.from, item.durationInFrames)) { + throw new Error('The selected word crosses a transition. Adjust the transition before cutting.') + } +} + +function transcriptOccurrenceTiming(item: TimelineItem, fps: number) { + const span = getItemSourceSpanSeconds(item, fps) + return [ + item.mediaId, + item.from, + item.durationInFrames, + item.speed ?? 1, + !!item.isReversed, + span?.start, + span?.end, + ] as const +} + +function assertTranscriptLinkedCohort(anchor: TimelineItem, linkedItems: TimelineItem[]): void { + const fps = useTimelineSettingsStore.getState().fps + const anchorTiming = transcriptOccurrenceTiming(anchor, fps) + const synchronized = linkedItems.every((item) => + transcriptOccurrenceTiming(item, fps).every((value, index) => value === anchorTiming[index]), + ) + if (!synchronized) { + throw new Error('Linked clips have different trims or timing. Align them before cutting words.') + } +} + +function assertTranscriptRangesRepresentable( + anchor: TimelineItem, + ranges: RemoveSilenceRange[], +): void { + const fps = useTimelineSettingsStore.getState().fps + const linkedItems = getLinkedItemsForEdit(useItemsStore.getState().items, anchor.id, true) + assertTranscriptLinkedCohort(anchor, linkedItems) + for (const range of ranges) { + const start = Math.max(anchor.from, sourceSecondsToTimelineFrame(anchor, range.start, fps)) + const end = Math.min( + anchor.from + anchor.durationInFrames, + sourceSecondsToTimelineFrame(anchor, range.end, fps), + ) + if (end <= start) throw new Error('The selected word is smaller than one timeline frame.') + for (const linked of linkedItems) { + assertTranscriptSplitOutsideTransition(linked, start) + assertTranscriptSplitOutsideTransition(linked, end) + } + } +} + function removeTimelineRangesFromItems( commandType: 'REMOVE_SILENCE' | 'REMOVE_FILLER_WORDS' | 'REMOVE_TRANSCRIPT_SELECTION', itemIds: string[], rangesByMediaId: Record, + rangesByItemId?: Record, ): RemoveSilenceResult { if (itemIds.length === 0) { return { analyzedItemCount: 0, removedRangeCount: 0, removedItemCount: 0, splitCount: 0 } } - const preflight = buildRangeRemovalPreflight(itemIds, rangesByMediaId) + const preflight = buildRangeRemovalPreflight(itemIds, rangesByMediaId, rangesByItemId) if (preflight.mutationIds.length === 0 || !canMutateTimelineItems(preflight.mutationIds)) { return { analyzedItemCount: preflight.analyzedItemCount, @@ -371,6 +436,12 @@ function removeTimelineRangesFromItems( } } + if (rangesByItemId) { + for (const anchor of getRangeRemovalAnchors(itemIds, rangesByMediaId, rangesByItemId)) { + assertTranscriptRangesRepresentable(anchor, rangesByItemId[anchor.id] ?? []) + } + } + return execute( commandType, () => { @@ -384,7 +455,8 @@ function removeTimelineRangesFromItems( item !== undefined && (item.type === 'video' || item.type === 'audio') && !!item.mediaId && - (rangesByMediaId[item.mediaId]?.length ?? 0) > 0, + ((rangesByItemId ? rangesByItemId[item.id] : rangesByMediaId[item.mediaId])?.length ?? + 0) > 0, ) if (anchors.length === 0) { @@ -395,11 +467,16 @@ function removeTimelineRangesFromItems( id: item.id, mediaId: item.mediaId!, originId: item.originId ?? item.id, + // Only these actual IDs and their split descendants belong to this selection. + // Shared originId/time bounds can also describe an independent stacked repeat. + descendantIds: new Set( + getLinkedItemsForEdit(initialItems, item.id, !!rangesByItemId).map((linked) => linked.id), + ), })) let splitCount = 0 for (const anchor of anchors) { - const ranges = rangesByMediaId[anchor.mediaId!] + const ranges = rangesByItemId ? rangesByItemId[anchor.id] : rangesByMediaId[anchor.mediaId!] if (!ranges || ranges.length === 0) continue const splitFrames = Array.from( @@ -418,7 +495,7 @@ function removeTimelineRangesFromItems( const itemsToSplit = getLinkedItemsForEdit( useItemsStore.getState().items, anchor.id, - isLinkedSelectionEnabled(), + !!rangesByItemId || isLinkedSelectionEnabled(), ) if (itemsToSplit.length === 0) continue @@ -454,6 +531,14 @@ function removeTimelineRangesFromItems( if (frameSplitResults.length !== itemsToSplit.length) continue + for (const descriptor of anchorDescriptors) { + for (const entry of frameSplitResults) { + if (descriptor.descendantIds.has(entry.originalId)) { + descriptor.descendantIds.add(entry.result.leftItem.id) + descriptor.descendantIds.add(entry.result.rightItem.id) + } + } + } applySplitBookkeeping(frameSplitResults) splitCount += 1 @@ -468,13 +553,19 @@ function removeTimelineRangesFromItems( const removedRangeKeys = new Set() for (const descriptor of anchorDescriptors) { - const ranges = rangesByMediaId[descriptor.mediaId] + const ranges = rangesByItemId + ? rangesByItemId[descriptor.id] + : rangesByMediaId[descriptor.mediaId] if (!ranges || ranges.length === 0) continue for (const candidate of currentItems) { if (candidate.type !== 'video' && candidate.type !== 'audio') continue - if (candidate.mediaId !== descriptor.mediaId) continue - if ((candidate.originId ?? candidate.id) !== descriptor.originId) continue + if (rangesByItemId) { + if (!descriptor.descendantIds.has(candidate.id)) continue + } else { + if (candidate.mediaId !== descriptor.mediaId) continue + if ((candidate.originId ?? candidate.id) !== descriptor.originId) continue + } const span = getItemSourceSpanSeconds(candidate, timelineFps) if (span !== null && isMostlyInsideRanges(span, ranges)) { @@ -497,7 +588,7 @@ function removeTimelineRangesFromItems( } } - const removalResult = applyRippleRemoval(Array.from(idsToRemove)) + const removalResult = applyRippleRemoval(Array.from(idsToRemove), !!rangesByItemId) const affectedIds = Array.from(new Set([...idsToRemove, ...removalResult.affectedIds])) requestPostEditWarmForItems(affectedIds) useTimelineSettingsStore.getState().markDirty() diff --git a/src/features/timeline/stores/actions/range-removal-actions.test.ts b/src/features/timeline/stores/actions/range-removal-actions.test.ts index 0419c8de5..ff29e3a0f 100644 --- a/src/features/timeline/stores/actions/range-removal-actions.test.ts +++ b/src/features/timeline/stores/actions/range-removal-actions.test.ts @@ -2,6 +2,7 @@ import { beforeEach, describe, expect, it } from 'vite-plus/test' import type { AudioItem, VideoItem } from '@/types/timeline' +import { makeTimelineTrack } from '../../test-helpers' import { useEditorStore } from '@/shared/state/editor' import { useSelectionStore } from '@/shared/state/selection' import { useKeyframesStore } from '../keyframes-store' @@ -9,7 +10,10 @@ import { useItemsStore } from '../items-store' import { useTimelineCommandStore } from '../timeline-command-store' import { useTimelineSettingsStore } from '../timeline-settings-store' import { useTransitionsStore } from '../transitions-store' -import { removeSilenceFromItems } from './edit/range-removal-actions' +import { + removeSilenceFromItems, + removeTranscriptRangesFromItems, +} from './edit/range-removal-actions' function makeVideoItem(overrides: Partial = {}): VideoItem { return { @@ -128,4 +132,126 @@ describe('removeSilenceFromItems', () => { expect(sourceSpans('video')).toHaveLength(2) expect(sourceSpans('audio')).toHaveLength(2) }) + it('cuts only selected occurrence ranges with linked audio and restores once', () => { + useEditorStore.setState({ linkedSelectionEnabled: false }) + const original = [ + makeVideoItem({ linkedGroupId: 'av' }), + makeAudioItem({ linkedGroupId: 'av' }), + makeVideoItem({ id: 'repeat', from: 300 }), + ] + useItemsStore.getState().setItems(original) + const result = removeTranscriptRangesFromItems( + ['video-1'], + { 'media-1': [{ start: 2, end: 4 }] }, + { 'video-1': [{ start: 2, end: 4 }] }, + ) + expect(result.removedItemCount).toBe(2) + expect(useItemsStore.getState().itemById.repeat).toMatchObject({ + sourceStart: 0, + sourceEnd: 300, + durationInFrames: 300, + from: 240, + }) + expect(sourceSpans('audio')).toEqual([ + { from: 0, durationInFrames: 60, sourceStart: 0, sourceEnd: 60 }, + { from: 60, durationInFrames: 180, sourceStart: 120, sourceEnd: 300 }, + ]) + useTimelineCommandStore.getState().undo() + expect(useItemsStore.getState().items).toEqual(original) + useTimelineCommandStore.getState().redo() + expect(useItemsStore.getState().itemById.repeat).toMatchObject({ + sourceStart: 0, + sourceEnd: 300, + durationInFrames: 300, + }) + }) + + it('does not cross-apply ranges when two selected occurrences share media and origin', () => { + useItemsStore.getState().setItems([makeVideoItem(), makeVideoItem({ id: 'repeat', from: 300 })]) + removeTranscriptRangesFromItems( + ['video-1', 'repeat'], + { + 'media-1': [ + { start: 2, end: 4 }, + { start: 6, end: 7 }, + ], + }, + { 'video-1': [{ start: 2, end: 4 }], repeat: [{ start: 6, end: 7 }] }, + ) + expect(sourceSpans('video')).toEqual([ + { from: 0, durationInFrames: 60, sourceStart: 0, sourceEnd: 60 }, + { from: 60, durationInFrames: 180, sourceStart: 120, sourceEnd: 300 }, + { from: 240, durationInFrames: 180, sourceStart: 0, sourceEnd: 180 }, + { from: 420, durationInFrames: 90, sourceStart: 210, sourceEnd: 300 }, + ]) + }) + it('preserves an unselected stacked repeat with shared origin inside the selected range', () => { + useItemsStore + .getState() + .setTracks([ + makeTimelineTrack({ id: 'video-track', name: 'A', order: 0, syncLock: false }), + makeTimelineTrack({ id: 'repeat-track', name: 'B', order: 1, syncLock: false }), + ]) + const repeat = makeVideoItem({ + id: 'repeat', + trackId: 'repeat-track', + from: 60, + durationInFrames: 60, + sourceStart: 60, + sourceEnd: 120, + }) + const original = [makeVideoItem(), repeat] + useItemsStore.getState().setItems(original) + const result = removeTranscriptRangesFromItems( + ['video-1'], + { 'media-1': [{ start: 2, end: 4 }] }, + { 'video-1': [{ start: 2, end: 4 }] }, + ) + expect(result.removedItemCount).toBe(1) + expect(useItemsStore.getState().itemById.repeat).toEqual(repeat) + expect( + useItemsStore + .getState() + .items.filter((item) => item.trackId === 'video-track') + .map(({ from, durationInFrames, sourceStart, sourceEnd }) => ({ + from, + durationInFrames, + sourceStart, + sourceEnd, + })), + ).toEqual([ + { from: 0, durationInFrames: 60, sourceStart: 0, sourceEnd: 60 }, + { from: 60, durationInFrames: 180, sourceStart: 120, sourceEnd: 300 }, + ]) + expect(useTimelineCommandStore.getState().undoStack).toHaveLength(1) + useTimelineCommandStore.getState().undo() + expect(useItemsStore.getState().items).toEqual(original) + useTimelineCommandStore.getState().redo() + expect(useItemsStore.getState().itemById.repeat).toEqual(repeat) + }) + + it.each([ + { label: 'asymmetric duration', audio: { durationInFrames: 300, sourceEnd: 300 } }, + { label: 'different source offset', audio: { sourceStart: 60, sourceEnd: 90 } }, + { label: 'reversed partner', audio: { isReversed: true } }, + ])('rejects $label before a word cut mutates linked footage or history', ({ audio }) => { + useEditorStore.setState({ linkedSelectionEnabled: false }) + const original = [ + makeVideoItem({ linkedGroupId: 'av', durationInFrames: 30, sourceEnd: 30 }), + makeAudioItem({ linkedGroupId: 'av', durationInFrames: 30, sourceEnd: 30, ...audio }), + ] + useItemsStore.getState().setItems(original) + expect(() => + removeTranscriptRangesFromItems( + ['video-1'], + { 'media-1': [{ start: 0, end: 1 }] }, + { 'video-1': [{ start: 0, end: 1 }] }, + ), + ).toThrow('Linked clips have different trims or timing') + expect(useItemsStore.getState().items).toEqual(original) + expect(useTimelineCommandStore.getState().undoStack).toHaveLength(0) + expect(useTimelineSettingsStore.getState().isDirty).toBe(false) + expect(useTransitionsStore.getState().transitions).toEqual([]) + expect(useKeyframesStore.getState().keyframes).toEqual([]) + }) }) diff --git a/src/features/timeline/types.ts b/src/features/timeline/types.ts index 2a4ffeacc..9af1ea4c3 100644 --- a/src/features/timeline/types.ts +++ b/src/features/timeline/types.ts @@ -118,6 +118,7 @@ export interface TimelineActions { removeTranscriptRangesFromItems: ( itemIds: string[], rangesByMediaId: Record>, + rangesByItemId?: Record>, ) => { analyzedItemCount: number removedRangeCount: number diff --git a/src/features/timeline/utils/transcript-edit-model.test.ts b/src/features/timeline/utils/transcript-edit-model.test.ts index 2b099cf89..3c55f91fb 100644 --- a/src/features/timeline/utils/transcript-edit-model.test.ts +++ b/src/features/timeline/utils/transcript-edit-model.test.ts @@ -91,8 +91,8 @@ describe('buildTranscriptTokens', () => { it('does not duplicate words for a linked video + audio pair', () => { // Same media, same source span (a linked companion) — words appear once. - const video = makeItem({ id: 'v', type: 'video', mediaId: 'm1' }) - const audio = makeItem({ id: 'a', type: 'audio', mediaId: 'm1' }) + const video = makeItem({ id: 'v', type: 'video', mediaId: 'm1', linkedGroupId: 'linked' }) + const audio = makeItem({ id: 'a', type: 'audio', mediaId: 'm1', linkedGroupId: 'linked' }) const transcript = makeTranscript('m1', [ { text: 'hello', start: 0, end: 0.5 }, { text: 'world', start: 0.5, end: 1.0 }, @@ -134,9 +134,8 @@ describe('buildTranscriptTokens', () => { expect(tokens.every((t) => t.itemId === 'v')).toBe(true) }) - it('collapses identical, identically-timed tokens from different media', () => { - // Same footage imported as two separate media (different mediaIds) stacked at - // the same spot escapes per-media dedup, so the token-level net must catch it. + it('retains independent identical words from different media', () => { + // Independently placed sources must remain independently selectable. const a = makeItem({ id: 'a', type: 'video', mediaId: 'm1', from: 0, durationInFrames: 300 }) const b = makeItem({ id: 'b', type: 'audio', mediaId: 'm2', from: 0, durationInFrames: 300 }) const words = [ @@ -149,7 +148,7 @@ describe('buildTranscriptTokens', () => { { m1: makeTranscript('m1', words), m2: makeTranscript('m2', words) }, FPS, ) - expect(tokens.map((t) => t.text)).toEqual(['hello', 'world']) + expect(tokens.map((t) => t.text)).toEqual(['hello', 'hello', 'world', 'world']) }) it('keeps distinct trims of the same media', () => { diff --git a/src/features/timeline/utils/transcript-edit-model.ts b/src/features/timeline/utils/transcript-edit-model.ts index bbce1cdf7..3ee67278a 100644 --- a/src/features/timeline/utils/transcript-edit-model.ts +++ b/src/features/timeline/utils/transcript-edit-model.ts @@ -62,12 +62,7 @@ export function buildTranscriptTokens( timelineFps: number, ): TranscriptToken[] { const tokens: TranscriptToken[] = [] - // Timeline ranges already emitted per media. A linked audio companion shares - // the exact `from`/duration of its video, so deduping on the timeline range - // (rather than the source span, which can differ between a video and its - // separately-based audio frames) reliably drops the companion. Distinct trims - // of the same media sit at different timeline positions and are both kept. - const acceptedRangesByMedia = new Map>() + const acceptedCohorts = new Set() // Prefer the video item when a video/audio pair covers the same range. const ordered = [...items].sort((a, b) => Number(a.type !== 'video') - Number(b.type !== 'video')) @@ -79,12 +74,11 @@ export function buildTranscriptTokens( const span = getItemSourceSpanSeconds(item, timelineFps) if (!span) continue - const from = item.from - const to = item.from + item.durationInFrames - const accepted = acceptedRangesByMedia.get(item.mediaId) ?? [] - if (accepted.some((other) => other.from < to && from < other.to)) continue - accepted.push({ from, to }) - acceptedRangesByMedia.set(item.mediaId, accepted) + const cohort = item.linkedGroupId + ? `${item.linkedGroupId}:${item.mediaId}:${item.from}:${item.durationInFrames}:${span.start}:${span.end}` + : null + if (cohort && acceptedCohorts.has(cohort)) continue + if (cohort) acceptedCohorts.add(cohort) const words = collectWords(transcript) words.forEach((word, index) => { @@ -106,20 +100,7 @@ export function buildTranscriptTokens( }) } - const sorted = tokens.toSorted((left, right) => left.startFrame - right.startFrame) - - // Final safety net: collapse tokens identical in text AND exact timeline timing. - // Two sources of the same spoken content (a linked companion, or the same - // footage imported as separate media so it escapes per-media dedup above) emit - // word-for-word identical, identically-timed tokens. Genuinely distinct clips - // map to different frames, so real repeated words are preserved. - const seen = new Set() - return sorted.filter((token) => { - const signature = `${token.startFrame}:${token.endFrame}:${token.text}` - if (seen.has(signature)) return false - seen.add(signature) - return true - }) + return tokens.toSorted((left, right) => left.startFrame - right.startFrame) } /** @@ -174,6 +155,20 @@ export function buildRemovalRangesByMediaId( return rangesByMediaId } +/** Keep source ranges isolated per selected occurrence, including repeated source runs. */ +export function buildRemovalRangesByItemId( + tokens: readonly TranscriptToken[], +): Record { + const ranges: Record = {} + for (const token of tokens) { + const list = ranges[token.itemId] ?? (ranges[token.itemId] = []) + const previous = list.at(-1) + if (previous) previous.end = Math.max(previous.end, token.sourceEnd) + else list.push({ start: token.sourceStart, end: token.sourceEnd }) + } + return ranges +} + /** Resolve a contiguous index range [anchor, focus] into the tokens it covers. */ export function getSelectedTokenSlice( tokens: readonly TranscriptToken[], diff --git a/src/shared/utils/transcript-selection.ts b/src/shared/utils/transcript-selection.ts new file mode 100644 index 000000000..3134bb149 --- /dev/null +++ b/src/shared/utils/transcript-selection.ts @@ -0,0 +1,19 @@ +/** Keyboard movement in a flat, document-ordered transcript. */ +export function transcriptSelectionIndex( + key: string, + current: number, + length: number, +): number | null { + if (length === 0) return null + if (key === 'Home') return 0 + if (key === 'End') return length - 1 + const directions: Record = { + ArrowLeft: -1, + ArrowUp: -1, + ArrowRight: 1, + ArrowDown: 1, + } + const direction = directions[key] + if (direction === undefined) return null + return Math.max(0, Math.min(length - 1, Math.max(0, current) + direction)) +}