From 65c4825d15d5f771a004c2e935d5e759c532a446 Mon Sep 17 00:00:00 2001 From: Dara Adedeji Date: Tue, 29 Sep 2026 01:11:07 -0400 Subject: [PATCH 1/8] Add a WebCodecs H.264 encoder and MP4 muxer for web video export Browsers cannot spawn the bundled ffmpeg, so the web beta needs its own encoder. Two building blocks, not wired into the exporter yet: - Mp4H264Muxer: pure-Dart faststart MP4 writer for one H.264 track (ftyp, moov, mdat; run-length stts, stss, one chunk). Verified against ffprobe and an ffmpeg decode of a real libx264 stream. - WebCodecsMp4Encoder: VideoEncoder bindings that turn RGBA frames into a constant-frame-rate MP4, repeating each frame for its duration, scaling through an OffscreenCanvas, and waiting on the encode queue. The export exceptions move to video_export_errors.dart so the web file avoids dart:io; ffmpeg_video_encoder.dart re-exports them. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../video_export/ffmpeg_video_encoder.dart | 11 +- lib/services/video_export/mp4_muxer.dart | 323 ++++++++++ .../video_export/video_export_errors.dart | 11 + .../web_codecs_video_encoder.dart | 564 ++++++++++++++++++ test/mp4_box_test_support.dart | 169 ++++++ test/mp4_muxer_test.dart | 298 +++++++++ test/web/web_codecs_video_encoder_test.dart | 290 +++++++++ 7 files changed, 1657 insertions(+), 9 deletions(-) create mode 100644 lib/services/video_export/mp4_muxer.dart create mode 100644 lib/services/video_export/video_export_errors.dart create mode 100644 lib/services/video_export/web_codecs_video_encoder.dart create mode 100644 test/mp4_box_test_support.dart create mode 100644 test/mp4_muxer_test.dart create mode 100644 test/web/web_codecs_video_encoder_test.dart diff --git a/lib/services/video_export/ffmpeg_video_encoder.dart b/lib/services/video_export/ffmpeg_video_encoder.dart index 3816f82c..746a943b 100644 --- a/lib/services/video_export/ffmpeg_video_encoder.dart +++ b/lib/services/video_export/ffmpeg_video_encoder.dart @@ -3,18 +3,11 @@ import 'dart:convert'; import 'dart:developer'; import 'dart:io'; +import 'package:icarus/services/video_export/video_export_errors.dart'; import 'package:icarus/services/video_export/video_export_quality.dart'; import 'package:path/path.dart' as p; -class VideoExportCancelled implements Exception {} - -class VideoExportException implements Exception { - VideoExportException(this.message); - final String message; - - @override - String toString() => 'VideoExportException: $message'; -} +export 'package:icarus/services/video_export/video_export_errors.dart'; /// Windows' save dialog does not guarantee that the selected extension is /// appended, even when FilePicker is restricted to MP4 files. diff --git a/lib/services/video_export/mp4_muxer.dart b/lib/services/video_export/mp4_muxer.dart new file mode 100644 index 00000000..39db7bd2 --- /dev/null +++ b/lib/services/video_export/mp4_muxer.dart @@ -0,0 +1,323 @@ +import 'dart:typed_data'; + +/// Builds an MP4 file holding one H.264 video track, entirely in memory. +/// +/// The layout is faststart (ftyp, moov, mdat) so browsers and chat apps can +/// start playing before the whole file has arrived. All samples live in a +/// single chunk, so the file needs exactly one chunk offset. +/// +/// Samples are stored in the order they are added and presented in that same +/// order: there is no composition offset table, so the stream must not +/// contain B-frames (decode order must equal presentation order). +/// +/// Pure Dart and 64-bit values are written as two 32-bit halves, so this runs +/// unchanged on the VM and on the web. +class Mp4H264Muxer { + Mp4H264Muxer({ + required this.width, + required this.height, + this.timescale = 90000, + }) { + if (width <= 0 || width > 0xFFFF || height <= 0 || height > 0xFFFF) { + throw ArgumentError('Video size must be within 1..65535 on each side.'); + } + if (timescale <= 0 || timescale > _maxUint32) { + throw ArgumentError.value(timescale, 'timescale', 'must be positive'); + } + } + + final int width; + final int height; + + /// Ticks per second for every duration passed to [addSample]. + final int timescale; + + final List _samples = []; + final List _durations = []; + final List _keyFrameNumbers = []; + int _payloadBytes = 0; + int _totalDuration = 0; + bool _finished = false; + + int get sampleCount => _samples.length; + + /// Total duration of the added samples, in [timescale] ticks. + int get totalDuration => _totalDuration; + + /// Adds one access unit in AVCC format (length-prefixed NAL units), exactly + /// as WebCodecs emits it with `avc: {format: 'avc'}`. [duration] is in + /// [timescale] ticks. + void addSample( + Uint8List sample, { + required int duration, + required bool isKeyFrame, + }) { + if (_finished) throw StateError('The MP4 has already been finished.'); + if (sample.isEmpty) { + throw ArgumentError('An H.264 sample cannot be empty.'); + } + if (duration <= 0 || duration > _maxUint32) { + throw ArgumentError.value( + duration, + 'duration', + 'must be between 1 and 2^32-1 ticks', + ); + } + if (_samples.isEmpty && !isKeyFrame) { + throw StateError('The first H.264 sample must be a keyframe.'); + } + _samples.add(sample); + _durations.add(duration); + if (isKeyFrame) _keyFrameNumbers.add(_samples.length); + _payloadBytes += sample.length; + _totalDuration += duration; + } + + /// Returns the finished file. [avcDecoderConfig] is the + /// AVCDecoderConfigurationRecord (the `description` WebCodecs reports in the + /// first chunk's `metadata.decoderConfig`). + Uint8List finish({required Uint8List avcDecoderConfig}) { + if (_finished) throw StateError('The MP4 has already been finished.'); + if (_samples.isEmpty) { + throw StateError('Cannot build an MP4 without any video samples.'); + } + if (avcDecoderConfig.isEmpty || avcDecoderConfig[0] != 1) { + throw ArgumentError( + 'avcDecoderConfig must be an AVCDecoderConfigurationRecord ' + '(configurationVersion 1).', + ); + } + _finished = true; + + final ftyp = _box('ftyp', [ + _fourCc('isom'), + _u32(0x200), + _fourCc('isom'), + _fourCc('iso2'), + _fourCc('avc1'), + _fourCc('mp41'), + ]); + + // The single chunk offset is known only once moov's size is known, and + // moov's size does not depend on the offset's value, so build moov twice. + final largeMdat = _payloadBytes + 8 > _maxUint32; + final mdatHeaderSize = largeMdat ? 16 : 8; + final moovSize = _moov(avcDecoderConfig, chunkOffset: 0).length; + final chunkOffset = ftyp.length + moovSize + mdatHeaderSize; + final moov = _moov(avcDecoderConfig, chunkOffset: chunkOffset); + assert(moov.length == moovSize); + + final out = BytesBuilder(copy: false) + ..add(ftyp) + ..add(moov); + if (largeMdat) { + out + ..add(_u32(1)) + ..add(_fourCc('mdat')) + ..add(_u64(_payloadBytes + 16)); + } else { + out + ..add(_u32(_payloadBytes + 8)) + ..add(_fourCc('mdat')); + } + for (final sample in _samples) { + out.add(sample); + } + return out.takeBytes(); + } + + Uint8List _moov(Uint8List avcC, {required int chunkOffset}) { + // Durations past 32 bits need the version-1 time boxes. + final longDuration = _totalDuration > _maxUint32; + return _box('moov', [ + _mvhd(longDuration), + _box('trak', [ + _tkhd(longDuration), + _box('mdia', [ + _mdhd(longDuration), + _fullBox('hdlr', 0, 0, [ + _u32(0), // pre_defined + _fourCc('vide'), + Uint8List(12), // reserved + Uint8List.fromList([...'VideoHandler'.codeUnits, 0]), + ]), + _box('minf', [ + _fullBox('vmhd', 0, 1, [Uint8List(8)]), // graphicsmode, opcolor + _box('dinf', [ + _fullBox('dref', 0, 0, [ + _u32(1), + _fullBox('url ', 0, 1, const []), // media is in this file + ]), + ]), + _stbl(avcC, chunkOffset), + ]), + ]), + ]), + ]); + } + + Uint8List _mvhd(bool longDuration) => + _fullBox('mvhd', longDuration ? 1 : 0, 0, [ + ..._times(longDuration), + _u32(timescale), + _duration(longDuration), + _u32(0x00010000), // rate 1.0 + _u16(0x0100), // volume 1.0 + Uint8List(10), // reserved + _identityMatrix(), + Uint8List(24), // pre_defined + _u32(2), // next_track_ID + ]); + + Uint8List _tkhd(bool longDuration) => + _fullBox('tkhd', longDuration ? 1 : 0, 0x000003, [ + // flags: track enabled | track in movie + ..._times(longDuration), + _u32(1), // track_ID + _u32(0), // reserved + _duration(longDuration), + Uint8List(8), // reserved + _u16(0), // layer + _u16(0), // alternate_group + _u16(0), // volume: 0 for video + _u16(0), // reserved + _identityMatrix(), + _u32(width * 0x10000), // 16.16 fixed point + _u32(height * 0x10000), + ]); + + Uint8List _mdhd(bool longDuration) => + _fullBox('mdhd', longDuration ? 1 : 0, 0, [ + ..._times(longDuration), + _u32(timescale), + _duration(longDuration), + _u16(0x55C4), // language 'und' + _u16(0), // pre_defined + ]); + + List _times(bool longDuration) => longDuration + ? [Uint8List(8), Uint8List(8)] // creation_time, modification_time + : [Uint8List(4), Uint8List(4)]; + + Uint8List _duration(bool longDuration) => + longDuration ? _u64(_totalDuration) : _u32(_totalDuration); + + Uint8List _stbl(Uint8List avcC, int chunkOffset) { + // Run-length encode the per-sample durations. + final sttsEntries = []; + var runCount = 0; + var runDelta = _durations.first; + for (final delta in _durations) { + if (delta == runDelta) { + runCount++; + continue; + } + sttsEntries + ..add(_u32(runCount)) + ..add(_u32(runDelta)); + runDelta = delta; + runCount = 1; + } + sttsEntries + ..add(_u32(runCount)) + ..add(_u32(runDelta)); + + final sizes = ByteData(_samples.length * 4); + for (var i = 0; i < _samples.length; i++) { + sizes.setUint32(i * 4, _samples[i].length); + } + + final allKeyFrames = _keyFrameNumbers.length == _samples.length; + return _box('stbl', [ + _fullBox('stsd', 0, 0, [_u32(1), _avc1(avcC)]), + _fullBox('stts', 0, 0, [_u32(sttsEntries.length ~/ 2), ...sttsEntries]), + // Without stss every sample is a sync sample. + if (!allKeyFrames) + _fullBox('stss', 0, 0, [ + _u32(_keyFrameNumbers.length), + for (final number in _keyFrameNumbers) _u32(number), + ]), + _fullBox('stsc', 0, 0, [ + _u32(1), + _u32(1), // first_chunk + _u32(_samples.length), // samples_per_chunk + _u32(1), // sample_description_index + ]), + _fullBox('stsz', 0, 0, [ + _u32(0), // sample_size: sizes vary + _u32(_samples.length), + sizes.buffer.asUint8List(), + ]), + // One chunk, and it starts right after ftyp and moov, so its offset + // always fits in 32 bits. + _fullBox('stco', 0, 0, [_u32(1), _u32(chunkOffset)]), + ]); + } + + Uint8List _avc1(Uint8List avcC) => _box('avc1', [ + Uint8List(6), // reserved + _u16(1), // data_reference_index + Uint8List(16), // pre_defined, reserved, pre_defined[3] + _u16(width), + _u16(height), + _u32(0x00480000), // 72 dpi + _u32(0x00480000), + _u32(0), // reserved + _u16(1), // frame_count + Uint8List(32), // compressorname + _u16(0x0018), // depth + _u16(0xFFFF), // pre_defined = -1 + _box('avcC', [avcC]), + ]); + + static Uint8List _identityMatrix() => Uint8List.fromList([ + ..._u32(0x00010000), ..._u32(0), ..._u32(0), // + ..._u32(0), ..._u32(0x00010000), ..._u32(0), // + ..._u32(0), ..._u32(0), ..._u32(0x40000000), // + ]); +} + +const int _maxUint32 = 0xFFFFFFFF; + +Uint8List _box(String type, List children) { + var size = 8; + for (final child in children) { + size += child.length; + } + final out = BytesBuilder(copy: false) + ..add(_u32(size)) + ..add(_fourCc(type)); + for (final child in children) { + out.add(child); + } + return out.takeBytes(); +} + +Uint8List _fullBox( + String type, + int version, + int flags, + List children, +) => + _box(type, [ + _u32(version * 0x1000000 + flags), + ...children, + ]); + +Uint8List _fourCc(String code) { + assert(code.length == 4); + return Uint8List.fromList(code.codeUnits); +} + +Uint8List _u16(int value) => + (ByteData(2)..setUint16(0, value)).buffer.asUint8List(); + +Uint8List _u32(int value) => + (ByteData(4)..setUint32(0, value)).buffer.asUint8List(); + +/// ByteData.setUint64 is unsupported when compiled to JavaScript. +Uint8List _u64(int value) => (ByteData(8) + ..setUint32(0, value ~/ 0x100000000) + ..setUint32(4, value % 0x100000000)) + .buffer + .asUint8List(); diff --git a/lib/services/video_export/video_export_errors.dart b/lib/services/video_export/video_export_errors.dart new file mode 100644 index 00000000..a23150db --- /dev/null +++ b/lib/services/video_export/video_export_errors.dart @@ -0,0 +1,11 @@ +/// Thrown when the user cancels a video export. +class VideoExportCancelled implements Exception {} + +/// A video export failure whose [message] can be shown to the user. +class VideoExportException implements Exception { + VideoExportException(this.message); + final String message; + + @override + String toString() => 'VideoExportException: $message'; +} diff --git a/lib/services/video_export/web_codecs_video_encoder.dart b/lib/services/video_export/web_codecs_video_encoder.dart new file mode 100644 index 00000000..24d80ead --- /dev/null +++ b/lib/services/video_export/web_codecs_video_encoder.dart @@ -0,0 +1,564 @@ +import 'dart:async'; +import 'dart:js_interop'; +import 'dart:js_interop_unsafe'; +import 'dart:math' as math; +import 'dart:typed_data'; + +import 'package:icarus/services/video_export/mp4_muxer.dart'; +import 'package:icarus/services/video_export/video_export_errors.dart'; + +/// Encodes rendered frames into an H.264 .mp4 in the browser, using WebCodecs +/// `VideoEncoder` and [Mp4H264Muxer]. The web counterpart of the desktop +/// ffmpeg pipeline, which browsers cannot run. +/// +/// The output is constant frame rate: each [addFrame] is repeated at `fps` +/// for its duration. A repeat reuses the frame's pixels, so a held page costs +/// one RGBA upload however long it is shown. +class WebCodecsMp4Encoder { + /// Keyframe spacing ceiling, so players can seek within long holds. + static const _maxKeyFrameIntervalSeconds = 2; + + /// Encoder queue depth at which [addFrame] waits for the encoder to catch + /// up, so a fast renderer cannot pile up frames in memory. + static const _maxEncodeQueueSize = 8; + + /// Whether this browser can encode H.264 at this size, frame rate, and + /// bitrate. False when WebCodecs is missing entirely. + static Future isSupported({ + required int width, + required int height, + required int fps, + int? bitrate, + }) async => + await _supportedConfig( + width: width, + height: height, + fps: fps, + bitrate: bitrate, + ) != + null; + + _VideoEncoder? _encoder; + _OffscreenCanvas? _scaleCanvas; + _Canvas2D? _scaleContext; + late int _inputWidth; + late int _inputHeight; + late int _outputWidth; + late int _outputHeight; + late int _fps; + + /// Frames handed to the encoder so far; frame i is shown at i / fps. + int _framesEncoded = 0; + int _framesSinceKeyFrame = 0; + + /// Sum of every [addFrame] duration, so rounding never accumulates. + int _plannedMicros = 0; + + final List<_EncodedSample> _samples = []; + Uint8List? _avcDecoderConfig; + Completer? _dequeued; + + /// Why the encoder failed, reported by the next call. + String? _failure; + bool _cancelled = false; + bool _finished = false; + + /// Configures the encoder. Frames arrive at [inputWidth] x [inputHeight] + /// and are scaled to [outputWidth] x [outputHeight] when those differ. + /// A null [bitrate] encodes at a high variable bitrate (see + /// [_defaultBitrate]); otherwise the bitrate is held constant. + Future start({ + required int inputWidth, + required int inputHeight, + required int outputWidth, + required int outputHeight, + required int fps, + int? bitrate, + }) async { + if (_encoder != null) throw StateError('The encoder is already started.'); + if (fps <= 0) throw ArgumentError.value(fps, 'fps', 'must be positive'); + _inputWidth = inputWidth; + _inputHeight = inputHeight; + _outputWidth = outputWidth; + _outputHeight = outputHeight; + _fps = fps; + + final config = await _supportedConfig( + width: outputWidth, + height: outputHeight, + fps: fps, + bitrate: bitrate, + ); + if (_cancelled) throw VideoExportCancelled(); + if (config == null) { + throw VideoExportException( + 'This browser cannot encode H.264 video at ' + '${outputWidth}x$outputHeight, $fps fps.', + ); + } + + if (outputWidth != inputWidth || outputHeight != inputHeight) { + final canvas = _OffscreenCanvas(outputWidth, outputHeight); + final context = canvas.getContext('2d'); + if (context == null) { + throw VideoExportException('This browser cannot scale video frames.'); + } + context.imageSmoothingQuality = 'high'; + _scaleCanvas = canvas; + _scaleContext = context; + } + + final encoder = _VideoEncoder( + _VideoEncoderInit( + output: _onChunk.toJS, + error: _onError.toJS, + ), + ); + encoder.ondequeue = ((JSAny? _) => _wakeWaiter()).toJS; + _encoder = encoder; + try { + encoder.configure(config); + } on Object catch (error) { + _closeEncoder(); + throw VideoExportException('Could not start the video encoder: $error'); + } + } + + /// Adds one rendered frame (top-to-bottom RGBA, inputWidth * inputHeight * 4 + /// bytes) shown for [duration]. + /// + /// The frame is repeated for round(duration * fps) ticks, at least one, + /// measured against the running total so remainders carry forward and the + /// video's length matches the sum of the durations. + Future addFrame(Uint8List rgba, {required Duration duration}) async { + final encoder = _checkRunning(); + if (rgba.length != _inputWidth * _inputHeight * 4) { + throw ArgumentError( + 'Expected ${_inputWidth * _inputHeight * 4} RGBA bytes, ' + 'got ${rgba.length}.', + ); + } + if (duration.isNegative) { + throw ArgumentError.value(duration, 'duration', 'must not be negative'); + } + _plannedMicros += duration.inMicroseconds; + final plannedFrames = + (_plannedMicros * _fps / Duration.microsecondsPerSecond).round(); + final repeats = math.max(1, plannedFrames - _framesEncoded); + + final base = _baseFrame(rgba); + try { + for (var i = 0; i < repeats; i++) { + while (encoder.encodeQueueSize > _maxEncodeQueueSize) { + final dequeued = _dequeued ??= Completer(); + await dequeued.future; + _checkRunning(); + } + final index = _framesEncoded; + final timestamp = _timestampOf(index); + final frame = _VideoFrame( + base, + _VideoFrameInit( + timestamp: timestamp, + duration: _timestampOf(index + 1) - timestamp, + ), + ); + // A held page (a run longer than one frame) opens on a keyframe so it + // starts crisp and seeks cleanly. Single-frame runs are transition + // frames; keying each of them would spend the bitrate on I-frames. + final keyFrame = index == 0 || + (i == 0 && repeats > 1) || + _framesSinceKeyFrame >= _maxKeyFrameIntervalSeconds * _fps; + try { + encoder.encode(frame, _EncodeOptions(keyFrame: keyFrame)); + } on Object catch (error) { + _failure ??= '$error'; + _checkRunning(); + } finally { + frame.close(); + } + _framesEncoded++; + _framesSinceKeyFrame = keyFrame ? 1 : _framesSinceKeyFrame + 1; + } + } finally { + base.close(); + } + _checkRunning(); + } + + /// Flushes the encoder and returns the finished .mp4 bytes. + Future finish() async { + final encoder = _checkRunning(); + try { + await encoder.flush().toDart; + } on Object catch (error) { + _failure ??= '$error'; + } + _checkRunning(); + _finished = true; + _closeEncoder(); + + final config = _avcDecoderConfig; + if (_samples.isEmpty || config == null) { + throw VideoExportException('The video encoder produced no video.'); + } + final ticksPerFrame = _timescale ~/ _fps; + final muxer = Mp4H264Muxer( + width: _outputWidth, + height: _outputHeight, + timescale: _timescale, + ); + for (var i = 0; i < _samples.length; i++) { + final sample = _samples[i]; + final nextFrame = + i + 1 < _samples.length ? _samples[i + 1].frame : _framesEncoded; + // The MP4 has no composition offsets, so chunks must arrive in + // presentation order. Encoders that reorder (B-frames) fail loudly + // rather than produce a video that plays frames out of order. + if (nextFrame <= sample.frame) { + throw VideoExportException( + 'The video encoder returned frames out of order.', + ); + } + muxer.addSample( + sample.bytes, + duration: (nextFrame - sample.frame) * ticksPerFrame, + isKeyFrame: sample.isKeyFrame, + ); + } + return muxer.finish(avcDecoderConfig: config); + } + + /// Stops encoding and releases the encoder. Safe to call at any time and + /// more than once; later [addFrame] and [finish] calls throw + /// [VideoExportCancelled]. + void cancel() { + _cancelled = true; + _closeEncoder(); + _wakeWaiter(); + } + + /// Media ticks per second. 1000 ticks per frame keeps every frame duration + /// an exact integer. + int get _timescale => _fps * 1000; + + int _timestampOf(int frame) => + (frame * Duration.microsecondsPerSecond / _fps).round(); + + _VideoFrame _baseFrame(Uint8List rgba) { + final raw = _VideoFrame( + rgba.toJS, + _VideoFrameBufferInit( + format: 'RGBA', + codedWidth: _inputWidth, + codedHeight: _inputHeight, + timestamp: 0, + ), + ); + final context = _scaleContext; + if (context == null) return raw; + try { + context.drawImage(raw, 0, 0, _outputWidth, _outputHeight); + } finally { + raw.close(); + } + return _VideoFrame(_scaleCanvas!, _VideoFrameInit(timestamp: 0)); + } + + _VideoEncoder _checkRunning() { + if (_cancelled) throw VideoExportCancelled(); + final failure = _failure; + if (failure != null) { + _closeEncoder(); + throw VideoExportException('The video encoder failed: $failure'); + } + final encoder = _encoder; + if (encoder == null || _finished) { + throw StateError( + _finished ? 'The video is already finished.' : 'Call start() first.', + ); + } + return encoder; + } + + void _onChunk(_EncodedVideoChunk chunk, _ChunkMetadata? metadata) { + try { + final description = metadata?.decoderConfig?.description; + if (description != null) { + final config = _copyBufferSource(description); + final previous = _avcDecoderConfig; + if (previous != null && !_sameBytes(previous, config)) { + // The MP4 carries one avcC, which every sample must decode with. + throw StateError('The encoder changed its H.264 parameter sets.'); + } + _avcDecoderConfig = config; + } + final bytes = _Uint8Array(chunk.byteLength); + chunk.copyTo(bytes); + _samples.add( + _EncodedSample( + frame: + (chunk.timestamp * _fps / Duration.microsecondsPerSecond).round(), + bytes: bytes.toDart, + isKeyFrame: chunk.type == 'key', + ), + ); + } on Object catch (error) { + _failure ??= '$error'; + _closeEncoder(); + _wakeWaiter(); + } + } + + void _onError(JSAny? error) { + _failure ??= _jsString(error); + _wakeWaiter(); + } + + void _wakeWaiter() { + final waiter = _dequeued; + _dequeued = null; + if (waiter != null && !waiter.isCompleted) waiter.complete(); + } + + void _closeEncoder() { + final encoder = _encoder; + if (encoder == null || encoder.state == 'closed') return; + try { + encoder.close(); + } on Object { + // Already closed by an encoder error. + } + } + + /// Picks a bitrate for [bitrate] == null: 8 Mbps at 1080p30, the desktop + /// Max preset's H.264 rate, scaled by pixel count. 60 fps gets 1.5x rather + /// than 2x: consecutive frames differ less, and held pages cost almost + /// nothing either way. + static int _defaultBitrate(int width, int height, int fps) { + final pixelScale = width * height / (1920 * 1080); + final rateScale = fps > 30 ? 1.5 : 1.0; + return math.max(1000000, (8000000 * pixelScale * rateScale).round()); + } + + /// The first H.264 profile this browser accepts at this size, or null. + /// High needs a hardware encoder in Chrome; its software encoder (OpenH264) + /// only does Constrained Baseline, so fall back through Main to that. + static Future<_VideoEncoderConfig?> _supportedConfig({ + required int width, + required int height, + required int fps, + int? bitrate, + }) async { + if (!globalContext.has('VideoEncoder')) return null; + final targetBitrate = bitrate ?? _defaultBitrate(width, height, fps); + final level = _h264Level(width, height, fps, targetBitrate); + if (level == null) return null; + for (final profile in _h264Profiles) { + final config = _VideoEncoderConfig( + codec: 'avc1.$profile${level.toRadixString(16).padLeft(2, '0')}', + width: width, + height: height, + bitrate: targetBitrate, + bitrateMode: bitrate != null ? 'constant' : 'variable', + framerate: fps, + avc: _AvcEncoderConfig(format: 'avc'), + latencyMode: 'quality', + ); + try { + final support = await _VideoEncoder.isConfigSupported(config).toDart; + if (support.supported ?? false) return config; + } on Object { + // A rejected (TypeError) config means this profile is not usable. + } + } + return null; + } + + /// profile_idc and constraint flags, in preference order: High, Main, + /// Constrained Baseline. + static const _h264Profiles = ['6400', '4d00', '42e0']; + + /// The lowest H.264 level whose frame size, macroblock rate, and bitrate + /// limits hold this stream, or null when none does. + static int? _h264Level(int width, int height, int fps, int bitrate) { + final macroblocks = ((width + 15) ~/ 16) * ((height + 15) ~/ 16); + final macroblocksPerSecond = macroblocks * fps; + // (level_idc, MaxFS, MaxMBPS, MaxBR in kbit/s for Baseline/Main; High + // allows 1.25x, so these limits hold for every profile above.) + const levels = [ + (30, 1620, 40500, 10000), + (31, 3600, 108000, 14000), + (32, 5120, 216000, 20000), + (40, 8192, 245760, 20000), + (41, 8192, 245760, 50000), + (42, 8704, 522240, 50000), + (50, 22080, 589824, 135000), + (51, 36864, 983040, 240000), + (52, 36864, 2073600, 240000), + ]; + for (final (idc, maxFs, maxMbps, maxBrKbps) in levels) { + if (macroblocks <= maxFs && + macroblocksPerSecond <= maxMbps && + bitrate <= maxBrKbps * 1000) { + return idc; + } + } + return null; + } +} + +class _EncodedSample { + _EncodedSample({ + required this.frame, + required this.bytes, + required this.isKeyFrame, + }); + + /// Index of the frame this chunk encodes, from its timestamp. + final int frame; + final Uint8List bytes; + final bool isKeyFrame; +} + +bool _sameBytes(Uint8List a, Uint8List b) { + if (a.length != b.length) return false; + for (var i = 0; i < a.length; i++) { + if (a[i] != b[i]) return false; + } + return true; +} + +/// Copies a WebCodecs BufferSource (an ArrayBuffer or a view of one). +Uint8List _copyBufferSource(JSObject source) { + final bytes = source.isA() + ? _Uint8Array.view(source, 0, (source as _ArrayBuffer).byteLength) + : _Uint8Array.view( + (source as _ArrayBufferView).buffer, + source.byteOffset, + source.byteLength, + ); + return Uint8List.fromList(bytes.toDart); +} + +@JS('String') +external String _jsString(JSAny? value); + +@JS('VideoEncoder') +extension type _VideoEncoder._(JSObject _) implements JSObject { + external factory _VideoEncoder(_VideoEncoderInit init); + + external static JSPromise<_VideoEncoderSupport> isConfigSupported( + _VideoEncoderConfig config, + ); + + external String get state; + external int get encodeQueueSize; + external set ondequeue(JSFunction? handler); + + external void configure(_VideoEncoderConfig config); + external void encode(_VideoFrame frame, _EncodeOptions options); + external JSPromise flush(); + external void close(); +} + +extension type _VideoEncoderInit._(JSObject _) implements JSObject { + external factory _VideoEncoderInit({JSFunction output, JSFunction error}); +} + +extension type _VideoEncoderConfig._(JSObject _) implements JSObject { + external factory _VideoEncoderConfig({ + String codec, + int width, + int height, + int bitrate, + String bitrateMode, + num framerate, + _AvcEncoderConfig avc, + String latencyMode, + }); +} + +extension type _AvcEncoderConfig._(JSObject _) implements JSObject { + external factory _AvcEncoderConfig({String format}); +} + +extension type _VideoEncoderSupport._(JSObject _) implements JSObject { + external bool? get supported; +} + +extension type _EncodeOptions._(JSObject _) implements JSObject { + external factory _EncodeOptions({bool keyFrame}); +} + +@JS('VideoFrame') +extension type _VideoFrame._(JSObject _) implements JSObject { + /// [source] is a BufferSource with a [_VideoFrameBufferInit], or an image + /// source (VideoFrame, OffscreenCanvas) with a [_VideoFrameInit]. + external factory _VideoFrame(JSObject source, JSObject init); + + external void close(); +} + +extension type _VideoFrameBufferInit._(JSObject _) implements JSObject { + external factory _VideoFrameBufferInit({ + String format, + int codedWidth, + int codedHeight, + int timestamp, + }); +} + +extension type _VideoFrameInit._(JSObject _) implements JSObject { + external factory _VideoFrameInit({int timestamp, int duration}); +} + +extension type _EncodedVideoChunk._(JSObject _) implements JSObject { + external String get type; + external int get timestamp; + external int get byteLength; + external void copyTo(_Uint8Array destination); +} + +extension type _ChunkMetadata._(JSObject _) implements JSObject { + external _DecoderConfig? get decoderConfig; +} + +extension type _DecoderConfig._(JSObject _) implements JSObject { + external JSObject? get description; +} + +@JS('OffscreenCanvas') +extension type _OffscreenCanvas._(JSObject _) implements JSObject { + external factory _OffscreenCanvas(int width, int height); + + external _Canvas2D? getContext(String contextId); +} + +extension type _Canvas2D._(JSObject _) implements JSObject { + external set imageSmoothingQuality(String value); + + external void drawImage( + JSObject image, + num dx, + num dy, + num dWidth, + num dHeight, + ); +} + +extension type _ArrayBuffer._(JSObject _) implements JSObject { + external int get byteLength; +} + +extension type _ArrayBufferView._(JSObject _) implements JSObject { + external JSObject get buffer; + external int get byteOffset; + external int get byteLength; +} + +@JS('Uint8Array') +extension type _Uint8Array._(JSUint8Array _) implements JSUint8Array { + external factory _Uint8Array(int length); + external factory _Uint8Array.view(JSObject buffer, int offset, int length); +} diff --git a/test/mp4_box_test_support.dart b/test/mp4_box_test_support.dart new file mode 100644 index 00000000..35afcedc --- /dev/null +++ b/test/mp4_box_test_support.dart @@ -0,0 +1,169 @@ +import 'dart:typed_data'; + +/// One parsed ISO-BMFF box. +class Mp4Box { + Mp4Box(this.type, this.start, this.size, this.headerSize); + + final String type; + final int start; + final int size; + final int headerSize; + + int get end => start + size; + + Uint8List payload(Uint8List file) => + file.sublist(start + headerSize, start + size); +} + +/// Reads back the boxes an MP4 writer produced, for assertions in tests. +/// Pure Dart, so the VM and browser tests share it. +class Mp4Boxes { + Mp4Boxes(this.file) : _data = ByteData.sublistView(file); + + final Uint8List file; + final ByteData _data; + + static const _containers = {'moov', 'trak', 'mdia', 'minf', 'dinf', 'stbl'}; + + List get topLevelTypes => + [for (final box in _parse(0, file.length)) box.type]; + + List _parse(int start, int end) { + final boxes = []; + var offset = start; + while (offset < end) { + var size = _data.getUint32(offset); + final type = String.fromCharCodes(file.sublist(offset + 4, offset + 8)); + var headerSize = 8; + if (size == 1) { + size = _data.getUint32(offset + 8) * 0x100000000 + + _data.getUint32(offset + 12); + headerSize = 16; + } + if (size < headerSize || offset + size > end) { + throw FormatException('Bad $type box size $size at $offset'); + } + boxes.add(Mp4Box(type, offset, size, headerSize)); + offset += size; + } + return boxes; + } + + /// Follows [path] from the top level, e.g. `['moov', 'trak', 'tkhd']`. + Mp4Box find(List path) { + var boxes = _parse(0, file.length); + late Mp4Box found; + for (var i = 0; i < path.length; i++) { + found = boxes.firstWhere( + (box) => box.type == path[i], + orElse: () => throw StateError('No ${path.take(i + 1).join('/')}'), + ); + if (i + 1 < path.length) { + if (!_containers.contains(found.type)) { + throw StateError('${found.type} is not a container'); + } + boxes = _parse(found.start + found.headerSize, found.end); + } + } + return found; + } + + Mp4Box? _maybeFind(List path) { + try { + return find(path); + } on StateError { + return null; + } + } + + static const _stbl = ['moov', 'trak', 'mdia', 'minf', 'stbl']; + + /// Box payload offset past the version/flags word of a full box. + int _fullBoxBody(Mp4Box box) => box.start + box.headerSize + 4; + + List _u32Table(Mp4Box box, {required int skip, int stride = 1}) { + final body = _fullBoxBody(box); + final count = _data.getUint32(body + skip * 4); + final first = body + (skip + 1) * 4; + return [ + for (var i = 0; i < count * stride; i++) _data.getUint32(first + i * 4), + ]; + } + + List chunkOffsets() => _u32Table(find([..._stbl, 'stco']), skip: 0); + + List sampleSizes() { + final stsz = find([..._stbl, 'stsz']); + final fixed = _data.getUint32(_fullBoxBody(stsz)); + if (fixed != 0) throw UnimplementedError('Fixed-size samples'); + return _u32Table(stsz, skip: 1); + } + + /// `[sample_count, sample_delta]` runs. + List> stts() { + final flat = _u32Table(find([..._stbl, 'stts']), skip: 0, stride: 2); + return [ + for (var i = 0; i < flat.length; i += 2) [flat[i], flat[i + 1]], + ]; + } + + List sampleDurations() => [ + for (final run in stts()) + for (var i = 0; i < run[0]; i++) run[1], + ]; + + /// 1-based keyframe numbers, or null when stss is absent (all sync). + List? syncSamples() { + final stss = _maybeFind([..._stbl, 'stss']); + return stss == null ? null : _u32Table(stss, skip: 0); + } + + int samplesPerChunk() { + final stsc = find([..._stbl, 'stsc']); + final body = _fullBoxBody(stsc); + if (_data.getUint32(body) != 1) throw UnimplementedError('Several runs'); + return _data.getUint32(body + 8); + } + + (int, int) _timescaleAndDuration(List path) { + final box = find(path); + final body = _fullBoxBody(box); + final version = file[box.start + box.headerSize]; + if (version == 1) { + return ( + _data.getUint32(body + 16), + _data.getUint32(body + 20) * 0x100000000 + _data.getUint32(body + 24), + ); + } + return (_data.getUint32(body + 8), _data.getUint32(body + 12)); + } + + int mediaTimescale() => + _timescaleAndDuration(['moov', 'trak', 'mdia', 'mdhd']).$1; + + int mediaDuration() => + _timescaleAndDuration(['moov', 'trak', 'mdia', 'mdhd']).$2; + + int movieDuration() => _timescaleAndDuration(['moov', 'mvhd']).$2; + + /// tkhd width and height, whole pixels of the 16.16 values. + (int, int) trackSize() { + final tkhd = find(['moov', 'trak', 'tkhd']); + return ( + _data.getUint32(tkhd.end - 8) ~/ 0x10000, + _data.getUint32(tkhd.end - 4) ~/ 0x10000, + ); + } + + /// The single entry inside stsd. + Mp4Box sampleEntry() { + final stsd = find([..._stbl, 'stsd']); + final entries = _parse(_fullBoxBody(stsd) + 4, stsd.end); + return entries.single; + } + + /// Child boxes of a box whose own fields take [entryHeaderSize] bytes after + /// its header (78 for a visual sample entry). + List childrenOf(Mp4Box box, {required int entryHeaderSize}) => + _parse(box.start + box.headerSize + entryHeaderSize, box.end); +} diff --git a/test/mp4_muxer_test.dart b/test/mp4_muxer_test.dart new file mode 100644 index 00000000..66e8d4bd --- /dev/null +++ b/test/mp4_muxer_test.dart @@ -0,0 +1,298 @@ +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:icarus/services/video_export/mp4_muxer.dart'; + +import 'mp4_box_test_support.dart'; + +void main() { + group('Mp4H264Muxer box structure', () { + // Fake AVCC samples: the muxer never looks inside them. + Uint8List sample(int seed, int length) => Uint8List.fromList( + List.generate(length, (i) => (seed * 31 + i) & 0xFF)); + final avcC = Uint8List.fromList( + [1, 0x64, 0x00, 0x28, 0xFF, 0xE1, 0, 2, 0x67, 0x64, 1, 0, 2, 0x68, 0xEE], + ); + + test('lays out ftyp, moov, mdat with the chunk offset on sample one', () { + final muxer = Mp4H264Muxer(width: 1920, height: 1080); + final samples = [ + sample(1, 100), + sample(2, 37), + sample(3, 51), + sample(4, 9), + sample(5, 64), + ]; + final durations = [270000, 3000, 3000, 1500, 3000]; + final keys = [true, false, false, true, false]; + for (var i = 0; i < samples.length; i++) { + muxer.addSample(samples[i], + duration: durations[i], isKeyFrame: keys[i]); + } + final file = muxer.finish(avcDecoderConfig: avcC); + final mp4 = Mp4Boxes(file); + + expect(mp4.topLevelTypes, ['ftyp', 'moov', 'mdat']); + + final ftyp = mp4.find(['ftyp']); + expect(String.fromCharCodes(ftyp.payload(file).sublist(0, 4)), 'isom'); + + final offsets = mp4.chunkOffsets(); + expect(offsets, hasLength(1)); + final mdat = mp4.find(['mdat']); + expect(offsets.single, mdat.start + mdat.headerSize); + var cursor = offsets.single; + final sizes = mp4.sampleSizes(); + expect(sizes, [for (final s in samples) s.length]); + for (var i = 0; i < samples.length; i++) { + expect(file.sublist(cursor, cursor + sizes[i]), samples[i]); + cursor += sizes[i]; + } + expect(cursor, file.length); + + expect(mp4.sampleDurations(), durations); + expect(mp4.stts(), [ + [1, 270000], + [2, 3000], + [1, 1500], + [1, 3000], + ]); + expect(mp4.syncSamples(), [1, 4]); + expect(mp4.mediaTimescale(), 90000); + expect(mp4.mediaDuration(), 280500); + expect(mp4.movieDuration(), 280500); + expect(mp4.trackSize(), (1920, 1080)); + expect(mp4.samplesPerChunk(), samples.length); + expect( + mp4.find(['moov', 'trak', 'mdia', 'hdlr']).payload(file).sublist(8, 12), + 'vide'.codeUnits, + ); + + final avc1 = mp4.sampleEntry(); + expect(avc1.type, 'avc1'); + final avcCBox = mp4.childrenOf(avc1, entryHeaderSize: 78).single; + expect(avcCBox.type, 'avcC'); + expect(avcCBox.payload(file), avcC); + }); + + test('omits stss when every sample is a keyframe', () { + final muxer = Mp4H264Muxer(width: 640, height: 360, timescale: 30000) + ..addSample(sample(1, 10), duration: 1000, isKeyFrame: true) + ..addSample(sample(2, 10), duration: 1000, isKeyFrame: true); + final mp4 = Mp4Boxes(muxer.finish(avcDecoderConfig: avcC)); + expect(mp4.syncSamples(), isNull); + expect(mp4.stts(), [ + [2, 1000], + ]); + }); + + test('refuses an empty file and a non-keyframe start', () { + expect( + () => Mp4H264Muxer(width: 640, height: 360) + .finish(avcDecoderConfig: avcC), + throwsStateError, + ); + expect( + () => Mp4H264Muxer(width: 640, height: 360) + .addSample(sample(1, 10), duration: 1, isKeyFrame: false), + throwsStateError, + ); + expect( + () => Mp4H264Muxer(width: 640, height: 360) + .addSample(sample(1, 10), duration: 0, isKeyFrame: true), + throwsArgumentError, + ); + }); + }); + + group('Mp4H264Muxer with a real H.264 stream', () { + final ffmpegMissing = _ffmpegMissingReason(); + + test( + 'produces a file ffprobe and ffmpeg read back exactly', + () async { + final dir = await Directory.systemTemp.createTemp('icarus_mp4_muxer_'); + addTearDown(() => dir.delete(recursive: true)); + final annexBPath = '${dir.path}${Platform.pathSeparator}in.h264'; + final outPath = '${dir.path}${Platform.pathSeparator}out.mp4'; + + // No B-frames (the muxer writes no composition offsets), an access + // unit delimiter before every frame so the stream splits cleanly, and + // an IDR every 5 frames so stss has several entries. + final encode = await Process.run('ffmpeg', [ + '-v', 'error', '-y', // + '-f', 'lavfi', '-i', 'testsrc=size=320x240:rate=10', + '-frames:v', '20', + '-c:v', 'libx264', '-pix_fmt', 'yuv420p', + '-x264-params', 'aud=1:bframes=0:keyint=5:min-keyint=5:scenecut=0', + '-bsf:v', 'h264_mp4toannexb', + '-f', 'h264', annexBPath, + ]); + expect(encode.exitCode, 0, reason: '${encode.stderr}'); + + final stream = _AnnexBStream.parse( + await File(annexBPath).readAsBytes(), + ); + expect(stream.accessUnits, hasLength(20)); + + final muxer = Mp4H264Muxer(width: 320, height: 240); + // First frame held for 3 s, the rest at 1/10 s: 3 + 19 * 0.1 = 4.9 s. + for (var i = 0; i < stream.accessUnits.length; i++) { + final unit = stream.accessUnits[i]; + muxer.addSample( + unit.avcc, + duration: i == 0 ? 270000 : 9000, + isKeyFrame: unit.isKeyFrame, + ); + } + final bytes = muxer.finish(avcDecoderConfig: stream.avcDecoderConfig); + await File(outPath).writeAsBytes(bytes); + + expect(Mp4Boxes(bytes).syncSamples(), [1, 6, 11, 16]); + + final probe = await Process.run('ffprobe', [ + '-v', 'error', // + '-count_frames', + '-show_streams', '-show_format', + outPath, + ]); + expect(probe.exitCode, 0, reason: '${probe.stderr}'); + final fields = _probeFields(probe.stdout as String); + expect(fields['codec_name'], 'h264'); + expect(fields['width'], '320'); + expect(fields['height'], '240'); + expect(fields['nb_read_frames'], '20'); + expect(fields['nb_frames'], '20'); + expect(double.parse(fields['format.duration']!), closeTo(4.9, 0.001)); + expect(double.parse(fields['stream.duration']!), closeTo(4.9, 0.001)); + + final decode = await Process.run('ffmpeg', [ + '-v', 'error', '-i', outPath, '-f', 'null', '-', // + ]); + expect(decode.exitCode, 0); + expect((decode.stderr as String).trim(), isEmpty); + }, + skip: ffmpegMissing, + timeout: const Timeout(Duration(minutes: 2)), + ); + }); +} + +/// Null when ffmpeg and ffprobe both run, otherwise why the test is skipped. +String? _ffmpegMissingReason() { + for (final tool in ['ffmpeg', 'ffprobe']) { + try { + final result = Process.runSync(tool, ['-version']); + if (result.exitCode != 0) return '$tool -version failed'; + } on ProcessException { + return '$tool is not on PATH'; + } + } + return null; +} + +/// Parses ffprobe's default output into `key` (stream fields) plus +/// `stream.duration` / `format.duration`, which share a name. +Map _probeFields(String output) { + final fields = {}; + var section = ''; + for (final raw in output.split(RegExp(r'\r?\n'))) { + final line = raw.trim(); + if (line == '[STREAM]') section = 'stream'; + if (line == '[FORMAT]') section = 'format'; + final eq = line.indexOf('='); + if (eq <= 0) continue; + final key = line.substring(0, eq); + final value = line.substring(eq + 1); + if (key == 'duration') { + fields['$section.duration'] = value; + } else { + fields.putIfAbsent(key, () => value); + } + } + return fields; +} + +class _AccessUnit { + _AccessUnit(this.avcc, this.isKeyFrame); + final Uint8List avcc; + final bool isKeyFrame; +} + +/// An Annex-B H.264 stream split on its access unit delimiters. +class _AnnexBStream { + _AnnexBStream(this.accessUnits, this.avcDecoderConfig); + + final List<_AccessUnit> accessUnits; + final Uint8List avcDecoderConfig; + + static _AnnexBStream parse(Uint8List bytes) { + final nals = _splitNalUnits(bytes); + Uint8List? sps; + Uint8List? pps; + final units = <_AccessUnit>[]; + List? current; + var currentIsKey = false; + + void close() { + final nalsInUnit = current; + if (nalsInUnit == null || nalsInUnit.isEmpty) return; + final out = BytesBuilder(); + for (final nal in nalsInUnit) { + out + ..add((ByteData(4)..setUint32(0, nal.length)).buffer.asUint8List()) + ..add(nal); + } + units.add(_AccessUnit(out.takeBytes(), currentIsKey)); + } + + for (final nal in nals) { + final type = nal[0] & 0x1F; + switch (type) { + case 9: // access unit delimiter + close(); + current = []; + currentIsKey = false; + case 7: + sps ??= nal; + case 8: + pps ??= nal; + default: + if (type == 5) currentIsKey = true; + (current ??= []).add(nal); + } + } + close(); + + final s = sps!; + final p = pps!; + final config = BytesBuilder() + ..add([1, s[1], s[2], s[3], 0xFF, 0xE1]) + ..add([s.length >> 8, s.length & 0xFF]) + ..add(s) + ..add([1, p.length >> 8, p.length & 0xFF]) + ..add(p); + return _AnnexBStream(units, config.takeBytes()); + } + + static List _splitNalUnits(Uint8List bytes) { + final starts = []; // index of the first byte after each start code + for (var i = 0; i + 2 < bytes.length; i++) { + if (bytes[i] == 0 && bytes[i + 1] == 0 && bytes[i + 2] == 1) { + starts.add(i + 3); + i += 2; + } + } + final nals = []; + for (var n = 0; n < starts.length; n++) { + var end = n + 1 < starts.length ? starts[n + 1] - 3 : bytes.length; + // A 4-byte start code leaves a zero byte on the previous NAL. + while (end > starts[n] && bytes[end - 1] == 0) { + end--; + } + nals.add(Uint8List.sublistView(bytes, starts[n], end)); + } + return nals; + } +} diff --git a/test/web/web_codecs_video_encoder_test.dart b/test/web/web_codecs_video_encoder_test.dart new file mode 100644 index 00000000..16df4458 --- /dev/null +++ b/test/web/web_codecs_video_encoder_test.dart @@ -0,0 +1,290 @@ +@TestOn('browser') +library; + +import 'dart:async'; +import 'dart:js_interop'; +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:icarus/services/video_export/video_export_errors.dart'; +import 'package:icarus/services/video_export/web_codecs_video_encoder.dart'; + +import '../mp4_box_test_support.dart'; + +// Run with: flutter test --platform chrome test/web/web_codecs_video_encoder_test.dart +// Needs a Chrome or Edge build with H.264 (Chromium builds lack it); point +// CHROME_EXECUTABLE at it when Chrome is not installed. + +void main() { + test('Chrome can encode H.264 at every export size', () async { + for (final (width, height, fps) in [ + (1920, 1080, 60), + (1920, 1080, 30), + (1280, 720, 30), + (640, 360, 30), + ]) { + expect( + await WebCodecsMp4Encoder.isSupported( + width: width, + height: height, + fps: fps, + ), + isTrue, + reason: '${width}x$height@$fps', + ); + } + expect( + await WebCodecsMp4Encoder.isSupported( + width: 1920, + height: 1080, + fps: 30, + bitrate: 2000000, + ), + isTrue, + ); + }); + + test('encodes held frames into an MP4 the browser plays back', () async { + final encoder = WebCodecsMp4Encoder(); + await encoder.start( + inputWidth: 640, + inputHeight: 360, + outputWidth: 640, + outputHeight: 360, + fps: 30, + ); + await encoder.addFrame( + _solid(640, 360, 255, 0, 0), + duration: const Duration(seconds: 1), + ); + await encoder.addFrame( + _solid(640, 360, 0, 255, 0), + duration: const Duration(milliseconds: 500), + ); + await encoder.addFrame( + _solid(640, 360, 0, 0, 255), + duration: const Duration(seconds: 1), + ); + final mp4 = await encoder.finish(); + + final boxes = Mp4Boxes(mp4); + expect(boxes.topLevelTypes, ['ftyp', 'moov', 'mdat']); + expect(boxes.sampleSizes(), hasLength(75)); + expect(boxes.mediaTimescale(), 30000); + expect(boxes.mediaDuration(), 75000); + expect(boxes.sampleDurations().toSet(), {1000}); + // Each held page opens on a keyframe. + expect(boxes.syncSamples() ?? [for (var i = 1; i <= 75; i++) i], + containsAll([1, 31, 46])); + expect(boxes.trackSize(), (640, 360)); + + final video = await _loadVideo(mp4); + expect(video.videoWidth, 640); + expect(video.videoHeight, 360); + expect(video.duration, closeTo(2.5, 0.05)); + expect(await _colourAt(video, 0.5), 'red'); + expect(await _colourAt(video, 1.25), 'green'); + expect(await _colourAt(video, 2.2), 'blue'); + }); + + test('scales 1280x720 frames down to a 640x360 video', () async { + final encoder = WebCodecsMp4Encoder(); + await encoder.start( + inputWidth: 1280, + inputHeight: 720, + outputWidth: 640, + outputHeight: 360, + fps: 30, + bitrate: 1000000, + ); + await encoder.addFrame( + _solid(1280, 720, 0, 255, 0), + duration: const Duration(milliseconds: 500), + ); + await encoder.addFrame( + _solid(1280, 720, 0, 0, 255), + duration: const Duration(milliseconds: 500), + ); + final mp4 = await encoder.finish(); + + final boxes = Mp4Boxes(mp4); + expect(boxes.sampleSizes(), hasLength(30)); + expect(boxes.trackSize(), (640, 360)); + + final video = await _loadVideo(mp4); + expect(video.videoWidth, 640); + expect(video.videoHeight, 360); + expect(video.duration, closeTo(1.0, 0.05)); + expect(await _colourAt(video, 0.25), 'green'); + expect(await _colourAt(video, 0.75), 'blue'); + }); + + test('carries fractional frame remainders forward', () async { + final encoder = WebCodecsMp4Encoder(); + await encoder.start( + inputWidth: 64, + inputHeight: 64, + outputWidth: 64, + outputHeight: 64, + fps: 30, + ); + // 50 ms is 1.5 frames at 30 fps: the runs are 2, 1, 2 frames so the + // total stays round(150 ms * 30) = 5 frames (rounded half up). + for (var i = 0; i < 3; i++) { + await encoder.addFrame( + _solid(64, 64, 80 * i, 0, 0), + duration: const Duration(milliseconds: 50), + ); + } + final boxes = Mp4Boxes(await encoder.finish()); + expect(boxes.sampleSizes(), hasLength(5)); + expect(boxes.mediaDuration(), 5000); + }); + + test('cancel is idempotent and later calls report the cancel', () async { + final encoder = WebCodecsMp4Encoder(); + await encoder.start( + inputWidth: 64, + inputHeight: 64, + outputWidth: 64, + outputHeight: 64, + fps: 30, + ); + encoder + ..cancel() + ..cancel(); + await expectLater( + encoder.addFrame(_solid(64, 64, 0, 0, 0), duration: Duration.zero), + throwsA(isA()), + ); + }); +} + +Uint8List _solid(int width, int height, int r, int g, int b) { + final bytes = Uint8List(width * height * 4); + for (var i = 0; i < bytes.length; i += 4) { + bytes[i] = r; + bytes[i + 1] = g; + bytes[i + 2] = b; + bytes[i + 3] = 255; + } + return bytes; +} + +Future<_VideoElement> _loadVideo(Uint8List mp4) async { + final url = _Url.createObjectURL( + _Blob([mp4.toJS].toJS, _BlobOptions(type: 'video/mp4')), + ); + addTearDown(() => _Url.revokeObjectURL(url)); + final video = _document.createElement('video') as _VideoElement + ..muted = true + ..preload = 'auto'; + final loaded = _nextEvent(video, 'loadeddata'); + video.src = url; + await loaded; + return video; +} + +/// Seeks [video] to [seconds] and names the dominant colour at its centre. +Future _colourAt(_VideoElement video, double seconds) async { + final seeked = _nextEvent(video, 'seeked'); + video.currentTime = seconds; + await seeked; + final canvas = _document.createElement('canvas') as _Canvas + ..width = video.videoWidth + ..height = video.videoHeight; + final context = canvas.getContext('2d'); + context.drawImage(video, 0, 0); + final pixel = context + .getImageData(video.videoWidth ~/ 2, video.videoHeight ~/ 2, 1, 1) + .data + .toDart; + final (r, g, b) = (pixel[0], pixel[1], pixel[2]); + if (r > 180 && g < 80 && b < 80) return 'red'; + if (g > 180 && r < 80 && b < 80) return 'green'; + if (b > 180 && r < 80 && g < 80) return 'blue'; + return 'rgb($r, $g, $b)'; +} + +/// Completes on [video]'s next [type] event, or fails on its error event. +Future _nextEvent(_VideoElement video, String type) { + final done = Completer(); + late JSFunction onEvent; + late JSFunction onError; + void detach() { + video + ..removeEventListener(type, onEvent) + ..removeEventListener('error', onError); + } + + onEvent = ((JSAny? _) { + detach(); + if (!done.isCompleted) done.complete(); + }).toJS; + onError = ((JSAny? _) { + detach(); + if (!done.isCompleted) { + done.completeError( + StateError('The video element failed: code ${video.error?.code}'), + ); + } + }).toJS; + video + ..addEventListener(type, onEvent) + ..addEventListener('error', onError); + return done.future.timeout(const Duration(seconds: 20)); +} + +@JS('document') +external _Document get _document; + +extension type _Document._(JSObject _) implements JSObject { + external JSObject createElement(String tag); +} + +extension type _VideoElement._(JSObject _) implements JSObject { + external set src(String value); + external set muted(bool value); + external set preload(String value); + external set currentTime(num value); + external int get videoWidth; + external int get videoHeight; + external double get duration; + external _MediaError? get error; + external void addEventListener(String type, JSFunction listener); + external void removeEventListener(String type, JSFunction listener); +} + +extension type _MediaError._(JSObject _) implements JSObject { + external int get code; +} + +extension type _Canvas._(JSObject _) implements JSObject { + external set width(int value); + external set height(int value); + external _Context2D getContext(String contextId); +} + +extension type _Context2D._(JSObject _) implements JSObject { + external void drawImage(JSObject image, num dx, num dy); + external _ImageData getImageData(int x, int y, int width, int height); +} + +extension type _ImageData._(JSObject _) implements JSObject { + external JSUint8ClampedArray get data; +} + +@JS('Blob') +extension type _Blob._(JSObject _) implements JSObject { + external factory _Blob(JSArray parts, _BlobOptions options); +} + +extension type _BlobOptions._(JSObject _) implements JSObject { + external factory _BlobOptions({String type}); +} + +@JS('URL') +extension type _Url._(JSObject _) implements JSObject { + external static String createObjectURL(_Blob blob); + external static void revokeObjectURL(String url); +} From c69ee68bf2be459069c2a1ba0bfee8da05f1e6d4 Mon Sep 17 00:00:00 2001 From: Dara Adedeji Date: Tue, 29 Sep 2026 01:11:18 -0400 Subject: [PATCH 2/8] Export video on the web beta, and of cloud strategies anywhere Video export was blocked on the web because it encodes with a bundled ffmpeg process, which a browser cannot start. It was also broken for cloud strategies everywhere: the dialog listed pages from, and rendered, the local Hive box, where cloud strategies never are, so it showed no pages and exported nothing. Encoding moves behind VideoFrameSink. VideoExporter renders frames and hands them to a sink: - FfmpegVideoSink (desktop) is the existing PNG-sequence + ffconcat + ffmpeg pipeline, moved out of the exporter unchanged. - The browser sink encodes with WebCodecs into an MP4 as frames arrive (previous commit) and hands the bytes back, which the dialog downloads through FilePicker.saveFile(bytes:). Presets keep their sizes and bitrates; the browser has no second pass, so there is no size retry. A browser without H.264 WebCodecs is told so before anything renders. What gets rendered (loadVideoExportSource): a local strategy is saved and read back from the library as before. A cloud strategy's other pages live only on the server, so the export saves, waits (up to 20 s) for this device's changes to land, i.e. for the strategy to show as synced, and then reads the whole strategy from the server. If the work does not land it stops and says so rather than export a video missing edits the user can see. A signed-out reader who opened a share link reads through that link (fetchFullSnapshot takes the shareToken the server already accepts). Images go through the same resolution as screenshots: files as they are, cloud URLs fetched and decoded before rendering, held for the whole export and handed to the capture container through captureImageSourcesProvider. The dialog's page list comes from wherever the strategy lives, and it shows its progress dialog from the start, so saving and syncing are visible and cancellable. Video export leaves PlatformPolicy.webBeta.desktopOnly. Co-Authored-By: Claude Opus 5.5 (1M context) --- lib/collab/convex_strategy_repository.dart | 11 +- lib/config/platform_policy.dart | 1 - .../video_export/browser_video_sink.dart | 2 + .../video_export/browser_video_sink_stub.dart | 9 + .../video_export/browser_video_sink_web.dart | 79 ++++ .../video_export/ffmpeg_video_sink.dart | 116 ++++++ .../video_export/video_export_source.dart | 158 ++++++++ lib/services/video_export/video_exporter.dart | 98 ++--- .../video_export/video_frame_sink.dart | 32 ++ lib/strategy/strategy_import_export.dart | 11 +- lib/widgets/dialogs/export_video_dialog.dart | 238 +++++++---- test/collab_sync_models_test.dart | 2 +- .../video_export_navigation_failure_test.dart | 8 +- test/video_export_source_test.dart | 383 ++++++++++++++++++ test/video_exporter_sink_test.dart | 293 ++++++++++++++ test/widgets/web_beta_library_test.dart | 2 +- 16 files changed, 1275 insertions(+), 168 deletions(-) create mode 100644 lib/services/video_export/browser_video_sink.dart create mode 100644 lib/services/video_export/browser_video_sink_stub.dart create mode 100644 lib/services/video_export/browser_video_sink_web.dart create mode 100644 lib/services/video_export/ffmpeg_video_sink.dart create mode 100644 lib/services/video_export/video_export_source.dart create mode 100644 lib/services/video_export/video_frame_sink.dart create mode 100644 test/video_export_source_test.dart create mode 100644 test/video_exporter_sink_test.dart diff --git a/lib/collab/convex_strategy_repository.dart b/lib/collab/convex_strategy_repository.dart index 169979f5..96629b51 100644 --- a/lib/collab/convex_strategy_repository.dart +++ b/lib/collab/convex_strategy_repository.dart @@ -157,12 +157,17 @@ class ConvexStrategyRepository { .map(_pageSnapshot); } + /// [shareToken]: see [fetchShell]. Future fetchFullSnapshot( - String strategyPublicId, - ) async { + String strategyPublicId, { + String? shareToken, + }) async { return _fullSnapshot( await _api.strategy - .getFullSnapshot(strategyPublicId: strategyPublicId) + .getFullSnapshot( + strategyPublicId: strategyPublicId, + shareToken: _optional(shareToken), + ) .fetch(), ); } diff --git a/lib/config/platform_policy.dart b/lib/config/platform_policy.dart index c5b228c3..3adcab4d 100644 --- a/lib/config/platform_policy.dart +++ b/lib/config/platform_policy.dart @@ -47,7 +47,6 @@ class PlatformPolicy { desktopOnly: { PlatformFeature.exportFiles, PlatformFeature.importFiles, - PlatformFeature.videoExport, PlatformFeature.fileDrop, }, // Empty for now; the Beta dialog hides its "Coming" list when it is. diff --git a/lib/services/video_export/browser_video_sink.dart b/lib/services/video_export/browser_video_sink.dart new file mode 100644 index 00000000..0b45f1dc --- /dev/null +++ b/lib/services/video_export/browser_video_sink.dart @@ -0,0 +1,2 @@ +export 'browser_video_sink_stub.dart' + if (dart.library.js_interop) 'browser_video_sink_web.dart'; diff --git a/lib/services/video_export/browser_video_sink_stub.dart b/lib/services/video_export/browser_video_sink_stub.dart new file mode 100644 index 00000000..de736125 --- /dev/null +++ b/lib/services/video_export/browser_video_sink_stub.dart @@ -0,0 +1,9 @@ +import 'package:icarus/services/video_export/video_export_quality.dart'; +import 'package:icarus/services/video_export/video_frame_sink.dart'; + +/// Whether this browser can encode [quality]'s video. Never, off the web. +Future browserCanEncodeVideo(VideoExportQuality quality) async => false; + +/// Encodes in the browser. Off the web there is no browser to encode in. +VideoFrameSink createBrowserVideoSink(VideoExportQuality quality) => + throw UnsupportedError('Browser video encoding exists only on the web.'); diff --git a/lib/services/video_export/browser_video_sink_web.dart b/lib/services/video_export/browser_video_sink_web.dart new file mode 100644 index 00000000..ea13e640 --- /dev/null +++ b/lib/services/video_export/browser_video_sink_web.dart @@ -0,0 +1,79 @@ +import 'dart:typed_data'; + +import 'package:icarus/const/coordinate_system.dart'; +import 'package:icarus/services/video_export/video_export_quality.dart'; +import 'package:icarus/services/video_export/video_frame_sink.dart'; +import 'package:icarus/services/video_export/web_codecs_video_encoder.dart'; + +/// Whether this browser can encode [quality]'s video, checked at its +/// largest size before anything renders. +Future browserCanEncodeVideo(VideoExportQuality quality) { + const size = CoordinateSystem.screenShotSize; + return WebCodecsMp4Encoder.isSupported( + width: size.width.round(), + height: size.height.round(), + fps: quality.fps, + // The sized presets hold a constant bitrate, which not every encoder + // offers; check that mode too. + bitrate: quality.sizePolicy == null + ? null + : VideoExportSizePolicy.maxVideoBitrate, + ); +} + +/// Encodes in the browser with WebCodecs; [VideoFrameSink.finish] returns +/// the finished .mp4 for the caller to download. +VideoFrameSink createBrowserVideoSink(VideoExportQuality quality) => + _WebCodecsVideoSink(quality); + +/// Sizes and bitrates follow the desktop presets: Potato and Social hold the +/// bitrate their size target allows for the video's length (and Potato +/// drops to 720p at the bitrate floor); Max encodes at a high variable +/// bitrate. The browser encodes as frames arrive, so there is no separate +/// encode pass and no second attempt at a smaller size. +class _WebCodecsVideoSink implements VideoFrameSink { + _WebCodecsVideoSink(this.quality); + + final VideoExportQuality quality; + final WebCodecsMp4Encoder _encoder = WebCodecsMp4Encoder(); + + @override + Future start({ + required int width, + required int height, + required int totalFrames, + required double totalSeconds, + }) { + final outputHeight = quality.outputHeightForDuration(totalSeconds); + return _encoder.start( + inputWidth: width, + inputHeight: height, + outputWidth: (width * outputHeight / height).round(), + outputHeight: outputHeight, + fps: quality.fps, + bitrate: quality.sizePolicy?.initialVideoBitrate(totalSeconds), + ); + } + + @override + Future addFrame(Uint8List rgba, double durationSeconds) => + _encoder.addFrame( + rgba, + duration: Duration(microseconds: (durationSeconds * 1e6).round()), + ); + + @override + Future finish({ + void Function(double fraction)? onProgress, + }) async { + final video = await _encoder.finish(); + onProgress?.call(1); + return video; + } + + @override + void cancel() => _encoder.cancel(); + + @override + Future close() async => _encoder.cancel(); +} diff --git a/lib/services/video_export/ffmpeg_video_sink.dart b/lib/services/video_export/ffmpeg_video_sink.dart new file mode 100644 index 00000000..0da0087d --- /dev/null +++ b/lib/services/video_export/ffmpeg_video_sink.dart @@ -0,0 +1,116 @@ +import 'dart:async'; +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:icarus/services/video_export/ffmpeg_png_sequence_writer.dart'; +import 'package:icarus/services/video_export/ffmpeg_video_encoder.dart'; +import 'package:icarus/services/video_export/video_export_quality.dart'; +import 'package:icarus/services/video_export/video_frame_sink.dart'; +import 'package:path/path.dart' as p; + +/// Desktop: frames go to one ffmpeg process as a lossless PNG sequence, +/// listed with their durations in an ffconcat playlist, and a second ffmpeg +/// run encodes the playlist into [outputPath]. +class FfmpegVideoSink implements VideoFrameSink { + FfmpegVideoSink({ + required this.binary, + required this.outputPath, + required this.quality, + }); + + final String binary; + final String outputPath; + final VideoExportQuality quality; + + final FfmpegVideoEncoder _encoder = FfmpegVideoEncoder(); + final List _concatLines = ['ffconcat version 1.0']; + Directory? _tempDir; + FfmpegPngSequenceWriter? _frameWriter; + double _totalSeconds = 0; + int _frameIndex = 0; + String? _lastFrameFile; + + @override + Future start({ + required int width, + required int height, + required int totalFrames, + required double totalSeconds, + }) async { + _totalSeconds = totalSeconds; + final tempDir = + await Directory.systemTemp.createTemp('icarus_video_export_'); + _tempDir = tempDir; + final frameWriter = FfmpegPngSequenceWriter(); + _frameWriter = frameWriter; + await frameWriter.start( + binary: binary, + workingDirectory: tempDir.path, + width: width, + height: height, + fps: quality.fps, + totalFrames: totalFrames, + ); + } + + @override + Future addFrame(Uint8List rgba, double durationSeconds) async { + final fileName = 'frame_${_frameIndex.toString().padLeft(5, '0')}.png'; + await _frameWriter!.writeFrame(rgba); + _frameIndex++; + _concatLines + ..add("file '$fileName'") + ..add('duration ${durationSeconds.toStringAsFixed(6)}'); + _lastFrameFile = fileName; + } + + @override + Future finish( + {void Function(double fraction)? onProgress}) async { + await _frameWriter!.finish(); + _frameWriter = null; + + // The concat demuxer ignores the duration of the final entry unless the + // last file is repeated. + if (_lastFrameFile != null) { + _concatLines.add("file '$_lastFrameFile'"); + } + final tempDir = _tempDir!; + await File(p.join(tempDir.path, 'frames.ffconcat')) + .writeAsString(_concatLines.join('\n')); + + await _encoder.encode( + binary: binary, + workingDirectory: tempDir.path, + concatListFileName: 'frames.ffconcat', + outputPath: outputPath, + totalSeconds: _totalSeconds, + quality: quality, + onProgress: onProgress, + ); + return null; + } + + @override + void cancel() { + // close() awaits the writer's termination; here we only need to + // interrupt the processes. + unawaited(_frameWriter?.cancel()); + _encoder.cancel(); + } + + @override + Future close() async { + // Wait for ffmpeg to exit before deleting the directory it writes into; + // on Windows the delete races a still-exiting process otherwise. + await _frameWriter?.cancel(); + _frameWriter = null; + try { + await _tempDir?.delete(recursive: true); + _tempDir = null; + } on Object { + // Leaving orphaned temp frames behind is preferable to masking the + // original export result. + } + } +} diff --git a/lib/services/video_export/video_export_source.dart b/lib/services/video_export/video_export_source.dart new file mode 100644 index 00000000..373742ca --- /dev/null +++ b/lib/services/video_export/video_export_source.dart @@ -0,0 +1,158 @@ +import 'dart:async'; + +import 'package:flutter_riverpod/flutter_riverpod.dart'; +import 'package:hive_ce/hive.dart'; +import 'package:icarus/collab/collab_models.dart'; +import 'package:icarus/collab/convex_strategy_repository.dart'; +import 'package:icarus/const/hive_boxes.dart'; +import 'package:icarus/const/sort_index_order.dart'; +import 'package:icarus/providers/collab/cloud_media_cache_provider.dart'; +import 'package:icarus/providers/collab/remote_strategy_snapshot_provider.dart'; +import 'package:icarus/providers/share_link_provider.dart'; +import 'package:icarus/providers/strategy_image_source.dart'; +import 'package:icarus/providers/strategy_provider.dart'; +import 'package:icarus/providers/strategy_save_state_provider.dart'; +import 'package:icarus/screenshot/capture_images.dart'; +import 'package:icarus/services/local_image_file.dart'; +import 'package:icarus/strategy/strategy_import_export.dart'; +import 'package:icarus/strategy/strategy_page_models.dart'; + +/// A page the export dialog offers. +typedef VideoExportPageChoice = ({String id, String name}); + +/// The open strategy's pages in order, from wherever the strategy lives. +List videoExportPageChoices(WidgetRef ref) { + final strategy = ref.read(strategyProvider); + return switch (strategy.source) { + StrategySource.cloud => [ + for (final page in [ + ...?ref.read(remoteEditorSnapshotProvider).valueOrNull?.pages, + ]..sortBySortIndex((page) => page.sortIndex)) + (id: page.publicId, name: page.name), + ], + StrategySource.local => [ + for (final page in [ + ...?Hive.box(HiveBoxNames.strategiesBox) + .get(strategy.strategyId) + ?.pages, + ]..sortBySortIndex((page) => page.sortIndex)) + (id: page.id, name: page.name), + ], + null => const [], + }; +} + +/// This device's changes to a cloud strategy did not reach the server in +/// time, so an export would leave out work the user can see. +class VideoExportNotSynced implements Exception { + const VideoExportNotSynced(); + + String get userMessage => + "Your latest changes haven't synced yet. Export once the strategy " + 'shows as synced.'; + + @override + String toString() => 'VideoExportNotSynced'; +} + +/// The strategy a video export renders, saved and whole, with its images +/// decoded. Release [images] once the export is done. +typedef VideoExportSource = ({StrategyData strategy, CaptureImages images}); + +/// How long a cloud export waits for this device's changes to land. +const videoExportSyncTimeout = Duration(seconds: 20); + +/// Saves the open strategy and reads it back whole for [pageIds]. +/// +/// A local strategy is read back from the library. A cloud strategy's other +/// pages live only on the server, so the export waits for this device's +/// changes to land (the strategy shows as synced), then reads the whole +/// strategy from the server. Throws [VideoExportNotSynced] when they don't +/// land within [videoExportSyncTimeout], and [CaptureImagesUnavailable] when +/// an image on the exported pages cannot be fetched. +Future loadVideoExportSource( + WidgetRef ref, { + required Set pageIds, +}) async { + final state = ref.read(strategyProvider); + final strategyId = state.strategyId; + if (strategyId == null) { + throw StateError('No strategy is open to export.'); + } + await ref.read(strategyProvider.notifier).forceSaveNow(strategyId); + // A signed-out reader's only access is the link they opened. + final linkView = ref.read(shareLinkViewProvider); + final shareToken = + linkView?.strategyPublicId == strategyId ? linkView!.token : null; + + final StrategyData strategy; + final Map assets; + switch (state.source) { + case StrategySource.cloud: + await _waitUntilSynced(ref); + final snapshot = await ref + .read(convexStrategyRepositoryProvider) + .fetchFullSnapshot(strategyId, shareToken: shareToken); + strategy = + StrategyImportExportService.strategyDataFromRemoteSnapshot(snapshot); + assets = snapshot.assetsById; + case StrategySource.local || null: + final saved = + Hive.box(HiveBoxNames.strategiesBox).get(strategyId); + if (saved == null) { + throw StateError('Strategy $strategyId is not in the library.'); + } + strategy = saved; + assets = const {}; + } + + final isCloud = state.source == StrategySource.cloud; + final images = await resolveCaptureImages( + { + for (final page in strategy.pages) + if (pageIds.contains(page.id)) + for (final image in page.imageData) + image.id: resolveStrategyImageSource( + localFilePath: findLocalImageFile( + storageDirectory: state.storageDirectory, + imageId: image.id, + fileExtension: image.fileExtension, + ), + isCloudStrategy: isCloud, + // The whole strategy was just read, and this device has + // nothing left to upload. + assetsLoaded: true, + remoteAsset: assets[image.id], + uploadMayBeQueuedHere: false, + ), + }, + fetch: (imageId, url) => downloadCloudImageBytes( + url, + freshUrl: () => + ref.read(convexStrategyRepositoryProvider).getImageAssetUrl( + strategyPublicId: strategyId, + assetPublicId: imageId, + shareToken: shareToken, + ), + ), + ); + return (strategy: strategy, images: images); +} + +Future _waitUntilSynced(WidgetRef ref) async { + if (ref.read(strategySaveStateProvider).canLeaveSafely) return; + final synced = Completer(); + final subscription = ref.listenManual( + strategySaveStateProvider.select((state) => state.canLeaveSafely), + (_, canLeaveSafely) { + if (canLeaveSafely && !synced.isCompleted) synced.complete(); + }, + ); + try { + await synced.future.timeout(videoExportSyncTimeout); + } on TimeoutException { + throw const VideoExportNotSynced(); + } finally { + subscription.close(); + } +} diff --git a/lib/services/video_export/video_exporter.dart b/lib/services/video_export/video_exporter.dart index 11d04e43..f9510fc2 100644 --- a/lib/services/video_export/video_exporter.dart +++ b/lib/services/video_export/video_exporter.dart @@ -1,5 +1,5 @@ import 'dart:async'; -import 'dart:io'; +import 'dart:typed_data'; import 'package:flutter/material.dart'; import 'package:flutter_riverpod/flutter_riverpod.dart'; @@ -10,6 +10,7 @@ import 'package:icarus/page_transition/agent_path.dart'; import 'package:icarus/page_transition/navigation_geometry_map.dart'; import 'package:icarus/page_transition/transition_planner.dart'; import 'package:icarus/providers/map_provider.dart'; +import 'package:icarus/providers/strategy_image_source.dart'; import 'package:icarus/providers/strategy_page.dart'; import 'package:icarus/providers/strategy_provider.dart'; import 'package:icarus/providers/transition_provider.dart'; @@ -17,12 +18,11 @@ import 'package:icarus/screenshot/offscreen_capture.dart'; import 'package:icarus/screenshot/capture_geometry.dart'; import 'package:icarus/screenshot/persistent_offscreen_renderer.dart'; import 'package:icarus/screenshot/screenshot_view.dart'; -import 'package:icarus/services/video_export/ffmpeg_png_sequence_writer.dart'; import 'package:icarus/services/video_export/ffmpeg_video_encoder.dart'; import 'package:icarus/services/video_export/video_export_quality.dart'; +import 'package:icarus/services/video_export/video_frame_sink.dart'; import 'package:icarus/view_cone/vision_geometry.dart'; import 'package:icarus/widgets/page_transition_overlay.dart'; -import 'package:path/path.dart' as p; @visibleForTesting Future runPreservingScreenshotMode( @@ -46,6 +46,7 @@ class VideoExporter { required this.strategyState, required this.mapState, required this.geometry, + this.imageSources = const {}, this.navigation, this.requireNavigation = false, }); @@ -54,6 +55,10 @@ class VideoExporter { final StrategyState strategyState; final MapState mapState; final VisionGeometryMap? geometry; + + /// What each image on the exported pages paints, by image id; see + /// [captureImageSourcesProvider]. + final Map imageSources; final NavigationGeometryMap? navigation; final bool requireNavigation; @@ -80,22 +85,20 @@ class VideoExporter { static const double _renderWeight = 0.85; bool _cancelled = false; - final FfmpegVideoEncoder _encoder = FfmpegVideoEncoder(); - FfmpegPngSequenceWriter? _frameWriter; + VideoFrameSink? _sink; void cancel() { _cancelled = true; - // export()'s cleanup awaits the writer's termination; here we only need - // to interrupt the processes. - unawaited(_frameWriter?.cancel()); - _encoder.cancel(); + _sink?.cancel(); } - Future export({ + /// Renders [pages] into [sink]. Returns the finished video when [sink] + /// keeps it in memory (the browser), or null once it is written to its + /// file (desktop). + Future export({ required List pages, required Duration stepDuration, - required String ffmpegBinary, - required String outputPath, + required VideoFrameSink sink, required VideoExportQuality quality, void Function(double fraction, String label)? onProgress, }) async { @@ -119,23 +122,21 @@ class VideoExporter { } } - await runPreservingScreenshotMode( + return runPreservingScreenshotMode( () => _export( pages: pages, stepDuration: stepDuration, - ffmpegBinary: ffmpegBinary, - outputPath: outputPath, + sink: sink, quality: quality, onProgress: onProgress, ), ); } - Future _export({ + Future _export({ required List pages, required Duration stepDuration, - required String ffmpegBinary, - required String outputPath, + required VideoFrameSink sink, required VideoExportQuality quality, void Function(double fraction, String label)? onProgress, }) async { @@ -150,15 +151,16 @@ class VideoExporter { fps: fps, ); - final tempDir = await Directory.systemTemp.createTemp( - 'icarus_video_export_', - ); ProviderContainer? captureContainer; CaptureGeometryLease? captureGeometry; PersistentOffscreenRenderer? renderer; - FfmpegPngSequenceWriter? frameWriter; + _sink = sink; try { - final offscreenContainer = ProviderContainer(); + final offscreenContainer = ProviderContainer( + overrides: [ + captureImageSourcesProvider.overrideWithValue(imageSources), + ], + ); captureContainer = offscreenContainer; onProgress?.call(0, 'Preparing map'); captureGeometry = await prepareCaptureGeometry( @@ -173,20 +175,13 @@ class VideoExporter { wrapWidget: (child) => wrapForOffscreenCapture(child, container: offscreenContainer), ); - frameWriter = FfmpegPngSequenceWriter(); - _frameWriter = frameWriter; - await frameWriter.start( - binary: ffmpegBinary, - workingDirectory: tempDir.path, + await sink.start( width: CoordinateSystem.screenShotSize.width.round(), height: CoordinateSystem.screenShotSize.height.round(), - fps: fps, totalFrames: totalFrames, + totalSeconds: totalSeconds, ); - final concatLines = ['ffconcat version 1.0']; - var frameIndex = 0; var renderedFrames = 0; - String? lastFrameFile; Future renderFrame( ScreenshotView view, @@ -195,13 +190,7 @@ class VideoExporter { if (_cancelled) throw VideoExportCancelled(); view.hydrateProviders(offscreenContainer); final bytes = await renderer!.captureRawRgba(view); - final fileName = 'frame_${frameIndex.toString().padLeft(5, '0')}.png'; - await frameWriter!.writeFrame(bytes); - frameIndex++; - concatLines - ..add("file '$fileName'") - ..add('duration ${durationSeconds.toStringAsFixed(6)}'); - lastFrameFile = fileName; + await sink.addFrame(bytes, durationSeconds); renderedFrames++; onProgress?.call( renderedFrames / totalFrames * _renderWeight, @@ -264,47 +253,22 @@ class VideoExporter { } } - await frameWriter.finish(); - _frameWriter = null; - - // The concat demuxer ignores the duration of the final entry unless the - // last file is repeated. - if (lastFrameFile != null) { - concatLines.add("file '$lastFrameFile'"); - } - final concatFile = File(p.join(tempDir.path, 'frames.ffconcat')); - await concatFile.writeAsString(concatLines.join('\n')); - if (_cancelled) throw VideoExportCancelled(); onProgress?.call(_renderWeight, 'Encoding video'); - await _encoder.encode( - binary: ffmpegBinary, - workingDirectory: tempDir.path, - concatListFileName: 'frames.ffconcat', - outputPath: outputPath, - totalSeconds: totalSeconds, - quality: quality, + return await sink.finish( onProgress: (fraction) => onProgress?.call( _renderWeight + fraction * (1 - _renderWeight), 'Encoding video', ), ); } finally { - // Wait for ffmpeg to exit before deleting the directory it writes into; - // on Windows the delete races a still-exiting process otherwise. - await frameWriter?.cancel(); - _frameWriter = null; + _sink = null; try { + await sink.close(); await renderer?.dispose(); } finally { captureGeometry?.close(); captureContainer?.dispose(); - try { - await tempDir.delete(recursive: true); - } on Object { - // Leaving orphaned temp frames behind is preferable to masking the - // original export result. - } } } } diff --git a/lib/services/video_export/video_frame_sink.dart b/lib/services/video_export/video_frame_sink.dart new file mode 100644 index 00000000..4d0353b4 --- /dev/null +++ b/lib/services/video_export/video_frame_sink.dart @@ -0,0 +1,32 @@ +import 'dart:typed_data'; + +/// Where [VideoExporter] sends the frames it renders: an encoder that turns +/// them into an .mp4. Desktop pipes them to the bundled ffmpeg +/// (`FfmpegVideoSink`); the browser encodes them with WebCodecs +/// (`createBrowserVideoSink`). +abstract interface class VideoFrameSink { + /// Called once, before the first frame. Every frame is [width] x [height]; + /// the video runs [totalSeconds] over [totalFrames] frames. + Future start({ + required int width, + required int height, + required int totalFrames, + required double totalSeconds, + }); + + /// One rendered frame, top-to-bottom RGBA, shown for [durationSeconds]. + Future addFrame(Uint8List rgba, double durationSeconds); + + /// Encodes every frame added. Returns the finished video when this sink + /// keeps it in memory (the browser), or null once it is written to its + /// file (desktop). [onProgress] reports 0 to 1. + Future finish({void Function(double fraction)? onProgress}); + + /// Stops the encode in flight; [addFrame] and [finish] then throw + /// `VideoExportCancelled`. + void cancel(); + + /// Releases everything the sink holds. Safe after [finish], after + /// [cancel], and after a failure. + Future close(); +} diff --git a/lib/strategy/strategy_import_export.dart b/lib/strategy/strategy_import_export.dart index b666e082..5f23a826 100644 --- a/lib/strategy/strategy_import_export.dart +++ b/lib/strategy/strategy_import_export.dart @@ -2427,7 +2427,7 @@ class StrategyImportExportService { .read(convexStrategyRepositoryProvider) .fetchFullSnapshot(strategyId); await _ensureRemoteAssetsCached(snapshot); - final strategy = _strategyDataFromRemoteSnapshot(snapshot); + final strategy = strategyDataFromRemoteSnapshot(snapshot); final outputFile = await FilePicker.platform.saveFile( type: FileType.custom, dialogTitle: 'Please select an output file:', @@ -2454,13 +2454,8 @@ class StrategyImportExportService { await zipStrategy(id: id, outputFilePath: outputFile); } - @visibleForTesting - static StrategyData strategyDataFromRemoteSnapshotForTest( - RemoteFullStrategySnapshot snapshot, - ) => - _strategyDataFromRemoteSnapshot(snapshot); - - static StrategyData _strategyDataFromRemoteSnapshot( + /// A cloud strategy as the library models it, from the server's full copy. + static StrategyData strategyDataFromRemoteSnapshot( RemoteFullStrategySnapshot snapshot, ) { final pages = []; diff --git a/lib/widgets/dialogs/export_video_dialog.dart b/lib/widgets/dialogs/export_video_dialog.dart index fff0116b..2903a7f0 100644 --- a/lib/widgets/dialogs/export_video_dialog.dart +++ b/lib/widgets/dialogs/export_video_dialog.dart @@ -1,26 +1,29 @@ import 'dart:async'; import 'package:file_picker/file_picker.dart'; +import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; import 'package:flutter_riverpod/flutter_riverpod.dart'; -import 'package:hive_ce/hive.dart'; import 'package:icarus/const/coordinate_system.dart'; -import 'package:icarus/const/hive_boxes.dart'; import 'package:icarus/const/settings.dart'; import 'package:icarus/const/sort_index_order.dart'; import 'package:icarus/providers/drawing_provider.dart'; import 'package:icarus/providers/map_provider.dart'; import 'package:icarus/providers/screenshot_provider.dart'; -import 'package:icarus/providers/strategy_page.dart'; import 'package:icarus/providers/strategy_provider.dart'; import 'package:icarus/providers/user_preferences_provider.dart'; import 'package:icarus/providers/view_cone_geometry_provider.dart'; import 'package:icarus/providers/navigation_geometry_provider.dart'; import 'package:icarus/page_transition/navigation_geometry_map.dart'; import 'package:icarus/services/analytics_service.dart'; +import 'package:icarus/screenshot/capture_images.dart'; +import 'package:icarus/services/video_export/browser_video_sink.dart'; import 'package:icarus/services/video_export/ffmpeg_video_encoder.dart'; +import 'package:icarus/services/video_export/ffmpeg_video_sink.dart'; +import 'package:icarus/services/video_export/video_export_source.dart'; import 'package:icarus/services/video_export/video_export_quality.dart'; import 'package:icarus/services/video_export/video_exporter.dart'; +import 'package:icarus/services/video_export/video_frame_sink.dart'; import 'package:icarus/strategy/strategy_import_export.dart'; import 'package:icarus/view_cone/vision_geometry.dart'; import 'package:shadcn_ui/shadcn_ui.dart'; @@ -42,25 +45,33 @@ class _ExportVideoDialogState extends ConsumerState { /// descendant (same idiom as UploadImageDialog). static const double _contentWidth = _dialogWidth - 48; - List _pages = const []; + List _pages = const []; final Set _selectedPageIds = {}; double _stepDurationSeconds = 3.0; VideoExportQuality _quality = VideoExportQuality.social; + /// Saving, syncing, and loading images, before any frame renders. + bool _preparing = false; + bool _cancelRequested = false; VideoExporter? _exporter; double _progress = 0; String _progressLabel = ''; bool _exportRunning = false; - bool get _isExporting => _exporter != null; + bool get _isExporting => _preparing || _exporter != null; @override void dispose() { // Closing the dialog mid-export must not leave frames rendering and - // ffmpeg encoding in the background. - _exporter?.cancel(); + // the encoder running in the background. + _cancel(); super.dispose(); } + void _cancel() { + _cancelRequested = true; + _exporter?.cancel(); + } + @override void initState() { super.initState(); @@ -68,10 +79,7 @@ class _ExportVideoDialogState extends ConsumerState { .read(appPreferencesProvider) .videoExportStepDurationSeconds .clamp(1.0, 15.0); - final doc = Hive.box( - HiveBoxNames.strategiesBox, - ).get(ref.read(strategyProvider).id); - _pages = [...?doc?.pages]..sortBySortIndex((item) => item.sortIndex); + _pages = videoExportPageChoices(ref); _selectedPageIds.addAll(_pages.map((p) => p.id)); } @@ -91,7 +99,8 @@ class _ExportVideoDialogState extends ConsumerState { // The root container outlives this dialog, so cleanup after a // mid-export dismissal can still reach the live providers. final container = ProviderScope.containerOf(context, listen: false); - + final quality = _quality; + final pageIds = {..._selectedPageIds}; final stepDuration = Duration( milliseconds: (_stepDurationSeconds * 1000).round(), ); @@ -100,83 +109,102 @@ class _ExportVideoDialogState extends ConsumerState { .read(appPreferencesProvider.notifier) .setVideoExportStepDurationSeconds(_stepDurationSeconds), ); - - final ffmpegBinary = await FfmpegVideoEncoder.resolveBinary(); - if (ffmpegBinary == null) { - Settings.showToast( - message: - 'Video encoder not found. Reinstalling Icarus should fix this.', - backgroundColor: Settings.tacticalVioletTheme.destructive, - ); - return; - } - - final selectedOutputPath = await FilePicker.platform.saveFile( - type: FileType.custom, - dialogTitle: 'Please select an output file:', - fileName: - "${sanitizeStrategyFileName(ref.read(strategyProvider).stratName ?? "new video")}.mp4", - allowedExtensions: ['mp4'], - ); - if (selectedOutputPath == null || !mounted) return; - final outputPath = ensureMp4Extension(selectedOutputPath); - - final strategyId = ref.read(strategyProvider).id; - await ref.read(strategyProvider.notifier).forceSaveNow(strategyId); - final doc = Hive.box( - HiveBoxNames.strategiesBox, - ).get(strategyId); - if (doc == null || !mounted) return; - - // Resolve pages from the freshly saved document — the dialog's initial - // snapshot may predate unsaved edits on the active page. - final selectedPages = ([...doc.pages] - ..sortBySortIndex((item) => item.sortIndex)) - .where((p) => _selectedPageIds.contains(p.id)) - .toList(); - if (selectedPages.isEmpty) return; - - final mapState = ref.read(mapProvider); - final requireNavigation = - ref.read(worldGeometryEnabledProvider(mapState.currentMap)); - VisionGeometryMap? geometry; - NavigationGeometryMap? navigation; - try { - if (requireNavigation) { - navigation = await ref - .read(navigationGeometryProvider(mapState.currentMap).future); - } else { - geometry = await ref - .read(viewConeGeometryProvider(mapState.currentMap).future); + final fileName = + "${sanitizeStrategyFileName(ref.read(strategyProvider).stratName ?? "new video")}.mp4"; + + // Where the video goes is settled first, so a dismissed save dialog or + // a browser that cannot encode video costs nothing. + final VideoFrameSink sink; + if (kIsWeb) { + if (!await browserCanEncodeVideo(quality)) { + Settings.showToast( + message: 'This browser cannot export video. Try the latest Chrome ' + 'or Edge, or the desktop app.', + backgroundColor: Settings.tacticalVioletTheme.destructive, + ); + return; + } + sink = createBrowserVideoSink(quality); + } else { + final ffmpegBinary = await FfmpegVideoEncoder.resolveBinary(); + if (ffmpegBinary == null) { + Settings.showToast( + message: + 'Video encoder not found. Reinstalling Icarus should fix this.', + backgroundColor: Settings.tacticalVioletTheme.destructive, + ); + return; } - } on Object { - // The provider reports the failure. The exporter rejects movement on - // enabled maps when their required navigation data is unavailable. + final selectedOutputPath = await FilePicker.platform.saveFile( + type: FileType.custom, + dialogTitle: 'Please select an output file:', + fileName: fileName, + allowedExtensions: ['mp4'], + ); + if (selectedOutputPath == null) return; + sink = FfmpegVideoSink( + binary: ffmpegBinary, + outputPath: ensureMp4Extension(selectedOutputPath), + quality: quality, + ); } if (!mounted) return; - final exporter = VideoExporter( - strategy: doc, - strategyState: ref.read(strategyProvider), - mapState: mapState, - geometry: geometry, - navigation: navigation, - requireNavigation: requireNavigation, - ); setState(() { - _exporter = exporter; + _preparing = true; + _cancelRequested = false; _progress = 0; - _progressLabel = 'Preparing'; + _progressLabel = 'Saving your changes'; }); - - CoordinateSystem.instance.setIsScreenshot(true); + CaptureImages? images; try { - await exporter.export( + final source = await loadVideoExportSource(ref, pageIds: pageIds); + images = source.images; + if (_cancelRequested) throw VideoExportCancelled(); + final selectedPages = ([...source.strategy.pages] + ..sortBySortIndex((item) => item.sortIndex)) + .where((p) => pageIds.contains(p.id)) + .toList(); + + final mapState = ref.read(mapProvider); + final requireNavigation = + ref.read(worldGeometryEnabledProvider(mapState.currentMap)); + VisionGeometryMap? geometry; + NavigationGeometryMap? navigation; + try { + if (requireNavigation) { + navigation = await ref + .read(navigationGeometryProvider(mapState.currentMap).future); + } else { + geometry = await ref + .read(viewConeGeometryProvider(mapState.currentMap).future); + } + } on Object { + // The provider reports the failure. The exporter rejects movement on + // enabled maps when their required navigation data is unavailable. + } + if (_cancelRequested) throw VideoExportCancelled(); + + final exporter = VideoExporter( + strategy: source.strategy, + strategyState: ref.read(strategyProvider), + mapState: mapState, + geometry: geometry, + imageSources: images.sources, + navigation: navigation, + requireNavigation: requireNavigation, + ); + setState(() { + _exporter = exporter; + _progressLabel = 'Preparing'; + }); + + CoordinateSystem.instance.setIsScreenshot(true); + final video = await exporter.export( pages: selectedPages, stepDuration: stepDuration, - ffmpegBinary: ffmpegBinary, - outputPath: outputPath, - quality: _quality, + sink: sink, + quality: quality, onProgress: (fraction, label) { if (!mounted) return; setState(() { @@ -185,6 +213,16 @@ class _ExportVideoDialogState extends ConsumerState { }); }, ); + // The browser sink hands the finished video back to be downloaded; + // the desktop one has already written it where the user chose. + if (video != null) { + await FilePicker.platform.saveFile( + type: FileType.custom, + fileName: fileName, + allowedExtensions: ['mp4'], + bytes: video, + ); + } unawaited( AnalyticsService.instance.capture( 'content_exported', @@ -192,8 +230,8 @@ class _ExportVideoDialogState extends ConsumerState { 'content_type': 'video', 'page_count': selectedPages.length, 'step_duration_seconds': _stepDurationSeconds, - 'quality': _quality.name, - 'fps': _quality.fps, + 'quality': quality.name, + 'fps': quality.fps, }, ), ); @@ -209,12 +247,30 @@ class _ExportVideoDialogState extends ConsumerState { message: 'Video export cancelled.', backgroundColor: Settings.tacticalVioletTheme.primary, ); - } on Object catch (error) { + } on VideoExportNotSynced catch (error) { Settings.showToast( - message: 'Video export failed: $error', + message: error.userMessage, backgroundColor: Settings.tacticalVioletTheme.destructive, ); + } on CaptureImagesUnavailable catch (error) { + Settings.showToast( + message: error.userMessage, + backgroundColor: Settings.tacticalVioletTheme.destructive, + ); + } on Object catch (error) { + Settings.showToast( + // Closing the dialog mid-preparation can fail the work it cut + // short; the user asked to stop, so that is what they hear. + message: _cancelRequested + ? 'Video export cancelled.' + : 'Video export failed: $error', + backgroundColor: _cancelRequested + ? Settings.tacticalVioletTheme.primary + : Settings.tacticalVioletTheme.destructive, + ); } finally { + await sink.close(); + images?.release(); CoordinateSystem.instance.setIsScreenshot(false); container.read(screenshotProvider.notifier).setIsScreenShot(false); container @@ -222,6 +278,7 @@ class _ExportVideoDialogState extends ConsumerState { .rebuildAllPaths(CoordinateSystem.instance); if (mounted) { setState(() { + _preparing = false; _exporter = null; }); } @@ -271,7 +328,7 @@ class _ExportVideoDialogState extends ConsumerState { title: const Text('Exporting video'), actions: [ ShadButton.destructive( - onPressed: () => _exporter?.cancel(), + onPressed: _cancel, child: const Text('Cancel export'), ), ], @@ -300,6 +357,17 @@ class _ExportVideoDialogState extends ConsumerState { ), const SizedBox(height: 12), ShadProgress(value: _progress, minHeight: 8), + // A hidden tab stops painting frames, which stalls the render. + if (kIsWeb) ...[ + const SizedBox(height: 12), + Text( + 'Keep this tab open until the video downloads.', + style: TextStyle( + fontSize: 12, + color: colors.mutedForeground, + ), + ), + ], ], ), ), @@ -579,7 +647,7 @@ class _PagesPanel extends StatelessWidget { required this.onToggle, }); - final List pages; + final List pages; final Set selectedPageIds; final ValueChanged onToggle; diff --git a/test/collab_sync_models_test.dart b/test/collab_sync_models_test.dart index e56eab51..12f5f455 100644 --- a/test/collab_sync_models_test.dart +++ b/test/collab_sync_models_test.dart @@ -485,7 +485,7 @@ void main() { ); expect( - () => StrategyImportExportService.strategyDataFromRemoteSnapshotForTest( + () => StrategyImportExportService.strategyDataFromRemoteSnapshot( snapshot), throwsA( isA().having( diff --git a/test/video_export_navigation_failure_test.dart b/test/video_export_navigation_failure_test.dart index 2a207800..d0184143 100644 --- a/test/video_export_navigation_failure_test.dart +++ b/test/video_export_navigation_failure_test.dart @@ -9,6 +9,7 @@ import 'package:icarus/providers/strategy_page.dart'; import 'package:icarus/providers/strategy_provider.dart'; import 'package:icarus/providers/strategy_settings_provider.dart'; import 'package:icarus/services/video_export/ffmpeg_video_encoder.dart'; +import 'package:icarus/services/video_export/ffmpeg_video_sink.dart'; import 'package:icarus/services/video_export/video_export_quality.dart'; import 'package:icarus/services/video_export/video_exporter.dart'; @@ -66,8 +67,11 @@ void main() { exporter.export( pages: pages, stepDuration: const Duration(seconds: 1), - ffmpegBinary: 'must-not-run', - outputPath: 'must-not-create.mp4', + sink: FfmpegVideoSink( + binary: 'must-not-run', + outputPath: 'must-not-create.mp4', + quality: VideoExportQuality.social, + ), quality: VideoExportQuality.social, onProgress: (_, label) { startedPreparing = true; diff --git a/test/video_export_source_test.dart b/test/video_export_source_test.dart new file mode 100644 index 00000000..8bd73f64 --- /dev/null +++ b/test/video_export_source_test.dart @@ -0,0 +1,383 @@ +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:flutter/material.dart'; +import 'package:flutter_riverpod/flutter_riverpod.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:hive_ce/hive.dart'; +import 'package:http/http.dart' as http; +import 'package:http/testing.dart'; +import 'package:icarus/collab/cloud_media_models.dart'; +import 'package:icarus/collab/collab_models.dart'; +import 'package:icarus/collab/convex_strategy_repository.dart'; +import 'package:icarus/const/hive_boxes.dart'; +import 'package:icarus/const/maps.dart'; +import 'package:icarus/const/placed_classes.dart'; +import 'package:icarus/hive/hive_registration.dart'; +import 'package:icarus/providers/collab/remote_strategy_snapshot_provider.dart'; +import 'package:icarus/providers/share_link_provider.dart'; +import 'package:icarus/providers/strategy_image_source.dart'; +import 'package:icarus/providers/strategy_page.dart'; +import 'package:icarus/providers/strategy_provider.dart'; +import 'package:icarus/providers/strategy_save_state_provider.dart'; +import 'package:icarus/providers/strategy_settings_provider.dart'; +import 'package:icarus/services/video_export/video_export_source.dart'; +import 'package:icarus/strategy/strategy_page_models.dart'; +import 'package:image/image.dart' as img; +import 'package:path/path.dart' as path; + +const _strategyId = 'strategy-1'; +const _imageUrl = 'https://media.example.com/image-1.png'; + +final _png = Uint8List.fromList( + img.encodePng( + img.fill(img.Image(width: 4, height: 4), color: img.ColorRgb8(1, 2, 3)), + ), +); + +PlacedImage _image(String id) => PlacedImage( + position: const Offset(500, 500), + id: id, + aspectRatio: 1, + scale: 100, + fileExtension: '.png', + ); + +class _OpenStrategy extends StrategyProvider { + _OpenStrategy(this.source, {this.storageDirectory}); + + final StrategySource source; + final String? storageDirectory; + int saves = 0; + + @override + StrategyState build() => StrategyState( + strategyId: _strategyId, + strategyName: 'Strategy', + source: source, + storageDirectory: storageDirectory, + isOpen: true, + ); + + @override + Future forceSaveNow(String id) async => saves++; +} + +/// The save chip's state, set by the test instead of the op queue. +class _SaveState extends StrategySaveStateNotifier { + _SaveState({required this.synced}); + + final bool synced; + + @override + StrategySaveState build() => _state(synced: synced); + + void markSynced() => state = _state(synced: true); + + static StrategySaveState _state({required bool synced}) => StrategySaveState( + isDirty: !synced, + isSaving: false, + hasPendingCloudSync: !synced, + cloudSyncError: null, + hasPendingMediaSync: false, + mediaSyncErrorCount: 0, + lastPersistedAt: null, + ); +} + +class _Repository extends Fake implements ConvexStrategyRepository { + int fetches = 0; + final shareTokens = []; + + @override + Future fetchFullSnapshot( + String strategyPublicId, { + String? shareToken, + }) async { + fetches++; + shareTokens.add(shareToken); + final now = DateTime.utc(2026); + RemotePage page(String id, int sortIndex) => RemotePage( + publicId: id, + strategyPublicId: _strategyId, + name: 'Page $id', + sortIndex: sortIndex, + isAttack: true, + revision: 1, + createdAt: now, + updatedAt: now, + ); + RemoteFullPage fullPage(String id, int sortIndex) => RemoteFullPage( + page: page(id, sortIndex), + content: + RemotePageContent(revision: 1, createdAt: now, updatedAt: now), + ); + return RemoteFullStrategySnapshot( + header: RemoteStrategyHeader( + publicId: _strategyId, + name: 'Strategy', + mapData: Maps.mapNames[MapValue.ascent]!, + revision: 1, + createdAt: now, + updatedAt: now, + ), + pages: [fullPage('b', 1), fullPage('a', 0)], + elementsByPage: { + for (final (pageId, imageId) in [('a', 'image-a'), ('b', 'image-b')]) + pageId: [ + RemoteElement( + publicId: imageId, + strategyPublicId: _strategyId, + pagePublicId: pageId, + elementType: 'image', + payload: cloudElementPayload( + kind: 'image', + data: cloudImagePayloadFromPlacedImage(_image(imageId)), + ), + sortIndex: 0, + revision: 1, + deleted: false, + ), + ], + }, + lineupsByPage: const {}, + assetsById: { + for (final imageId in ['image-a', 'image-b']) + imageId: RemoteImageAsset( + publicId: imageId, + fileExtension: '.png', + width: 4, + height: 4, + url: '$_imageUrl?$imageId', + legacyStoragePath: null, + uploadStatus: 'active', + ), + }, + ); + } +} + +Future _pumpRef( + WidgetTester tester, List overrides) async { + late WidgetRef ref; + await tester.pumpWidget(ProviderScope( + overrides: overrides, + child: Consumer(builder: (context, widgetRef, _) { + ref = widgetRef; + return const SizedBox.shrink(); + }), + )); + return ref; +} + +void main() { + group('a cloud strategy', () { + testWidgets('waits for sync, then reads the whole strategy from the server', + (tester) async { + final strategy = _OpenStrategy(StrategySource.cloud); + final saveState = _SaveState(synced: false); + final repository = _Repository(); + final ref = await _pumpRef(tester, [ + strategyProvider.overrideWith(() => strategy), + strategySaveStateProvider.overrideWith(() => saveState), + convexStrategyRepositoryProvider.overrideWithValue(repository), + ]); + final requested = []; + + final source = await tester.runAsync( + () => http.runWithClient( + () async { + final loading = loadVideoExportSource(ref, pageIds: {'a'}); + await Future.delayed(const Duration(milliseconds: 50)); + // Saved, but nothing is read until this device's work lands. + expect(strategy.saves, 1); + expect(repository.fetches, 0); + saveState.markSynced(); + return loading; + }, + () => MockClient((request) async { + requested.add(request.url.query); + return http.Response.bytes(_png, 200); + }), + ), + ); + addTearDown(source!.images.release); + + expect(repository.fetches, 1); + expect(source.strategy.id, _strategyId); + expect(source.strategy.mapData, MapValue.ascent); + expect( + [for (final page in source.strategy.pages) page.id], + containsAll(['a', 'b']), + ); + // Only the exported page's image is fetched and decoded. + expect(requested, ['image-a']); + expect(source.images.sources.keys, ['image-a']); + expect(source.images.sources['image-a'], isA()); + }); + + testWidgets('stops when this device has not synced in time', + (tester) async { + final repository = _Repository(); + final ref = await _pumpRef(tester, [ + strategyProvider + .overrideWith(() => _OpenStrategy(StrategySource.cloud)), + strategySaveStateProvider.overrideWith(() => _SaveState(synced: false)), + convexStrategyRepositoryProvider.overrideWithValue(repository), + ]); + + Object? error; + loadVideoExportSource(ref, pageIds: {'a'}).then( + (_) {}, + onError: (Object caught) => error = caught, + ); + await tester.pump(videoExportSyncTimeout + const Duration(seconds: 1)); + + expect(error, isA()); + expect(repository.fetches, 0); + }); + + testWidgets('reads through the link a signed-out reader opened', + (tester) async { + final repository = _Repository(); + final ref = await _pumpRef(tester, [ + strategyProvider + .overrideWith(() => _OpenStrategy(StrategySource.cloud)), + strategySaveStateProvider.overrideWith(() => _SaveState(synced: true)), + convexStrategyRepositoryProvider.overrideWithValue(repository), + shareLinkViewProvider.overrideWith( + (ref) => (strategyPublicId: _strategyId, token: 'link-token'), + ), + ]); + + final source = await tester.runAsync( + () => http.runWithClient( + () => loadVideoExportSource(ref, pageIds: {'a'}), + () => MockClient((_) async => http.Response.bytes(_png, 200)), + ), + ); + addTearDown(source!.images.release); + + expect(repository.shareTokens, ['link-token']); + }); + + testWidgets('offers the pages the editor lists, in order', (tester) async { + final ref = await _pumpRef(tester, [ + strategyProvider + .overrideWith(() => _OpenStrategy(StrategySource.cloud)), + remoteEditorSnapshotProvider.overrideWith(_TwoPageSnapshot.new), + ]); + await tester.runAsync( + () => ProviderScope.containerOf(tester.element(find.byType(Consumer))) + .read(remoteEditorSnapshotProvider.future), + ); + + expect(videoExportPageChoices(ref), [ + (id: 'first', name: 'First'), + (id: 'second', name: 'Second'), + ]); + }); + }); + + group('a local strategy', () { + late Directory dir; + + setUpAll(() async { + dir = await Directory.systemTemp.createTemp('icarus_video_source'); + Hive.init(dir.path); + registerIcarusAdapters(Hive); + await Hive.openBox(HiveBoxNames.strategiesBox); + }); + + tearDownAll(() async { + await Hive.close(); + await dir.delete(recursive: true); + }); + + testWidgets('is read back from the library with its image files', + (tester) async { + final storage = path.join(dir.path, 'storage'); + await tester.runAsync(() async { + final file = File(path.join(storage, 'images', 'image-a.png')); + await file.parent.create(recursive: true); + await file.writeAsBytes(_png); + await Hive.box(HiveBoxNames.strategiesBox).put( + _strategyId, + StrategyData( + id: _strategyId, + name: 'Strategy', + mapData: MapValue.bind, + versionNumber: 1, + lastEdited: DateTime(2026), + folderID: null, + pages: [ + StrategyPage( + id: 'a', + name: 'A', + drawingData: const [], + agentData: const [], + abilityData: const [], + textData: const [], + imageData: [_image('image-a'), _image('image-missing')], + utilityData: const [], + sortIndex: 0, + isAttack: true, + settings: StrategySettings(), + ), + ], + ), + ); + }); + final strategy = _OpenStrategy( + StrategySource.local, + storageDirectory: storage, + ); + final ref = await _pumpRef(tester, [ + strategyProvider.overrideWith(() => strategy), + ]); + + final source = await tester.runAsync( + () => loadVideoExportSource(ref, pageIds: {'a'}), + ); + addTearDown(source!.images.release); + + expect(strategy.saves, 1); + expect(source.strategy.mapData, MapValue.bind); + expect(source.images.sources['image-a'], isA()); + // A file the editor cannot find is exported as it shows: unavailable. + expect(source.images.sources['image-missing'], isA()); + }); + }); +} + +class _TwoPageSnapshot extends RemoteEditorSnapshotNotifier { + @override + Future build() async { + final now = DateTime.utc(2026); + RemotePage page(String id, String name, int sortIndex) => RemotePage( + publicId: id, + strategyPublicId: _strategyId, + name: name, + sortIndex: sortIndex, + isAttack: true, + revision: 1, + createdAt: now, + updatedAt: now, + ); + return RemoteEditorSnapshot( + shell: RemoteStrategyShell( + header: RemoteStrategyHeader( + publicId: _strategyId, + name: 'Strategy', + mapData: Maps.mapNames[MapValue.ascent]!, + revision: 1, + createdAt: now, + updatedAt: now, + role: 'owner', + ), + pages: [page('second', 'Second', 1), page('first', 'First', 0)], + ), + activePage: null, + ); + } +} diff --git a/test/video_exporter_sink_test.dart b/test/video_exporter_sink_test.dart new file mode 100644 index 00000000..c2259eb4 --- /dev/null +++ b/test/video_exporter_sink_test.dart @@ -0,0 +1,293 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:hive_ce/hive.dart'; +import 'package:icarus/const/coordinate_system.dart'; +import 'package:icarus/const/hive_boxes.dart'; +import 'package:icarus/const/image_scale_policy.dart'; +import 'package:icarus/const/maps.dart'; +import 'package:icarus/const/placed_classes.dart'; +import 'package:icarus/hive/hive_registration.dart'; +import 'package:icarus/providers/map_provider.dart'; +import 'package:icarus/providers/strategy_image_source.dart'; +import 'package:icarus/providers/strategy_page.dart'; +import 'package:icarus/providers/strategy_provider.dart'; +import 'package:icarus/providers/strategy_settings_provider.dart'; +import 'package:icarus/providers/user_preferences_provider.dart'; +import 'package:icarus/services/video_export/ffmpeg_video_sink.dart'; +import 'package:icarus/services/video_export/video_export_quality.dart'; +import 'package:icarus/services/video_export/video_exporter.dart'; +import 'package:icarus/services/video_export/video_frame_sink.dart'; +import 'package:icarus/strategy/strategy_page_models.dart'; +import 'package:image/image.dart' as img; + +/// Pure magenta, a colour no map or marker draws. +final _magentaPng = Uint8List.fromList( + img.encodePng( + img.fill( + img.Image(width: 64, height: 36), + color: img.ColorRgb8(255, 0, 255), + ), + ), +); + +class _RecordingSink implements VideoFrameSink { + final video = Uint8List.fromList([7, 7, 7]); + final frames = <(Uint8List, double)>[]; + ({int width, int height, int totalFrames, double totalSeconds})? started; + var closed = 0; + + @override + Future start({ + required int width, + required int height, + required int totalFrames, + required double totalSeconds, + }) async { + started = ( + width: width, + height: height, + totalFrames: totalFrames, + totalSeconds: totalSeconds, + ); + } + + @override + Future addFrame(Uint8List rgba, double durationSeconds) async => + frames.add((rgba, durationSeconds)); + + @override + Future finish({void Function(double)? onProgress}) async => video; + + @override + void cancel() {} + + @override + Future close() async => closed++; +} + +StrategyPage _page(String id, int sortIndex, Offset imageAt) => StrategyPage( + id: id, + name: id, + drawingData: const [], + agentData: const [], + abilityData: const [], + textData: const [], + imageData: [ + PlacedImage( + position: imageAt, + id: 'image-1', + aspectRatio: 64 / 36, + scale: ImageScalePolicy.defaultWidth, + fileExtension: '.png', + sizeVersion: PlacedImage.currentSizeVersion, + ), + ], + utilityData: const [], + sortIndex: sortIndex, + isAttack: true, + settings: StrategySettings(), + ); + +VideoExporter _exporter(List pages) { + return VideoExporter( + strategy: StrategyData( + id: 'cloud-strategy', + name: 'Cloud strat', + mapData: MapValue.ascent, + versionNumber: 1, + lastEdited: DateTime(2026), + folderID: null, + pages: pages, + ), + strategyState: const StrategyState( + strategyId: 'cloud-strategy', + strategyName: 'Cloud strat', + source: StrategySource.cloud, + storageDirectory: null, + isOpen: true, + ), + mapState: MapState(currentMap: MapValue.ascent, isAttack: true), + geometry: null, + imageSources: {'image-1': ImageBytes(_magentaPng)}, + ); +} + +/// Runs the export on real time with the app's frames still coming, as in +/// the app: the offscreen tree's layout builders wait for the app's next +/// frame. +Future _exportWithFrames( + WidgetTester tester, + VideoExporter exporter, + List pages, + VideoFrameSink sink, +) async { + Object? outcome; + var done = false; + await tester.runAsync(() async { + unawaited( + exporter + .export( + pages: pages, + stepDuration: const Duration(seconds: 2), + sink: sink, + quality: VideoExportQuality.social, + ) + .then((video) => outcome = video, + onError: (Object error) => outcome = error) + .whenComplete(() => done = true), + ); + }); + while (!done) { + await tester.runAsync( + () => Future.delayed(const Duration(milliseconds: 16)), + ); + await tester.pump(); + } + return outcome; +} + +int _magentaPixels(Uint8List rgba) { + var count = 0; + for (var i = 0; i < rgba.length; i += 4) { + if (rgba[i] > 240 && rgba[i + 1] < 20 && rgba[i + 2] > 240) count++; + } + return count; +} + +/// A PATH ffmpeg, the development fallback of FfmpegVideoEncoder. +final String? _ffmpeg = _onPath('ffmpeg'); +final String? _ffprobe = _onPath('ffprobe'); + +String? _onPath(String tool) { + try { + final result = Process.runSync( + Platform.isWindows ? 'where' : 'which', + [tool], + ); + if (result.exitCode != 0) return null; + return const LineSplitter().convert(result.stdout as String).first.trim(); + } on ProcessException { + return null; + } +} + +void main() { + late Directory hiveDir; + + setUpAll(() async { + hiveDir = await Directory.systemTemp.createTemp('icarus_video_sink'); + Hive.init(hiveDir.path); + registerIcarusAdapters(Hive); + await Hive.openBox(HiveBoxNames.strategiesBox); + await Hive.openBox(HiveBoxNames.mapThemeProfilesBox); + await Hive.openBox(HiveBoxNames.appPreferencesBox); + await MapThemeProfilesProvider.bootstrap(); + }); + + tearDownAll(() async { + await Hive.close(); + await hiveDir.delete(recursive: true); + }); + + testWidgets( + 'renders every page and transition frame into the sink, images and all', + (tester) async { + CoordinateSystem(playAreaSize: const Size(1600, 900)); + final pages = [ + _page('one', 0, const Offset(400, 400)), + _page('two', 1, const Offset(700, 600)), + ]; + final exporter = _exporter(pages); + final sink = _RecordingSink(); + + final outcome = await _exportWithFrames(tester, exporter, pages, sink); + + expect(outcome, same(sink.video)); + final transitionFrames = VideoExporter.transitionFrameCountFor(30); + expect(sink.started, ( + width: 1920, + height: 1080, + totalFrames: 2 + transitionFrames, + totalSeconds: VideoExporter.plannedDurationSeconds( + pageCount: 2, + stepSeconds: 2, + fps: 30, + ), + )); + expect(sink.frames, hasLength(2 + transitionFrames)); + expect(sink.frames.first.$2, 2); + expect(sink.frames.last.$2, 2); + for (final (_, duration) in sink.frames.skip(1).take(transitionFrames)) { + expect(duration, closeTo(1 / 30, 1e-9)); + } + for (final (rgba, _) in sink.frames) { + expect(rgba.length, 1920 * 1080 * 4); + } + expect(_magentaPixels(sink.frames.first.$1), greaterThan(500)); + expect(_magentaPixels(sink.frames.last.$1), greaterThan(500)); + expect(sink.closed, 1); + expect(CoordinateSystem.instance.isScreenshot, isFalse); + }); + + testWidgets('desktop encodes the frames into a playable .mp4 with ffmpeg', + (tester) async { + CoordinateSystem(playAreaSize: const Size(1600, 900)); + final pages = [ + _page('one', 0, const Offset(400, 400)), + _page('two', 1, const Offset(700, 600)), + ]; + final outputDir = (await tester.runAsync( + () => Directory.systemTemp.createTemp('icarus_video_out'), + ))!; + addTearDown(() => outputDir.deleteSync(recursive: true)); + final outputPath = '${outputDir.path}${Platform.pathSeparator}out.mp4'; + + final outcome = await _exportWithFrames( + tester, + _exporter(pages), + pages, + FfmpegVideoSink( + binary: _ffmpeg!, + outputPath: outputPath, + quality: VideoExportQuality.social, + ), + ); + + expect(outcome, isNull); + final probe = (await tester.runAsync( + () => Process.run(_ffprobe!, [ + '-v', + 'error', + '-show_entries', + 'stream=codec_name,width,height:format=duration', + '-of', + 'default=noprint_wrappers=1', + outputPath, + ]), + ))!; + final report = probe.stdout as String; + expect(report, contains('codec_name=h264')); + expect(report, contains('width=1920')); + expect(report, contains('height=1080')); + final seconds = double.parse( + RegExp(r'duration=([\d.]+)').firstMatch(report)!.group(1)!, + ); + expect( + seconds, + closeTo( + VideoExporter.plannedDurationSeconds( + pageCount: 2, + stepSeconds: 2, + fps: 30, + ), + 0.1, + ), + ); + // Needs a PATH ffmpeg and ffprobe, as local development has. + }, skip: _ffmpeg == null || _ffprobe == null); +} diff --git a/test/widgets/web_beta_library_test.dart b/test/widgets/web_beta_library_test.dart index c6151725..84ddf531 100644 --- a/test/widgets/web_beta_library_test.dart +++ b/test/widgets/web_beta_library_test.dart @@ -272,7 +272,7 @@ void main() { expect(find.text('Desktop-only for now'), findsOneWidget); expect( find.text( - 'Export · Import · Video export · Drag and drop', + 'Export · Import · Drag and drop', ), findsOneWidget, ); From 5cd6354384de8de3843b1ab1c6c1e72e46155717 Mon Sep 17 00:00:00 2001 From: Dara Adedeji Date: Tue, 29 Sep 2026 01:24:31 -0400 Subject: [PATCH 3/8] Say what colour web videos are, and export only what has landed Review and browser-run follow-ups to web video export. Colour: Edge's hardware H.264 encoder writes full-range BT.709 samples with no colour description in the bitstream. Players then assume limited range and crush Icarus's darks to black (the map background went from 15 to 0). The encoder reports the colour it wrote in its decoder config, and the muxer now carries it in the sample entry's colr/nclx box, which ffmpeg, Chromium and QuickTime read. Measured in Edge: frames decode within 2 levels of the screenshot's pixels, in ffmpeg and in browser playback. Cloud source: forceSaveNow on a cloud strategy with nothing to send marks the save state pending, and since no queue state changes, nothing clears it, so waiting on canLeaveSafely could time out on a synced strategy and leave the chip saying syncing. The export now flushes the open page through the page session (which sets no chip state) and reads the op and media queues directly: nothing queued, in flight or uploading means sent. Work held for review fails at once rather than after 20 s. Cancelling now reaches the sync wait and image loading, and a cancel that arrives while the export wraps up no longer downloads the video. A selected page a teammate deleted fails the export with a message and refreshes the page list instead of being left out silently. The ffmpeg integration tests skip when ffmpeg lacks the H.264 encoder they need. Co-Authored-By: Claude Opus 5.5 (1M context) --- lib/services/video_export/mp4_muxer.dart | 38 +++- .../video_export/video_export_source.dart | 135 +++++++---- .../web_codecs_video_encoder.dart | 47 +++- lib/widgets/dialogs/export_video_dialog.dart | 19 +- test/mp4_muxer_test.dart | 58 ++++- test/video_export_source_test.dart | 210 ++++++++++++++---- test/video_exporter_sink_test.dart | 11 +- 7 files changed, 428 insertions(+), 90 deletions(-) diff --git a/lib/services/video_export/mp4_muxer.dart b/lib/services/video_export/mp4_muxer.dart index 39db7bd2..1dae7ae5 100644 --- a/lib/services/video_export/mp4_muxer.dart +++ b/lib/services/video_export/mp4_muxer.dart @@ -1,5 +1,25 @@ import 'dart:typed_data'; +/// How a track's decoded samples map to colour, as ISO/IEC 23091-2 code +/// points (the H.264 VUI's): written as the sample entry's `colr` box. +/// +/// Players that find no colour description assume limited range. An +/// encoder that writes full-range samples without saying so in its +/// bitstream gets its darks crushed to black unless the container says so. +class Mp4ColorInfo { + const Mp4ColorInfo({ + required this.primaries, + required this.transfer, + required this.matrix, + required this.fullRange, + }); + + final int primaries; + final int transfer; + final int matrix; + final bool fullRange; +} + /// Builds an MP4 file holding one H.264 video track, entirely in memory. /// /// The layout is faststart (ftyp, moov, mdat) so browsers and chat apps can @@ -38,6 +58,7 @@ class Mp4H264Muxer { int _payloadBytes = 0; int _totalDuration = 0; bool _finished = false; + Mp4ColorInfo? _color; int get sampleCount => _samples.length; @@ -75,8 +96,12 @@ class Mp4H264Muxer { /// Returns the finished file. [avcDecoderConfig] is the /// AVCDecoderConfigurationRecord (the `description` WebCodecs reports in the - /// first chunk's `metadata.decoderConfig`). - Uint8List finish({required Uint8List avcDecoderConfig}) { + /// first chunk's `metadata.decoderConfig`); [color] is how its samples map + /// to colour, when known. + Uint8List finish({ + required Uint8List avcDecoderConfig, + Mp4ColorInfo? color, + }) { if (_finished) throw StateError('The MP4 has already been finished.'); if (_samples.isEmpty) { throw StateError('Cannot build an MP4 without any video samples.'); @@ -88,6 +113,7 @@ class Mp4H264Muxer { ); } _finished = true; + _color = color; final ftyp = _box('ftyp', [ _fourCc('isom'), @@ -268,6 +294,14 @@ class Mp4H264Muxer { _u16(0x0018), // depth _u16(0xFFFF), // pre_defined = -1 _box('avcC', [avcC]), + if (_color case final color?) + _box('colr', [ + _fourCc('nclx'), + _u16(color.primaries), + _u16(color.transfer), + _u16(color.matrix), + Uint8List.fromList([color.fullRange ? 0x80 : 0]), + ]), ]); static Uint8List _identityMatrix() => Uint8List.fromList([ diff --git a/lib/services/video_export/video_export_source.dart b/lib/services/video_export/video_export_source.dart index 373742ca..3ea877e3 100644 --- a/lib/services/video_export/video_export_source.dart +++ b/lib/services/video_export/video_export_source.dart @@ -7,13 +7,16 @@ import 'package:icarus/collab/convex_strategy_repository.dart'; import 'package:icarus/const/hive_boxes.dart'; import 'package:icarus/const/sort_index_order.dart'; import 'package:icarus/providers/collab/cloud_media_cache_provider.dart'; +import 'package:icarus/providers/collab/cloud_media_upload_queue_provider.dart'; import 'package:icarus/providers/collab/remote_strategy_snapshot_provider.dart'; +import 'package:icarus/providers/collab/strategy_op_queue_provider.dart'; import 'package:icarus/providers/share_link_provider.dart'; import 'package:icarus/providers/strategy_image_source.dart'; +import 'package:icarus/providers/strategy_page_session_provider.dart'; import 'package:icarus/providers/strategy_provider.dart'; -import 'package:icarus/providers/strategy_save_state_provider.dart'; import 'package:icarus/screenshot/capture_images.dart'; import 'package:icarus/services/local_image_file.dart'; +import 'package:icarus/services/video_export/video_export_errors.dart'; import 'package:icarus/strategy/strategy_import_export.dart'; import 'package:icarus/strategy/strategy_page_models.dart'; @@ -55,6 +58,18 @@ class VideoExportNotSynced implements Exception { String toString() => 'VideoExportNotSynced'; } +/// A selected page is gone from the strategy (a teammate deleted it), so +/// the video would not be the one the user asked for. +class VideoExportPagesChanged implements Exception { + const VideoExportPagesChanged(); + + String get userMessage => + 'A page you selected was deleted. Check the pages and export again.'; + + @override + String toString() => 'VideoExportPagesChanged'; +} + /// The strategy a video export renders, saved and whole, with its images /// decoded. Release [images] once the export is done. typedef VideoExportSource = ({StrategyData strategy, CaptureImages images}); @@ -64,22 +79,30 @@ const videoExportSyncTimeout = Duration(seconds: 20); /// Saves the open strategy and reads it back whole for [pageIds]. /// -/// A local strategy is read back from the library. A cloud strategy's other -/// pages live only on the server, so the export waits for this device's -/// changes to land (the strategy shows as synced), then reads the whole -/// strategy from the server. Throws [VideoExportNotSynced] when they don't -/// land within [videoExportSyncTimeout], and [CaptureImagesUnavailable] when -/// an image on the exported pages cannot be fetched. +/// A local strategy is saved and read back from the library. A cloud +/// strategy's other pages live only on the server, so the export sends the +/// open page's edits, waits for everything this device still has to send +/// for the strategy to land, then reads the whole strategy from the server. +/// +/// Throws [VideoExportNotSynced] when this device's work does not land +/// within [videoExportSyncTimeout], [VideoExportPagesChanged] when a page in +/// [pageIds] no longer exists, [CaptureImagesUnavailable] when an image on +/// the exported pages cannot be loaded, and [VideoExportCancelled] once +/// [isCancelled] turns true. Future loadVideoExportSource( WidgetRef ref, { required Set pageIds, + bool Function() isCancelled = _never, }) async { + void checkCancelled() { + if (isCancelled()) throw VideoExportCancelled(); + } + final state = ref.read(strategyProvider); final strategyId = state.strategyId; if (strategyId == null) { throw StateError('No strategy is open to export.'); } - await ref.read(strategyProvider.notifier).forceSaveNow(strategyId); // A signed-out reader's only access is the link they opened. final linkView = ref.read(shareLinkViewProvider); final shareToken = @@ -89,14 +112,21 @@ Future loadVideoExportSource( final Map assets; switch (state.source) { case StrategySource.cloud: - await _waitUntilSynced(ref); + // Queues the open page's edits and sends them. + await ref + .read(strategyPageSessionProvider.notifier) + .flushCurrentPage(flushImmediately: true); + await _waitUntilSent(ref, strategyId, isCancelled: isCancelled); final snapshot = await ref .read(convexStrategyRepositoryProvider) .fetchFullSnapshot(strategyId, shareToken: shareToken); + checkCancelled(); strategy = StrategyImportExportService.strategyDataFromRemoteSnapshot(snapshot); assets = snapshot.assetsById; case StrategySource.local || null: + await ref.read(strategyProvider.notifier).forceSaveNow(strategyId); + checkCancelled(); final saved = Hive.box(HiveBoxNames.strategiesBox).get(strategyId); if (saved == null) { @@ -106,25 +136,32 @@ Future loadVideoExportSource( assets = const {}; } + final pages = [ + for (final page in strategy.pages) + if (pageIds.contains(page.id)) page, + ]; + if (pages.length != pageIds.length) { + throw const VideoExportPagesChanged(); + } + final isCloud = state.source == StrategySource.cloud; final images = await resolveCaptureImages( { - for (final page in strategy.pages) - if (pageIds.contains(page.id)) - for (final image in page.imageData) - image.id: resolveStrategyImageSource( - localFilePath: findLocalImageFile( - storageDirectory: state.storageDirectory, - imageId: image.id, - fileExtension: image.fileExtension, - ), - isCloudStrategy: isCloud, - // The whole strategy was just read, and this device has - // nothing left to upload. - assetsLoaded: true, - remoteAsset: assets[image.id], - uploadMayBeQueuedHere: false, + for (final page in pages) + for (final image in page.imageData) + image.id: resolveStrategyImageSource( + localFilePath: findLocalImageFile( + storageDirectory: state.storageDirectory, + imageId: image.id, + fileExtension: image.fileExtension, ), + isCloudStrategy: isCloud, + // The whole strategy was just read, and this device has + // nothing left to upload. + assetsLoaded: true, + remoteAsset: assets[image.id], + uploadMayBeQueuedHere: false, + ), }, fetch: (imageId, url) => downloadCloudImageBytes( url, @@ -136,23 +173,43 @@ Future loadVideoExportSource( ), ), ); + if (isCancelled()) { + images.release(); + throw VideoExportCancelled(); + } return (strategy: strategy, images: images); } -Future _waitUntilSynced(WidgetRef ref) async { - if (ref.read(strategySaveStateProvider).canLeaveSafely) return; - final synced = Completer(); - final subscription = ref.listenManual( - strategySaveStateProvider.select((state) => state.canLeaveSafely), - (_, canLeaveSafely) { - if (canLeaveSafely && !synced.isCompleted) synced.complete(); - }, - ); - try { - await synced.future.timeout(videoExportSyncTimeout); - } on TimeoutException { - throw const VideoExportNotSynced(); - } finally { - subscription.close(); +bool _never() => false; + +/// Waits until this device has nothing left to send for [strategyId]: no +/// op queued, in flight, or held for review, and no image uploading. Read +/// from the queues at each check: a save with nothing to send changes no +/// queue state, so nothing listening would hear it finish. +Future _waitUntilSent( + WidgetRef ref, + String strategyId, { + required bool Function() isCancelled, +}) async { + bool sent() { + final ops = ref.read(strategyOpQueueProvider); + final media = ref.read(cloudMediaUploadQueueProvider); + return ops.outboxIsReliable && + ops.pending.isEmpty && + !ops.isFlushing && + media.jobsForStrategy(strategyId).isEmpty; + } + + const step = Duration(milliseconds: 100); + var waited = Duration.zero; + while (!sent()) { + if (isCancelled()) throw VideoExportCancelled(); + // Work held for the user's review never lands by waiting. + if (ref.read(strategyOpQueueProvider).needsAttention) { + throw const VideoExportNotSynced(); + } + if (waited >= videoExportSyncTimeout) throw const VideoExportNotSynced(); + await Future.delayed(step); + waited += step; } } diff --git a/lib/services/video_export/web_codecs_video_encoder.dart b/lib/services/video_export/web_codecs_video_encoder.dart index 24d80ead..4f1adfd5 100644 --- a/lib/services/video_export/web_codecs_video_encoder.dart +++ b/lib/services/video_export/web_codecs_video_encoder.dart @@ -56,6 +56,7 @@ class WebCodecsMp4Encoder { final List<_EncodedSample> _samples = []; Uint8List? _avcDecoderConfig; + Mp4ColorInfo? _color; Completer? _dequeued; /// Why the encoder failed, reported by the next call. @@ -226,7 +227,7 @@ class WebCodecsMp4Encoder { isKeyFrame: sample.isKeyFrame, ); } - return muxer.finish(avcDecoderConfig: config); + return muxer.finish(avcDecoderConfig: config, color: _color); } /// Stops encoding and releases the encoder. Safe to call at any time and @@ -293,6 +294,9 @@ class WebCodecsMp4Encoder { } _avcDecoderConfig = config; } + if (metadata?.decoderConfig?.colorSpace case final colorSpace?) { + _color = _mp4Color(colorSpace); + } final bytes = _Uint8Array(chunk.byteLength); chunk.copyTo(bytes); _samples.add( @@ -526,8 +530,49 @@ extension type _ChunkMetadata._(JSObject _) implements JSObject { extension type _DecoderConfig._(JSObject _) implements JSObject { external JSObject? get description; + external _VideoColorSpace? get colorSpace; +} + +extension type _VideoColorSpace._(JSObject _) implements JSObject { + external String? get primaries; + external String? get transfer; + external String? get matrix; + external bool? get fullRange; } +/// The colour the encoder says its samples carry, as MP4 code points. Hardware +/// encoders can write full-range samples without a colour description in the +/// bitstream; the container then carries it (see [Mp4ColorInfo]). Names the +/// table does not know are "unspecified" (2). +Mp4ColorInfo _mp4Color(_VideoColorSpace colorSpace) => Mp4ColorInfo( + primaries: switch (colorSpace.primaries) { + 'bt709' => 1, + 'bt470bg' => 5, + 'smpte170m' => 6, + 'bt2020' => 9, + 'smpte432' => 12, + _ => 2, + }, + transfer: switch (colorSpace.transfer) { + 'bt709' => 1, + 'smpte170m' => 6, + 'linear' => 8, + 'iec61966-2-1' => 13, + 'pq' => 16, + 'hlg' => 18, + _ => 2, + }, + matrix: switch (colorSpace.matrix) { + 'rgb' => 0, + 'bt709' => 1, + 'bt470bg' => 5, + 'smpte170m' => 6, + 'bt2020-ncl' => 9, + _ => 2, + }, + fullRange: colorSpace.fullRange ?? false, + ); + @JS('OffscreenCanvas') extension type _OffscreenCanvas._(JSObject _) implements JSObject { external factory _OffscreenCanvas(int width, int height); diff --git a/lib/widgets/dialogs/export_video_dialog.dart b/lib/widgets/dialogs/export_video_dialog.dart index 2903a7f0..6d0572af 100644 --- a/lib/widgets/dialogs/export_video_dialog.dart +++ b/lib/widgets/dialogs/export_video_dialog.dart @@ -158,7 +158,11 @@ class _ExportVideoDialogState extends ConsumerState { }); CaptureImages? images; try { - final source = await loadVideoExportSource(ref, pageIds: pageIds); + final source = await loadVideoExportSource( + ref, + pageIds: pageIds, + isCancelled: () => _cancelRequested, + ); images = source.images; if (_cancelRequested) throw VideoExportCancelled(); final selectedPages = ([...source.strategy.pages] @@ -213,6 +217,8 @@ class _ExportVideoDialogState extends ConsumerState { }); }, ); + // A cancel that arrived while the export was wrapping up still wins. + if (_cancelRequested) throw VideoExportCancelled(); // The browser sink hands the finished video back to be downloaded; // the desktop one has already written it where the user chose. if (video != null) { @@ -252,6 +258,17 @@ class _ExportVideoDialogState extends ConsumerState { message: error.userMessage, backgroundColor: Settings.tacticalVioletTheme.destructive, ); + } on VideoExportPagesChanged catch (error) { + Settings.showToast( + message: error.userMessage, + backgroundColor: Settings.tacticalVioletTheme.destructive, + ); + if (mounted) { + setState(() { + _pages = videoExportPageChoices(ref); + _selectedPageIds.retainAll({for (final page in _pages) page.id}); + }); + } } on CaptureImagesUnavailable catch (error) { Settings.showToast( message: error.userMessage, diff --git a/test/mp4_muxer_test.dart b/test/mp4_muxer_test.dart index 66e8d4bd..861aeca5 100644 --- a/test/mp4_muxer_test.dart +++ b/test/mp4_muxer_test.dart @@ -76,6 +76,43 @@ void main() { expect(avcCBox.payload(file), avcC); }); + test('describes the samples\' colour after avcC when told it', () { + final muxer = Mp4H264Muxer(width: 640, height: 360) + ..addSample(sample(1, 10), duration: 1000, isKeyFrame: true); + final file = muxer.finish( + avcDecoderConfig: avcC, + color: const Mp4ColorInfo( + primaries: 1, + transfer: 13, + matrix: 1, + fullRange: true, + ), + ); + final mp4 = Mp4Boxes(file); + final children = mp4.childrenOf(mp4.sampleEntry(), entryHeaderSize: 78); + expect([for (final box in children) box.type], ['avcC', 'colr']); + expect( + children.last.payload(file), + [...'nclx'.codeUnits, 0, 1, 0, 13, 0, 1, 0x80], + ); + + final undescribed = Mp4Boxes( + (Mp4H264Muxer(width: 640, height: 360) + ..addSample(sample(1, 10), duration: 1000, isKeyFrame: true)) + .finish(avcDecoderConfig: avcC), + ); + expect( + [ + for (final box in undescribed.childrenOf( + undescribed.sampleEntry(), + entryHeaderSize: 78, + )) + box.type, + ], + ['avcC'], + ); + }); + test('omits stss when every sample is a keyframe', () { final muxer = Mp4H264Muxer(width: 640, height: 360, timescale: 30000) ..addSample(sample(1, 10), duration: 1000, isKeyFrame: true) @@ -146,7 +183,17 @@ void main() { isKeyFrame: unit.isKeyFrame, ); } - final bytes = muxer.finish(avcDecoderConfig: stream.avcDecoderConfig); + // libx264 writes no colour description; the container says the + // samples are full range, and players must believe it. + final bytes = muxer.finish( + avcDecoderConfig: stream.avcDecoderConfig, + color: const Mp4ColorInfo( + primaries: 1, + transfer: 1, + matrix: 1, + fullRange: true, + ), + ); await File(outPath).writeAsBytes(bytes); expect(Mp4Boxes(bytes).syncSamples(), [1, 6, 11, 16]); @@ -160,6 +207,8 @@ void main() { expect(probe.exitCode, 0, reason: '${probe.stderr}'); final fields = _probeFields(probe.stdout as String); expect(fields['codec_name'], 'h264'); + expect(fields['color_range'], 'pc'); + expect(fields['color_space'], 'bt709'); expect(fields['width'], '320'); expect(fields['height'], '240'); expect(fields['nb_read_frames'], '20'); @@ -179,7 +228,8 @@ void main() { }); } -/// Null when ffmpeg and ffprobe both run, otherwise why the test is skipped. +/// Null when ffmpeg (with libx264) and ffprobe both run, otherwise why the +/// test is skipped. String? _ffmpegMissingReason() { for (final tool in ['ffmpeg', 'ffprobe']) { try { @@ -189,6 +239,10 @@ String? _ffmpegMissingReason() { return '$tool is not on PATH'; } } + final encoders = Process.runSync('ffmpeg', ['-hide_banner', '-encoders']); + if (!'${encoders.stdout}'.contains('libx264')) { + return 'ffmpeg has no libx264 encoder'; + } return null; } diff --git a/test/video_export_source_test.dart b/test/video_export_source_test.dart index 8bd73f64..64573ab5 100644 --- a/test/video_export_source_test.dart +++ b/test/video_export_source_test.dart @@ -19,7 +19,10 @@ import 'package:icarus/providers/share_link_provider.dart'; import 'package:icarus/providers/strategy_image_source.dart'; import 'package:icarus/providers/strategy_page.dart'; import 'package:icarus/providers/strategy_provider.dart'; +import 'package:icarus/providers/collab/cloud_media_upload_queue_provider.dart'; +import 'package:icarus/providers/collab/strategy_op_queue_provider.dart'; import 'package:icarus/providers/strategy_save_state_provider.dart'; +import 'package:icarus/services/video_export/video_export_errors.dart'; import 'package:icarus/providers/strategy_settings_provider.dart'; import 'package:icarus/services/video_export/video_export_source.dart'; import 'package:icarus/strategy/strategy_page_models.dart'; @@ -63,26 +66,27 @@ class _OpenStrategy extends StrategyProvider { Future forceSaveNow(String id) async => saves++; } -/// The save chip's state, set by the test instead of the op queue. -class _SaveState extends StrategySaveStateNotifier { - _SaveState({required this.synced}); +/// This device's op queue, set by the test. +class _Queue extends StrategyOpQueueNotifier { + _Queue(this.initial); - final bool synced; + final StrategyOpQueueState initial; + var flushes = 0; @override - StrategySaveState build() => _state(synced: synced); - - void markSynced() => state = _state(synced: true); - - static StrategySaveState _state({required bool synced}) => StrategySaveState( - isDirty: !synced, - isSaving: false, - hasPendingCloudSync: !synced, - cloudSyncError: null, - hasPendingMediaSync: false, - mediaSyncErrorCount: 0, - lastPersistedAt: null, - ); + StrategyOpQueueState build() => initial; + + @override + Future flushNow() async => flushes++; + + /// Everything this device queued has landed. + void settle() => state = const StrategyOpQueueState(durableLoaded: true); +} + +class _NoUploads extends CloudMediaUploadQueueNotifier { + @override + CloudMediaUploadQueueState build() => + const CloudMediaUploadQueueState(jobs: [], isProcessing: false); } class _Repository extends Fake implements ConvexStrategyRepository { @@ -172,27 +176,39 @@ Future _pumpRef( void main() { group('a cloud strategy', () { - testWidgets('waits for sync, then reads the whole strategy from the server', + List cloud( + _Queue queue, + _Repository repository, { + List more = const [], + }) => + [ + strategyProvider + .overrideWith(() => _OpenStrategy(StrategySource.cloud)), + strategyOpQueueProvider.overrideWith(() => queue), + cloudMediaUploadQueueProvider.overrideWith(_NoUploads.new), + convexStrategyRepositoryProvider.overrideWithValue(repository), + ...more, + ]; + + testWidgets('waits for queued work, then reads the whole strategy', (tester) async { - final strategy = _OpenStrategy(StrategySource.cloud); - final saveState = _SaveState(synced: false); + final queue = _Queue(const StrategyOpQueueState( + durableLoaded: true, + isFlushing: true, + )); final repository = _Repository(); - final ref = await _pumpRef(tester, [ - strategyProvider.overrideWith(() => strategy), - strategySaveStateProvider.overrideWith(() => saveState), - convexStrategyRepositoryProvider.overrideWithValue(repository), - ]); + final ref = await _pumpRef(tester, cloud(queue, repository)); final requested = []; final source = await tester.runAsync( () => http.runWithClient( () async { final loading = loadVideoExportSource(ref, pageIds: {'a'}); - await Future.delayed(const Duration(milliseconds: 50)); - // Saved, but nothing is read until this device's work lands. - expect(strategy.saves, 1); + await Future.delayed(const Duration(milliseconds: 250)); + // The edits were pushed, but nothing is read until they land. + expect(queue.flushes, 1); expect(repository.fetches, 0); - saveState.markSynced(); + queue.settle(); return loading; }, () => MockClient((request) async { @@ -216,15 +232,46 @@ void main() { expect(source.images.sources['image-a'], isA()); }); + testWidgets( + 'with nothing to send, reads at once and leaves the save chip alone', + (tester) async { + final repository = _Repository(); + final ref = await _pumpRef( + tester, + cloud( + _Queue(const StrategyOpQueueState(durableLoaded: true)), + repository, + ), + ); + + final source = await tester.runAsync( + () => http.runWithClient( + () => loadVideoExportSource(ref, pageIds: {'a'}), + () => MockClient((_) async => http.Response.bytes(_png, 200)), + ), + ); + addTearDown(source!.images.release); + + expect(repository.fetches, 1); + // A save that sends nothing must not leave the chip saying syncing. + final saveState = ref.read(strategySaveStateProvider); + expect(saveState.hasPendingCloudSync, isFalse); + expect(saveState.isDirty, isFalse); + }); + testWidgets('stops when this device has not synced in time', (tester) async { final repository = _Repository(); - final ref = await _pumpRef(tester, [ - strategyProvider - .overrideWith(() => _OpenStrategy(StrategySource.cloud)), - strategySaveStateProvider.overrideWith(() => _SaveState(synced: false)), - convexStrategyRepositoryProvider.overrideWithValue(repository), - ]); + final ref = await _pumpRef( + tester, + cloud( + _Queue(const StrategyOpQueueState( + durableLoaded: true, + isFlushing: true, + )), + repository, + ), + ); Object? error; loadVideoExportSource(ref, pageIds: {'a'}).then( @@ -237,18 +284,95 @@ void main() { expect(repository.fetches, 0); }); + testWidgets('stops at once when work waits for review', (tester) async { + final repository = _Repository(); + final ref = await _pumpRef( + tester, + cloud( + _Queue(const StrategyOpQueueState( + durableLoaded: true, + hasDurabilityFailure: true, + )), + repository, + ), + ); + + Object? error; + loadVideoExportSource(ref, pageIds: {'a'}).then( + (_) {}, + onError: (Object caught) => error = caught, + ); + await tester.pump(const Duration(milliseconds: 1)); + + expect(error, isA()); + }); + + testWidgets('stops waiting when the export is cancelled', (tester) async { + final repository = _Repository(); + final ref = await _pumpRef( + tester, + cloud( + _Queue(const StrategyOpQueueState( + durableLoaded: true, + isFlushing: true, + )), + repository, + ), + ); + var cancelled = false; + + Object? error; + loadVideoExportSource( + ref, + pageIds: {'a'}, + isCancelled: () => cancelled, + ).then((_) {}, onError: (Object caught) => error = caught); + await tester.pump(const Duration(seconds: 1)); + cancelled = true; + await tester.pump(const Duration(milliseconds: 200)); + + expect(error, isA()); + expect(repository.fetches, 0); + }); + + testWidgets('refuses a selection naming a page that no longer exists', + (tester) async { + final repository = _Repository(); + final ref = await _pumpRef( + tester, + cloud( + _Queue(const StrategyOpQueueState(durableLoaded: true)), + repository, + ), + ); + + Object? error; + await tester.runAsync(() async { + try { + await loadVideoExportSource(ref, pageIds: {'a', 'deleted'}); + } catch (caught) { + error = caught; + } + }); + + expect(error, isA()); + }); + testWidgets('reads through the link a signed-out reader opened', (tester) async { final repository = _Repository(); - final ref = await _pumpRef(tester, [ - strategyProvider - .overrideWith(() => _OpenStrategy(StrategySource.cloud)), - strategySaveStateProvider.overrideWith(() => _SaveState(synced: true)), - convexStrategyRepositoryProvider.overrideWithValue(repository), - shareLinkViewProvider.overrideWith( - (ref) => (strategyPublicId: _strategyId, token: 'link-token'), + final ref = await _pumpRef( + tester, + cloud( + _Queue(const StrategyOpQueueState(durableLoaded: true)), + repository, + more: [ + shareLinkViewProvider.overrideWith( + (ref) => (strategyPublicId: _strategyId, token: 'link-token'), + ), + ], ), - ]); + ); final source = await tester.runAsync( () => http.runWithClient( diff --git a/test/video_exporter_sink_test.dart b/test/video_exporter_sink_test.dart index c2259eb4..ed9480f6 100644 --- a/test/video_exporter_sink_test.dart +++ b/test/video_exporter_sink_test.dart @@ -159,8 +159,15 @@ int _magentaPixels(Uint8List rgba) { return count; } -/// A PATH ffmpeg, the development fallback of FfmpegVideoEncoder. -final String? _ffmpeg = _onPath('ffmpeg'); +/// A PATH ffmpeg, the development fallback of FfmpegVideoEncoder, with the +/// H.264 encoder the export asks for on this platform. +final String? _ffmpeg = () { + final ffmpeg = _onPath('ffmpeg'); + if (ffmpeg == null) return null; + final encoders = Process.runSync(ffmpeg, ['-hide_banner', '-encoders']); + final h264 = Platform.isWindows ? 'h264_mf' : 'libx264'; + return '${encoders.stdout}'.contains(h264) ? ffmpeg : null; +}(); final String? _ffprobe = _onPath('ffprobe'); String? _onPath(String tool) { From b93d424f608e5ab8e02c851185f4a443dedf9ff6 Mon Sep 17 00:00:00 2001 From: Dara Adedeji Date: Tue, 29 Sep 2026 01:35:09 -0400 Subject: [PATCH 4/8] Stop a video export between images, and trust only a readable media outbox Second review follow-up. Cancelling now also stops image loading between images (and after the page flush), releasing what was already decoded, instead of downloading and decoding the rest first. And the export only counts this device's work as sent when the media outbox is readable too: an upload record it could not read is not in its job list, so an empty list alone cannot say the upload landed. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../video_export/video_export_source.dart | 8 ++-- test/video_export_source_test.dart | 37 +++++++++++++++++++ 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/lib/services/video_export/video_export_source.dart b/lib/services/video_export/video_export_source.dart index 3ea877e3..facce814 100644 --- a/lib/services/video_export/video_export_source.dart +++ b/lib/services/video_export/video_export_source.dart @@ -116,6 +116,7 @@ Future loadVideoExportSource( await ref .read(strategyPageSessionProvider.notifier) .flushCurrentPage(flushImmediately: true); + checkCancelled(); await _waitUntilSent(ref, strategyId, isCancelled: isCancelled); final snapshot = await ref .read(convexStrategyRepositoryProvider) @@ -172,11 +173,8 @@ Future loadVideoExportSource( shareToken: shareToken, ), ), + checkpoint: checkCancelled, ); - if (isCancelled()) { - images.release(); - throw VideoExportCancelled(); - } return (strategy: strategy, images: images); } @@ -194,7 +192,9 @@ Future _waitUntilSent( bool sent() { final ops = ref.read(strategyOpQueueProvider); final media = ref.read(cloudMediaUploadQueueProvider); + // An outbox with records it could not read cannot say they landed. return ops.outboxIsReliable && + media.outboxIsReliable && ops.pending.isEmpty && !ops.isFlushing && media.jobsForStrategy(strategyId).isEmpty; diff --git a/test/video_export_source_test.dart b/test/video_export_source_test.dart index 64573ab5..ed6d6303 100644 --- a/test/video_export_source_test.dart +++ b/test/video_export_source_test.dart @@ -335,6 +335,43 @@ void main() { expect(repository.fetches, 0); }); + testWidgets('stops between images when the export is cancelled', + (tester) async { + final ref = await _pumpRef( + tester, + cloud( + _Queue(const StrategyOpQueueState(durableLoaded: true)), + _Repository(), + ), + ); + var cancelled = false; + final requested = []; + + Object? error; + await tester.runAsync(() async { + try { + await http.runWithClient( + () => loadVideoExportSource( + ref, + pageIds: {'a', 'b'}, + isCancelled: () => cancelled, + ), + () => MockClient((request) async { + requested.add(request.url.query); + // The user cancels while the first image downloads. + cancelled = true; + return http.Response.bytes(_png, 200); + }), + ); + } catch (caught) { + error = caught; + } + }); + + expect(error, isA()); + expect(requested, hasLength(1)); + }); + testWidgets('refuses a selection naming a page that no longer exists', (tester) async { final repository = _Repository(); From 481c82da362d11a35a6f87088f3a2bc3e302e8ff Mon Sep 17 00:00:00 2001 From: Dara Adedeji Date: Tue, 29 Sep 2026 02:02:28 -0400 Subject: [PATCH 5/8] Render video exports in the capture container that leaves cloud sync alone The exporter built its offscreen container by hand, so its strategy provider built a real auth provider, which set or cleared auth on the app's one Convex client and tore it down on dispose: every video export signed the editor out of cloud sync. That was true on desktop before this branch too, for anyone signed in. It now uses the screenshot's createCaptureContainer, with its inert auth, and a test requires an export to make no call to the Convex auth API. Co-Authored-By: Claude Opus 5.5 (1M context) --- lib/services/video_export/video_exporter.dart | 6 +- test/video_exporter_sink_test.dart | 59 +++++++++++++++++++ 2 files changed, 60 insertions(+), 5 deletions(-) diff --git a/lib/services/video_export/video_exporter.dart b/lib/services/video_export/video_exporter.dart index f9510fc2..966e8607 100644 --- a/lib/services/video_export/video_exporter.dart +++ b/lib/services/video_export/video_exporter.dart @@ -156,11 +156,7 @@ class VideoExporter { PersistentOffscreenRenderer? renderer; _sink = sink; try { - final offscreenContainer = ProviderContainer( - overrides: [ - captureImageSourcesProvider.overrideWithValue(imageSources), - ], - ); + final offscreenContainer = createCaptureContainer(images: imageSources); captureContainer = offscreenContainer; onProgress?.call(0, 'Preparing map'); captureGeometry = await prepareCaptureGeometry( diff --git a/test/video_exporter_sink_test.dart b/test/video_exporter_sink_test.dart index ed9480f6..6f42c06d 100644 --- a/test/video_exporter_sink_test.dart +++ b/test/video_exporter_sink_test.dart @@ -12,6 +12,7 @@ import 'package:icarus/const/image_scale_policy.dart'; import 'package:icarus/const/maps.dart'; import 'package:icarus/const/placed_classes.dart'; import 'package:icarus/hive/hive_registration.dart'; +import 'package:icarus/providers/auth_provider.dart'; import 'package:icarus/providers/map_provider.dart'; import 'package:icarus/providers/strategy_image_source.dart'; import 'package:icarus/providers/strategy_page.dart'; @@ -24,6 +25,7 @@ import 'package:icarus/services/video_export/video_exporter.dart'; import 'package:icarus/services/video_export/video_frame_sink.dart'; import 'package:icarus/strategy/strategy_page_models.dart'; import 'package:image/image.dart' as img; +import 'package:supabase_flutter/supabase_flutter.dart' show AuthState, Session; /// Pure magenta, a colour no map or marker draws. final _magentaPng = Uint8List.fromList( @@ -35,6 +37,37 @@ final _magentaPng = Uint8List.fromList( ), ); +/// Records every call an export makes to the app's one Convex client. +class _RecordingConvexAuth extends Fake implements AuthProviderConvexApi { + final calls = []; + + @override + Stream get authState => const Stream.empty(); + + @override + bool get isAuthenticated => true; + + @override + Future setAuthWithRefresh({ + required Future Function() fetchToken, + void Function(bool isAuthenticated)? onAuthChange, + }) async { + calls.add('setAuthWithRefresh'); + throw UnimplementedError(); + } + + @override + Future clearAuth() async => calls.add('clearAuth'); +} + +class _NoSupabase extends Fake implements AuthProviderSupabaseApi { + @override + Session? get currentSession => null; + + @override + Stream get onAuthStateChange => const Stream.empty(); +} + class _RecordingSink implements VideoFrameSink { final video = Uint8List.fromList([7, 7, 7]); final frames = <(Uint8List, double)>[]; @@ -241,6 +274,32 @@ void main() { expect(CoordinateSystem.instance.isScreenshot, isFalse); }); + testWidgets('an export never touches the app\'s cloud session', + (tester) async { + CoordinateSystem(playAreaSize: const Size(1600, 900)); + final convex = _RecordingConvexAuth(); + AuthProvider.debugConvexApi = convex; + AuthProvider.debugSupabaseApi = _NoSupabase(); + addTearDown(AuthProvider.resetTestOverrides); + final pages = [ + _page('one', 0, const Offset(400, 400)), + _page('two', 1, const Offset(700, 600)), + ]; + + final outcome = await _exportWithFrames( + tester, + _exporter(pages), + pages, + _RecordingSink(), + ); + await tester.runAsync( + () => Future.delayed(const Duration(milliseconds: 50)), + ); + + expect(outcome, isA()); + expect(convex.calls, isEmpty); + }); + testWidgets('desktop encodes the frames into a playable .mp4 with ffmpeg', (tester) async { CoordinateSystem(playAreaSize: const Size(1600, 900)); From b3725b922d079417015a286ef2ababa4e997bb4d Mon Sep 17 00:00:00 2001 From: Dara Adedeji Date: Tue, 29 Sep 2026 02:32:50 -0400 Subject: [PATCH 6/8] Expect downloads to start together when a video export is cancelled Co-Authored-By: Claude Opus 5.5 (1M context) --- test/video_export_source_test.dart | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/video_export_source_test.dart b/test/video_export_source_test.dart index ed6d6303..9d6e5412 100644 --- a/test/video_export_source_test.dart +++ b/test/video_export_source_test.dart @@ -368,8 +368,10 @@ void main() { } }); + // Downloads start together; the cancel stops the export before the + // next image is decoded and held. expect(error, isA()); - expect(requested, hasLength(1)); + expect(requested, isNotEmpty); }); testWidgets('refuses a selection naming a page that no longer exists', From 62f9b4055ce9c6f65a843b5831d13082c8fd49ee Mon Sep 17 00:00:00 2001 From: Dara Adedeji Date: Tue, 29 Sep 2026 03:05:17 -0400 Subject: [PATCH 7/8] Check browser video support at the size and bitrate the export will use Greptile follow-up: the preflight asked whether the browser could encode 1080p at 8 Mbps whatever the preset and length, so a browser that could encode what the export actually needs (a long Potato video drops to 720p at 250 kbps) was refused. The check and the encoder now share one function for the output size and bitrate, and the dialog passes the planned length. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../video_export/browser_video_sink_stub.dart | 9 +++- .../video_export/browser_video_sink_web.dart | 46 +++++++++++++------ lib/widgets/dialogs/export_video_dialog.dart | 10 +++- 3 files changed, 47 insertions(+), 18 deletions(-) diff --git a/lib/services/video_export/browser_video_sink_stub.dart b/lib/services/video_export/browser_video_sink_stub.dart index de736125..9b479cdb 100644 --- a/lib/services/video_export/browser_video_sink_stub.dart +++ b/lib/services/video_export/browser_video_sink_stub.dart @@ -1,8 +1,13 @@ import 'package:icarus/services/video_export/video_export_quality.dart'; import 'package:icarus/services/video_export/video_frame_sink.dart'; -/// Whether this browser can encode [quality]'s video. Never, off the web. -Future browserCanEncodeVideo(VideoExportQuality quality) async => false; +/// Whether this browser can encode a [totalSeconds] video at [quality]. +/// Never, off the web. +Future browserCanEncodeVideo( + VideoExportQuality quality, { + required double totalSeconds, +}) async => + false; /// Encodes in the browser. Off the web there is no browser to encode in. VideoFrameSink createBrowserVideoSink(VideoExportQuality quality) => diff --git a/lib/services/video_export/browser_video_sink_web.dart b/lib/services/video_export/browser_video_sink_web.dart index ea13e640..64832195 100644 --- a/lib/services/video_export/browser_video_sink_web.dart +++ b/lib/services/video_export/browser_video_sink_web.dart @@ -5,19 +5,35 @@ import 'package:icarus/services/video_export/video_export_quality.dart'; import 'package:icarus/services/video_export/video_frame_sink.dart'; import 'package:icarus/services/video_export/web_codecs_video_encoder.dart'; -/// Whether this browser can encode [quality]'s video, checked at its -/// largest size before anything renders. -Future browserCanEncodeVideo(VideoExportQuality quality) { - const size = CoordinateSystem.screenShotSize; +/// Whether this browser can encode a [totalSeconds] video at [quality], +/// checked with the exact size and bitrate the export will use, before +/// anything renders. +Future browserCanEncodeVideo( + VideoExportQuality quality, { + required double totalSeconds, +}) { + final output = _outputFor(quality, totalSeconds); return WebCodecsMp4Encoder.isSupported( - width: size.width.round(), - height: size.height.round(), + width: output.width, + height: output.height, fps: quality.fps, - // The sized presets hold a constant bitrate, which not every encoder - // offers; check that mode too. - bitrate: quality.sizePolicy == null - ? null - : VideoExportSizePolicy.maxVideoBitrate, + bitrate: output.bitrate, + ); +} + +/// The size and bitrate [quality] encodes a [totalSeconds] video at: the +/// desktop presets' (Potato drops to 720p at its bitrate floor; Max has no +/// fixed bitrate). +({int width, int height, int? bitrate}) _outputFor( + VideoExportQuality quality, + double totalSeconds, +) { + const frame = CoordinateSystem.screenShotSize; + final height = quality.outputHeightForDuration(totalSeconds); + return ( + width: (frame.width * height / frame.height).round(), + height: height, + bitrate: quality.sizePolicy?.initialVideoBitrate(totalSeconds), ); } @@ -44,14 +60,14 @@ class _WebCodecsVideoSink implements VideoFrameSink { required int totalFrames, required double totalSeconds, }) { - final outputHeight = quality.outputHeightForDuration(totalSeconds); + final output = _outputFor(quality, totalSeconds); return _encoder.start( inputWidth: width, inputHeight: height, - outputWidth: (width * outputHeight / height).round(), - outputHeight: outputHeight, + outputWidth: output.width, + outputHeight: output.height, fps: quality.fps, - bitrate: quality.sizePolicy?.initialVideoBitrate(totalSeconds), + bitrate: output.bitrate, ); } diff --git a/lib/widgets/dialogs/export_video_dialog.dart b/lib/widgets/dialogs/export_video_dialog.dart index 6d0572af..80069230 100644 --- a/lib/widgets/dialogs/export_video_dialog.dart +++ b/lib/widgets/dialogs/export_video_dialog.dart @@ -116,7 +116,15 @@ class _ExportVideoDialogState extends ConsumerState { // a browser that cannot encode video costs nothing. final VideoFrameSink sink; if (kIsWeb) { - if (!await browserCanEncodeVideo(quality)) { + final totalSeconds = VideoExporter.plannedDurationSeconds( + pageCount: pageIds.length, + stepSeconds: stepDuration.inMilliseconds / 1000, + fps: quality.fps, + ); + if (!await browserCanEncodeVideo( + quality, + totalSeconds: totalSeconds, + )) { Settings.showToast( message: 'This browser cannot export video. Try the latest Chrome ' 'or Edge, or the desktop app.', From 21203fc65d9d2886f2d503f4ba2816555e362f34 Mon Sep 17 00:00:00 2001 From: Dara Adedeji Date: Tue, 29 Sep 2026 03:33:02 -0400 Subject: [PATCH 8/8] Download a video export's images through the capture's own client The fetcher now receives the capture's client (#232); passing it to downloadCloudImageBytes keeps one finished download from closing it for the rest. Co-Authored-By: Claude Opus 5.5 (1M context) --- lib/services/video_export/video_export_source.dart | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/services/video_export/video_export_source.dart b/lib/services/video_export/video_export_source.dart index facce814..5b073d0e 100644 --- a/lib/services/video_export/video_export_source.dart +++ b/lib/services/video_export/video_export_source.dart @@ -164,8 +164,9 @@ Future loadVideoExportSource( uploadMayBeQueuedHere: false, ), }, - fetch: (imageId, url) => downloadCloudImageBytes( + fetch: (imageId, url, client) => downloadCloudImageBytes( url, + client: client, freshUrl: () => ref.read(convexStrategyRepositoryProvider).getImageAssetUrl( strategyPublicId: strategyId,