diff --git a/lib/collab/convex_strategy_repository.dart b/lib/collab/convex_strategy_repository.dart index 169979f5..96629b51 100644 --- a/lib/collab/convex_strategy_repository.dart +++ b/lib/collab/convex_strategy_repository.dart @@ -157,12 +157,17 @@ class ConvexStrategyRepository { .map(_pageSnapshot); } + /// [shareToken]: see [fetchShell]. Future fetchFullSnapshot( - String strategyPublicId, - ) async { + String strategyPublicId, { + String? shareToken, + }) async { return _fullSnapshot( await _api.strategy - .getFullSnapshot(strategyPublicId: strategyPublicId) + .getFullSnapshot( + strategyPublicId: strategyPublicId, + shareToken: _optional(shareToken), + ) .fetch(), ); } diff --git a/lib/config/platform_policy.dart b/lib/config/platform_policy.dart index c5b228c3..3adcab4d 100644 --- a/lib/config/platform_policy.dart +++ b/lib/config/platform_policy.dart @@ -47,7 +47,6 @@ class PlatformPolicy { desktopOnly: { PlatformFeature.exportFiles, PlatformFeature.importFiles, - PlatformFeature.videoExport, PlatformFeature.fileDrop, }, // Empty for now; the Beta dialog hides its "Coming" list when it is. diff --git a/lib/services/video_export/browser_video_sink.dart b/lib/services/video_export/browser_video_sink.dart new file mode 100644 index 00000000..0b45f1dc --- /dev/null +++ b/lib/services/video_export/browser_video_sink.dart @@ -0,0 +1,2 @@ +export 'browser_video_sink_stub.dart' + if (dart.library.js_interop) 'browser_video_sink_web.dart'; diff --git a/lib/services/video_export/browser_video_sink_stub.dart b/lib/services/video_export/browser_video_sink_stub.dart new file mode 100644 index 00000000..9b479cdb --- /dev/null +++ b/lib/services/video_export/browser_video_sink_stub.dart @@ -0,0 +1,14 @@ +import 'package:icarus/services/video_export/video_export_quality.dart'; +import 'package:icarus/services/video_export/video_frame_sink.dart'; + +/// Whether this browser can encode a [totalSeconds] video at [quality]. +/// Never, off the web. +Future browserCanEncodeVideo( + VideoExportQuality quality, { + required double totalSeconds, +}) async => + false; + +/// Encodes in the browser. Off the web there is no browser to encode in. +VideoFrameSink createBrowserVideoSink(VideoExportQuality quality) => + throw UnsupportedError('Browser video encoding exists only on the web.'); diff --git a/lib/services/video_export/browser_video_sink_web.dart b/lib/services/video_export/browser_video_sink_web.dart new file mode 100644 index 00000000..64832195 --- /dev/null +++ b/lib/services/video_export/browser_video_sink_web.dart @@ -0,0 +1,95 @@ +import 'dart:typed_data'; + +import 'package:icarus/const/coordinate_system.dart'; +import 'package:icarus/services/video_export/video_export_quality.dart'; +import 'package:icarus/services/video_export/video_frame_sink.dart'; +import 'package:icarus/services/video_export/web_codecs_video_encoder.dart'; + +/// Whether this browser can encode a [totalSeconds] video at [quality], +/// checked with the exact size and bitrate the export will use, before +/// anything renders. +Future browserCanEncodeVideo( + VideoExportQuality quality, { + required double totalSeconds, +}) { + final output = _outputFor(quality, totalSeconds); + return WebCodecsMp4Encoder.isSupported( + width: output.width, + height: output.height, + fps: quality.fps, + bitrate: output.bitrate, + ); +} + +/// The size and bitrate [quality] encodes a [totalSeconds] video at: the +/// desktop presets' (Potato drops to 720p at its bitrate floor; Max has no +/// fixed bitrate). +({int width, int height, int? bitrate}) _outputFor( + VideoExportQuality quality, + double totalSeconds, +) { + const frame = CoordinateSystem.screenShotSize; + final height = quality.outputHeightForDuration(totalSeconds); + return ( + width: (frame.width * height / frame.height).round(), + height: height, + bitrate: quality.sizePolicy?.initialVideoBitrate(totalSeconds), + ); +} + +/// Encodes in the browser with WebCodecs; [VideoFrameSink.finish] returns +/// the finished .mp4 for the caller to download. +VideoFrameSink createBrowserVideoSink(VideoExportQuality quality) => + _WebCodecsVideoSink(quality); + +/// Sizes and bitrates follow the desktop presets: Potato and Social hold the +/// bitrate their size target allows for the video's length (and Potato +/// drops to 720p at the bitrate floor); Max encodes at a high variable +/// bitrate. The browser encodes as frames arrive, so there is no separate +/// encode pass and no second attempt at a smaller size. +class _WebCodecsVideoSink implements VideoFrameSink { + _WebCodecsVideoSink(this.quality); + + final VideoExportQuality quality; + final WebCodecsMp4Encoder _encoder = WebCodecsMp4Encoder(); + + @override + Future start({ + required int width, + required int height, + required int totalFrames, + required double totalSeconds, + }) { + final output = _outputFor(quality, totalSeconds); + return _encoder.start( + inputWidth: width, + inputHeight: height, + outputWidth: output.width, + outputHeight: output.height, + fps: quality.fps, + bitrate: output.bitrate, + ); + } + + @override + Future addFrame(Uint8List rgba, double durationSeconds) => + _encoder.addFrame( + rgba, + duration: Duration(microseconds: (durationSeconds * 1e6).round()), + ); + + @override + Future finish({ + void Function(double fraction)? onProgress, + }) async { + final video = await _encoder.finish(); + onProgress?.call(1); + return video; + } + + @override + void cancel() => _encoder.cancel(); + + @override + Future close() async => _encoder.cancel(); +} diff --git a/lib/services/video_export/ffmpeg_video_encoder.dart b/lib/services/video_export/ffmpeg_video_encoder.dart index 3816f82c..746a943b 100644 --- a/lib/services/video_export/ffmpeg_video_encoder.dart +++ b/lib/services/video_export/ffmpeg_video_encoder.dart @@ -3,18 +3,11 @@ import 'dart:convert'; import 'dart:developer'; import 'dart:io'; +import 'package:icarus/services/video_export/video_export_errors.dart'; import 'package:icarus/services/video_export/video_export_quality.dart'; import 'package:path/path.dart' as p; -class VideoExportCancelled implements Exception {} - -class VideoExportException implements Exception { - VideoExportException(this.message); - final String message; - - @override - String toString() => 'VideoExportException: $message'; -} +export 'package:icarus/services/video_export/video_export_errors.dart'; /// Windows' save dialog does not guarantee that the selected extension is /// appended, even when FilePicker is restricted to MP4 files. diff --git a/lib/services/video_export/ffmpeg_video_sink.dart b/lib/services/video_export/ffmpeg_video_sink.dart new file mode 100644 index 00000000..0da0087d --- /dev/null +++ b/lib/services/video_export/ffmpeg_video_sink.dart @@ -0,0 +1,116 @@ +import 'dart:async'; +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:icarus/services/video_export/ffmpeg_png_sequence_writer.dart'; +import 'package:icarus/services/video_export/ffmpeg_video_encoder.dart'; +import 'package:icarus/services/video_export/video_export_quality.dart'; +import 'package:icarus/services/video_export/video_frame_sink.dart'; +import 'package:path/path.dart' as p; + +/// Desktop: frames go to one ffmpeg process as a lossless PNG sequence, +/// listed with their durations in an ffconcat playlist, and a second ffmpeg +/// run encodes the playlist into [outputPath]. +class FfmpegVideoSink implements VideoFrameSink { + FfmpegVideoSink({ + required this.binary, + required this.outputPath, + required this.quality, + }); + + final String binary; + final String outputPath; + final VideoExportQuality quality; + + final FfmpegVideoEncoder _encoder = FfmpegVideoEncoder(); + final List _concatLines = ['ffconcat version 1.0']; + Directory? _tempDir; + FfmpegPngSequenceWriter? _frameWriter; + double _totalSeconds = 0; + int _frameIndex = 0; + String? _lastFrameFile; + + @override + Future start({ + required int width, + required int height, + required int totalFrames, + required double totalSeconds, + }) async { + _totalSeconds = totalSeconds; + final tempDir = + await Directory.systemTemp.createTemp('icarus_video_export_'); + _tempDir = tempDir; + final frameWriter = FfmpegPngSequenceWriter(); + _frameWriter = frameWriter; + await frameWriter.start( + binary: binary, + workingDirectory: tempDir.path, + width: width, + height: height, + fps: quality.fps, + totalFrames: totalFrames, + ); + } + + @override + Future addFrame(Uint8List rgba, double durationSeconds) async { + final fileName = 'frame_${_frameIndex.toString().padLeft(5, '0')}.png'; + await _frameWriter!.writeFrame(rgba); + _frameIndex++; + _concatLines + ..add("file '$fileName'") + ..add('duration ${durationSeconds.toStringAsFixed(6)}'); + _lastFrameFile = fileName; + } + + @override + Future finish( + {void Function(double fraction)? onProgress}) async { + await _frameWriter!.finish(); + _frameWriter = null; + + // The concat demuxer ignores the duration of the final entry unless the + // last file is repeated. + if (_lastFrameFile != null) { + _concatLines.add("file '$_lastFrameFile'"); + } + final tempDir = _tempDir!; + await File(p.join(tempDir.path, 'frames.ffconcat')) + .writeAsString(_concatLines.join('\n')); + + await _encoder.encode( + binary: binary, + workingDirectory: tempDir.path, + concatListFileName: 'frames.ffconcat', + outputPath: outputPath, + totalSeconds: _totalSeconds, + quality: quality, + onProgress: onProgress, + ); + return null; + } + + @override + void cancel() { + // close() awaits the writer's termination; here we only need to + // interrupt the processes. + unawaited(_frameWriter?.cancel()); + _encoder.cancel(); + } + + @override + Future close() async { + // Wait for ffmpeg to exit before deleting the directory it writes into; + // on Windows the delete races a still-exiting process otherwise. + await _frameWriter?.cancel(); + _frameWriter = null; + try { + await _tempDir?.delete(recursive: true); + _tempDir = null; + } on Object { + // Leaving orphaned temp frames behind is preferable to masking the + // original export result. + } + } +} diff --git a/lib/services/video_export/mp4_muxer.dart b/lib/services/video_export/mp4_muxer.dart new file mode 100644 index 00000000..1dae7ae5 --- /dev/null +++ b/lib/services/video_export/mp4_muxer.dart @@ -0,0 +1,357 @@ +import 'dart:typed_data'; + +/// How a track's decoded samples map to colour, as ISO/IEC 23091-2 code +/// points (the H.264 VUI's): written as the sample entry's `colr` box. +/// +/// Players that find no colour description assume limited range. An +/// encoder that writes full-range samples without saying so in its +/// bitstream gets its darks crushed to black unless the container says so. +class Mp4ColorInfo { + const Mp4ColorInfo({ + required this.primaries, + required this.transfer, + required this.matrix, + required this.fullRange, + }); + + final int primaries; + final int transfer; + final int matrix; + final bool fullRange; +} + +/// Builds an MP4 file holding one H.264 video track, entirely in memory. +/// +/// The layout is faststart (ftyp, moov, mdat) so browsers and chat apps can +/// start playing before the whole file has arrived. All samples live in a +/// single chunk, so the file needs exactly one chunk offset. +/// +/// Samples are stored in the order they are added and presented in that same +/// order: there is no composition offset table, so the stream must not +/// contain B-frames (decode order must equal presentation order). +/// +/// Pure Dart and 64-bit values are written as two 32-bit halves, so this runs +/// unchanged on the VM and on the web. +class Mp4H264Muxer { + Mp4H264Muxer({ + required this.width, + required this.height, + this.timescale = 90000, + }) { + if (width <= 0 || width > 0xFFFF || height <= 0 || height > 0xFFFF) { + throw ArgumentError('Video size must be within 1..65535 on each side.'); + } + if (timescale <= 0 || timescale > _maxUint32) { + throw ArgumentError.value(timescale, 'timescale', 'must be positive'); + } + } + + final int width; + final int height; + + /// Ticks per second for every duration passed to [addSample]. + final int timescale; + + final List _samples = []; + final List _durations = []; + final List _keyFrameNumbers = []; + int _payloadBytes = 0; + int _totalDuration = 0; + bool _finished = false; + Mp4ColorInfo? _color; + + int get sampleCount => _samples.length; + + /// Total duration of the added samples, in [timescale] ticks. + int get totalDuration => _totalDuration; + + /// Adds one access unit in AVCC format (length-prefixed NAL units), exactly + /// as WebCodecs emits it with `avc: {format: 'avc'}`. [duration] is in + /// [timescale] ticks. + void addSample( + Uint8List sample, { + required int duration, + required bool isKeyFrame, + }) { + if (_finished) throw StateError('The MP4 has already been finished.'); + if (sample.isEmpty) { + throw ArgumentError('An H.264 sample cannot be empty.'); + } + if (duration <= 0 || duration > _maxUint32) { + throw ArgumentError.value( + duration, + 'duration', + 'must be between 1 and 2^32-1 ticks', + ); + } + if (_samples.isEmpty && !isKeyFrame) { + throw StateError('The first H.264 sample must be a keyframe.'); + } + _samples.add(sample); + _durations.add(duration); + if (isKeyFrame) _keyFrameNumbers.add(_samples.length); + _payloadBytes += sample.length; + _totalDuration += duration; + } + + /// Returns the finished file. [avcDecoderConfig] is the + /// AVCDecoderConfigurationRecord (the `description` WebCodecs reports in the + /// first chunk's `metadata.decoderConfig`); [color] is how its samples map + /// to colour, when known. + Uint8List finish({ + required Uint8List avcDecoderConfig, + Mp4ColorInfo? color, + }) { + if (_finished) throw StateError('The MP4 has already been finished.'); + if (_samples.isEmpty) { + throw StateError('Cannot build an MP4 without any video samples.'); + } + if (avcDecoderConfig.isEmpty || avcDecoderConfig[0] != 1) { + throw ArgumentError( + 'avcDecoderConfig must be an AVCDecoderConfigurationRecord ' + '(configurationVersion 1).', + ); + } + _finished = true; + _color = color; + + final ftyp = _box('ftyp', [ + _fourCc('isom'), + _u32(0x200), + _fourCc('isom'), + _fourCc('iso2'), + _fourCc('avc1'), + _fourCc('mp41'), + ]); + + // The single chunk offset is known only once moov's size is known, and + // moov's size does not depend on the offset's value, so build moov twice. + final largeMdat = _payloadBytes + 8 > _maxUint32; + final mdatHeaderSize = largeMdat ? 16 : 8; + final moovSize = _moov(avcDecoderConfig, chunkOffset: 0).length; + final chunkOffset = ftyp.length + moovSize + mdatHeaderSize; + final moov = _moov(avcDecoderConfig, chunkOffset: chunkOffset); + assert(moov.length == moovSize); + + final out = BytesBuilder(copy: false) + ..add(ftyp) + ..add(moov); + if (largeMdat) { + out + ..add(_u32(1)) + ..add(_fourCc('mdat')) + ..add(_u64(_payloadBytes + 16)); + } else { + out + ..add(_u32(_payloadBytes + 8)) + ..add(_fourCc('mdat')); + } + for (final sample in _samples) { + out.add(sample); + } + return out.takeBytes(); + } + + Uint8List _moov(Uint8List avcC, {required int chunkOffset}) { + // Durations past 32 bits need the version-1 time boxes. + final longDuration = _totalDuration > _maxUint32; + return _box('moov', [ + _mvhd(longDuration), + _box('trak', [ + _tkhd(longDuration), + _box('mdia', [ + _mdhd(longDuration), + _fullBox('hdlr', 0, 0, [ + _u32(0), // pre_defined + _fourCc('vide'), + Uint8List(12), // reserved + Uint8List.fromList([...'VideoHandler'.codeUnits, 0]), + ]), + _box('minf', [ + _fullBox('vmhd', 0, 1, [Uint8List(8)]), // graphicsmode, opcolor + _box('dinf', [ + _fullBox('dref', 0, 0, [ + _u32(1), + _fullBox('url ', 0, 1, const []), // media is in this file + ]), + ]), + _stbl(avcC, chunkOffset), + ]), + ]), + ]), + ]); + } + + Uint8List _mvhd(bool longDuration) => + _fullBox('mvhd', longDuration ? 1 : 0, 0, [ + ..._times(longDuration), + _u32(timescale), + _duration(longDuration), + _u32(0x00010000), // rate 1.0 + _u16(0x0100), // volume 1.0 + Uint8List(10), // reserved + _identityMatrix(), + Uint8List(24), // pre_defined + _u32(2), // next_track_ID + ]); + + Uint8List _tkhd(bool longDuration) => + _fullBox('tkhd', longDuration ? 1 : 0, 0x000003, [ + // flags: track enabled | track in movie + ..._times(longDuration), + _u32(1), // track_ID + _u32(0), // reserved + _duration(longDuration), + Uint8List(8), // reserved + _u16(0), // layer + _u16(0), // alternate_group + _u16(0), // volume: 0 for video + _u16(0), // reserved + _identityMatrix(), + _u32(width * 0x10000), // 16.16 fixed point + _u32(height * 0x10000), + ]); + + Uint8List _mdhd(bool longDuration) => + _fullBox('mdhd', longDuration ? 1 : 0, 0, [ + ..._times(longDuration), + _u32(timescale), + _duration(longDuration), + _u16(0x55C4), // language 'und' + _u16(0), // pre_defined + ]); + + List _times(bool longDuration) => longDuration + ? [Uint8List(8), Uint8List(8)] // creation_time, modification_time + : [Uint8List(4), Uint8List(4)]; + + Uint8List _duration(bool longDuration) => + longDuration ? _u64(_totalDuration) : _u32(_totalDuration); + + Uint8List _stbl(Uint8List avcC, int chunkOffset) { + // Run-length encode the per-sample durations. + final sttsEntries = []; + var runCount = 0; + var runDelta = _durations.first; + for (final delta in _durations) { + if (delta == runDelta) { + runCount++; + continue; + } + sttsEntries + ..add(_u32(runCount)) + ..add(_u32(runDelta)); + runDelta = delta; + runCount = 1; + } + sttsEntries + ..add(_u32(runCount)) + ..add(_u32(runDelta)); + + final sizes = ByteData(_samples.length * 4); + for (var i = 0; i < _samples.length; i++) { + sizes.setUint32(i * 4, _samples[i].length); + } + + final allKeyFrames = _keyFrameNumbers.length == _samples.length; + return _box('stbl', [ + _fullBox('stsd', 0, 0, [_u32(1), _avc1(avcC)]), + _fullBox('stts', 0, 0, [_u32(sttsEntries.length ~/ 2), ...sttsEntries]), + // Without stss every sample is a sync sample. + if (!allKeyFrames) + _fullBox('stss', 0, 0, [ + _u32(_keyFrameNumbers.length), + for (final number in _keyFrameNumbers) _u32(number), + ]), + _fullBox('stsc', 0, 0, [ + _u32(1), + _u32(1), // first_chunk + _u32(_samples.length), // samples_per_chunk + _u32(1), // sample_description_index + ]), + _fullBox('stsz', 0, 0, [ + _u32(0), // sample_size: sizes vary + _u32(_samples.length), + sizes.buffer.asUint8List(), + ]), + // One chunk, and it starts right after ftyp and moov, so its offset + // always fits in 32 bits. + _fullBox('stco', 0, 0, [_u32(1), _u32(chunkOffset)]), + ]); + } + + Uint8List _avc1(Uint8List avcC) => _box('avc1', [ + Uint8List(6), // reserved + _u16(1), // data_reference_index + Uint8List(16), // pre_defined, reserved, pre_defined[3] + _u16(width), + _u16(height), + _u32(0x00480000), // 72 dpi + _u32(0x00480000), + _u32(0), // reserved + _u16(1), // frame_count + Uint8List(32), // compressorname + _u16(0x0018), // depth + _u16(0xFFFF), // pre_defined = -1 + _box('avcC', [avcC]), + if (_color case final color?) + _box('colr', [ + _fourCc('nclx'), + _u16(color.primaries), + _u16(color.transfer), + _u16(color.matrix), + Uint8List.fromList([color.fullRange ? 0x80 : 0]), + ]), + ]); + + static Uint8List _identityMatrix() => Uint8List.fromList([ + ..._u32(0x00010000), ..._u32(0), ..._u32(0), // + ..._u32(0), ..._u32(0x00010000), ..._u32(0), // + ..._u32(0), ..._u32(0), ..._u32(0x40000000), // + ]); +} + +const int _maxUint32 = 0xFFFFFFFF; + +Uint8List _box(String type, List children) { + var size = 8; + for (final child in children) { + size += child.length; + } + final out = BytesBuilder(copy: false) + ..add(_u32(size)) + ..add(_fourCc(type)); + for (final child in children) { + out.add(child); + } + return out.takeBytes(); +} + +Uint8List _fullBox( + String type, + int version, + int flags, + List children, +) => + _box(type, [ + _u32(version * 0x1000000 + flags), + ...children, + ]); + +Uint8List _fourCc(String code) { + assert(code.length == 4); + return Uint8List.fromList(code.codeUnits); +} + +Uint8List _u16(int value) => + (ByteData(2)..setUint16(0, value)).buffer.asUint8List(); + +Uint8List _u32(int value) => + (ByteData(4)..setUint32(0, value)).buffer.asUint8List(); + +/// ByteData.setUint64 is unsupported when compiled to JavaScript. +Uint8List _u64(int value) => (ByteData(8) + ..setUint32(0, value ~/ 0x100000000) + ..setUint32(4, value % 0x100000000)) + .buffer + .asUint8List(); diff --git a/lib/services/video_export/video_export_errors.dart b/lib/services/video_export/video_export_errors.dart new file mode 100644 index 00000000..a23150db --- /dev/null +++ b/lib/services/video_export/video_export_errors.dart @@ -0,0 +1,11 @@ +/// Thrown when the user cancels a video export. +class VideoExportCancelled implements Exception {} + +/// A video export failure whose [message] can be shown to the user. +class VideoExportException implements Exception { + VideoExportException(this.message); + final String message; + + @override + String toString() => 'VideoExportException: $message'; +} diff --git a/lib/services/video_export/video_export_source.dart b/lib/services/video_export/video_export_source.dart new file mode 100644 index 00000000..5b073d0e --- /dev/null +++ b/lib/services/video_export/video_export_source.dart @@ -0,0 +1,216 @@ +import 'dart:async'; + +import 'package:flutter_riverpod/flutter_riverpod.dart'; +import 'package:hive_ce/hive.dart'; +import 'package:icarus/collab/collab_models.dart'; +import 'package:icarus/collab/convex_strategy_repository.dart'; +import 'package:icarus/const/hive_boxes.dart'; +import 'package:icarus/const/sort_index_order.dart'; +import 'package:icarus/providers/collab/cloud_media_cache_provider.dart'; +import 'package:icarus/providers/collab/cloud_media_upload_queue_provider.dart'; +import 'package:icarus/providers/collab/remote_strategy_snapshot_provider.dart'; +import 'package:icarus/providers/collab/strategy_op_queue_provider.dart'; +import 'package:icarus/providers/share_link_provider.dart'; +import 'package:icarus/providers/strategy_image_source.dart'; +import 'package:icarus/providers/strategy_page_session_provider.dart'; +import 'package:icarus/providers/strategy_provider.dart'; +import 'package:icarus/screenshot/capture_images.dart'; +import 'package:icarus/services/local_image_file.dart'; +import 'package:icarus/services/video_export/video_export_errors.dart'; +import 'package:icarus/strategy/strategy_import_export.dart'; +import 'package:icarus/strategy/strategy_page_models.dart'; + +/// A page the export dialog offers. +typedef VideoExportPageChoice = ({String id, String name}); + +/// The open strategy's pages in order, from wherever the strategy lives. +List videoExportPageChoices(WidgetRef ref) { + final strategy = ref.read(strategyProvider); + return switch (strategy.source) { + StrategySource.cloud => [ + for (final page in [ + ...?ref.read(remoteEditorSnapshotProvider).valueOrNull?.pages, + ]..sortBySortIndex((page) => page.sortIndex)) + (id: page.publicId, name: page.name), + ], + StrategySource.local => [ + for (final page in [ + ...?Hive.box(HiveBoxNames.strategiesBox) + .get(strategy.strategyId) + ?.pages, + ]..sortBySortIndex((page) => page.sortIndex)) + (id: page.id, name: page.name), + ], + null => const [], + }; +} + +/// This device's changes to a cloud strategy did not reach the server in +/// time, so an export would leave out work the user can see. +class VideoExportNotSynced implements Exception { + const VideoExportNotSynced(); + + String get userMessage => + "Your latest changes haven't synced yet. Export once the strategy " + 'shows as synced.'; + + @override + String toString() => 'VideoExportNotSynced'; +} + +/// A selected page is gone from the strategy (a teammate deleted it), so +/// the video would not be the one the user asked for. +class VideoExportPagesChanged implements Exception { + const VideoExportPagesChanged(); + + String get userMessage => + 'A page you selected was deleted. Check the pages and export again.'; + + @override + String toString() => 'VideoExportPagesChanged'; +} + +/// The strategy a video export renders, saved and whole, with its images +/// decoded. Release [images] once the export is done. +typedef VideoExportSource = ({StrategyData strategy, CaptureImages images}); + +/// How long a cloud export waits for this device's changes to land. +const videoExportSyncTimeout = Duration(seconds: 20); + +/// Saves the open strategy and reads it back whole for [pageIds]. +/// +/// A local strategy is saved and read back from the library. A cloud +/// strategy's other pages live only on the server, so the export sends the +/// open page's edits, waits for everything this device still has to send +/// for the strategy to land, then reads the whole strategy from the server. +/// +/// Throws [VideoExportNotSynced] when this device's work does not land +/// within [videoExportSyncTimeout], [VideoExportPagesChanged] when a page in +/// [pageIds] no longer exists, [CaptureImagesUnavailable] when an image on +/// the exported pages cannot be loaded, and [VideoExportCancelled] once +/// [isCancelled] turns true. +Future loadVideoExportSource( + WidgetRef ref, { + required Set pageIds, + bool Function() isCancelled = _never, +}) async { + void checkCancelled() { + if (isCancelled()) throw VideoExportCancelled(); + } + + final state = ref.read(strategyProvider); + final strategyId = state.strategyId; + if (strategyId == null) { + throw StateError('No strategy is open to export.'); + } + // A signed-out reader's only access is the link they opened. + final linkView = ref.read(shareLinkViewProvider); + final shareToken = + linkView?.strategyPublicId == strategyId ? linkView!.token : null; + + final StrategyData strategy; + final Map assets; + switch (state.source) { + case StrategySource.cloud: + // Queues the open page's edits and sends them. + await ref + .read(strategyPageSessionProvider.notifier) + .flushCurrentPage(flushImmediately: true); + checkCancelled(); + await _waitUntilSent(ref, strategyId, isCancelled: isCancelled); + final snapshot = await ref + .read(convexStrategyRepositoryProvider) + .fetchFullSnapshot(strategyId, shareToken: shareToken); + checkCancelled(); + strategy = + StrategyImportExportService.strategyDataFromRemoteSnapshot(snapshot); + assets = snapshot.assetsById; + case StrategySource.local || null: + await ref.read(strategyProvider.notifier).forceSaveNow(strategyId); + checkCancelled(); + final saved = + Hive.box(HiveBoxNames.strategiesBox).get(strategyId); + if (saved == null) { + throw StateError('Strategy $strategyId is not in the library.'); + } + strategy = saved; + assets = const {}; + } + + final pages = [ + for (final page in strategy.pages) + if (pageIds.contains(page.id)) page, + ]; + if (pages.length != pageIds.length) { + throw const VideoExportPagesChanged(); + } + + final isCloud = state.source == StrategySource.cloud; + final images = await resolveCaptureImages( + { + for (final page in pages) + for (final image in page.imageData) + image.id: resolveStrategyImageSource( + localFilePath: findLocalImageFile( + storageDirectory: state.storageDirectory, + imageId: image.id, + fileExtension: image.fileExtension, + ), + isCloudStrategy: isCloud, + // The whole strategy was just read, and this device has + // nothing left to upload. + assetsLoaded: true, + remoteAsset: assets[image.id], + uploadMayBeQueuedHere: false, + ), + }, + fetch: (imageId, url, client) => downloadCloudImageBytes( + url, + client: client, + freshUrl: () => + ref.read(convexStrategyRepositoryProvider).getImageAssetUrl( + strategyPublicId: strategyId, + assetPublicId: imageId, + shareToken: shareToken, + ), + ), + checkpoint: checkCancelled, + ); + return (strategy: strategy, images: images); +} + +bool _never() => false; + +/// Waits until this device has nothing left to send for [strategyId]: no +/// op queued, in flight, or held for review, and no image uploading. Read +/// from the queues at each check: a save with nothing to send changes no +/// queue state, so nothing listening would hear it finish. +Future _waitUntilSent( + WidgetRef ref, + String strategyId, { + required bool Function() isCancelled, +}) async { + bool sent() { + final ops = ref.read(strategyOpQueueProvider); + final media = ref.read(cloudMediaUploadQueueProvider); + // An outbox with records it could not read cannot say they landed. + return ops.outboxIsReliable && + media.outboxIsReliable && + ops.pending.isEmpty && + !ops.isFlushing && + media.jobsForStrategy(strategyId).isEmpty; + } + + const step = Duration(milliseconds: 100); + var waited = Duration.zero; + while (!sent()) { + if (isCancelled()) throw VideoExportCancelled(); + // Work held for the user's review never lands by waiting. + if (ref.read(strategyOpQueueProvider).needsAttention) { + throw const VideoExportNotSynced(); + } + if (waited >= videoExportSyncTimeout) throw const VideoExportNotSynced(); + await Future.delayed(step); + waited += step; + } +} diff --git a/lib/services/video_export/video_exporter.dart b/lib/services/video_export/video_exporter.dart index 11d04e43..966e8607 100644 --- a/lib/services/video_export/video_exporter.dart +++ b/lib/services/video_export/video_exporter.dart @@ -1,5 +1,5 @@ import 'dart:async'; -import 'dart:io'; +import 'dart:typed_data'; import 'package:flutter/material.dart'; import 'package:flutter_riverpod/flutter_riverpod.dart'; @@ -10,6 +10,7 @@ import 'package:icarus/page_transition/agent_path.dart'; import 'package:icarus/page_transition/navigation_geometry_map.dart'; import 'package:icarus/page_transition/transition_planner.dart'; import 'package:icarus/providers/map_provider.dart'; +import 'package:icarus/providers/strategy_image_source.dart'; import 'package:icarus/providers/strategy_page.dart'; import 'package:icarus/providers/strategy_provider.dart'; import 'package:icarus/providers/transition_provider.dart'; @@ -17,12 +18,11 @@ import 'package:icarus/screenshot/offscreen_capture.dart'; import 'package:icarus/screenshot/capture_geometry.dart'; import 'package:icarus/screenshot/persistent_offscreen_renderer.dart'; import 'package:icarus/screenshot/screenshot_view.dart'; -import 'package:icarus/services/video_export/ffmpeg_png_sequence_writer.dart'; import 'package:icarus/services/video_export/ffmpeg_video_encoder.dart'; import 'package:icarus/services/video_export/video_export_quality.dart'; +import 'package:icarus/services/video_export/video_frame_sink.dart'; import 'package:icarus/view_cone/vision_geometry.dart'; import 'package:icarus/widgets/page_transition_overlay.dart'; -import 'package:path/path.dart' as p; @visibleForTesting Future runPreservingScreenshotMode( @@ -46,6 +46,7 @@ class VideoExporter { required this.strategyState, required this.mapState, required this.geometry, + this.imageSources = const {}, this.navigation, this.requireNavigation = false, }); @@ -54,6 +55,10 @@ class VideoExporter { final StrategyState strategyState; final MapState mapState; final VisionGeometryMap? geometry; + + /// What each image on the exported pages paints, by image id; see + /// [captureImageSourcesProvider]. + final Map imageSources; final NavigationGeometryMap? navigation; final bool requireNavigation; @@ -80,22 +85,20 @@ class VideoExporter { static const double _renderWeight = 0.85; bool _cancelled = false; - final FfmpegVideoEncoder _encoder = FfmpegVideoEncoder(); - FfmpegPngSequenceWriter? _frameWriter; + VideoFrameSink? _sink; void cancel() { _cancelled = true; - // export()'s cleanup awaits the writer's termination; here we only need - // to interrupt the processes. - unawaited(_frameWriter?.cancel()); - _encoder.cancel(); + _sink?.cancel(); } - Future export({ + /// Renders [pages] into [sink]. Returns the finished video when [sink] + /// keeps it in memory (the browser), or null once it is written to its + /// file (desktop). + Future export({ required List pages, required Duration stepDuration, - required String ffmpegBinary, - required String outputPath, + required VideoFrameSink sink, required VideoExportQuality quality, void Function(double fraction, String label)? onProgress, }) async { @@ -119,23 +122,21 @@ class VideoExporter { } } - await runPreservingScreenshotMode( + return runPreservingScreenshotMode( () => _export( pages: pages, stepDuration: stepDuration, - ffmpegBinary: ffmpegBinary, - outputPath: outputPath, + sink: sink, quality: quality, onProgress: onProgress, ), ); } - Future _export({ + Future _export({ required List pages, required Duration stepDuration, - required String ffmpegBinary, - required String outputPath, + required VideoFrameSink sink, required VideoExportQuality quality, void Function(double fraction, String label)? onProgress, }) async { @@ -150,15 +151,12 @@ class VideoExporter { fps: fps, ); - final tempDir = await Directory.systemTemp.createTemp( - 'icarus_video_export_', - ); ProviderContainer? captureContainer; CaptureGeometryLease? captureGeometry; PersistentOffscreenRenderer? renderer; - FfmpegPngSequenceWriter? frameWriter; + _sink = sink; try { - final offscreenContainer = ProviderContainer(); + final offscreenContainer = createCaptureContainer(images: imageSources); captureContainer = offscreenContainer; onProgress?.call(0, 'Preparing map'); captureGeometry = await prepareCaptureGeometry( @@ -173,20 +171,13 @@ class VideoExporter { wrapWidget: (child) => wrapForOffscreenCapture(child, container: offscreenContainer), ); - frameWriter = FfmpegPngSequenceWriter(); - _frameWriter = frameWriter; - await frameWriter.start( - binary: ffmpegBinary, - workingDirectory: tempDir.path, + await sink.start( width: CoordinateSystem.screenShotSize.width.round(), height: CoordinateSystem.screenShotSize.height.round(), - fps: fps, totalFrames: totalFrames, + totalSeconds: totalSeconds, ); - final concatLines = ['ffconcat version 1.0']; - var frameIndex = 0; var renderedFrames = 0; - String? lastFrameFile; Future renderFrame( ScreenshotView view, @@ -195,13 +186,7 @@ class VideoExporter { if (_cancelled) throw VideoExportCancelled(); view.hydrateProviders(offscreenContainer); final bytes = await renderer!.captureRawRgba(view); - final fileName = 'frame_${frameIndex.toString().padLeft(5, '0')}.png'; - await frameWriter!.writeFrame(bytes); - frameIndex++; - concatLines - ..add("file '$fileName'") - ..add('duration ${durationSeconds.toStringAsFixed(6)}'); - lastFrameFile = fileName; + await sink.addFrame(bytes, durationSeconds); renderedFrames++; onProgress?.call( renderedFrames / totalFrames * _renderWeight, @@ -264,47 +249,22 @@ class VideoExporter { } } - await frameWriter.finish(); - _frameWriter = null; - - // The concat demuxer ignores the duration of the final entry unless the - // last file is repeated. - if (lastFrameFile != null) { - concatLines.add("file '$lastFrameFile'"); - } - final concatFile = File(p.join(tempDir.path, 'frames.ffconcat')); - await concatFile.writeAsString(concatLines.join('\n')); - if (_cancelled) throw VideoExportCancelled(); onProgress?.call(_renderWeight, 'Encoding video'); - await _encoder.encode( - binary: ffmpegBinary, - workingDirectory: tempDir.path, - concatListFileName: 'frames.ffconcat', - outputPath: outputPath, - totalSeconds: totalSeconds, - quality: quality, + return await sink.finish( onProgress: (fraction) => onProgress?.call( _renderWeight + fraction * (1 - _renderWeight), 'Encoding video', ), ); } finally { - // Wait for ffmpeg to exit before deleting the directory it writes into; - // on Windows the delete races a still-exiting process otherwise. - await frameWriter?.cancel(); - _frameWriter = null; + _sink = null; try { + await sink.close(); await renderer?.dispose(); } finally { captureGeometry?.close(); captureContainer?.dispose(); - try { - await tempDir.delete(recursive: true); - } on Object { - // Leaving orphaned temp frames behind is preferable to masking the - // original export result. - } } } } diff --git a/lib/services/video_export/video_frame_sink.dart b/lib/services/video_export/video_frame_sink.dart new file mode 100644 index 00000000..4d0353b4 --- /dev/null +++ b/lib/services/video_export/video_frame_sink.dart @@ -0,0 +1,32 @@ +import 'dart:typed_data'; + +/// Where [VideoExporter] sends the frames it renders: an encoder that turns +/// them into an .mp4. Desktop pipes them to the bundled ffmpeg +/// (`FfmpegVideoSink`); the browser encodes them with WebCodecs +/// (`createBrowserVideoSink`). +abstract interface class VideoFrameSink { + /// Called once, before the first frame. Every frame is [width] x [height]; + /// the video runs [totalSeconds] over [totalFrames] frames. + Future start({ + required int width, + required int height, + required int totalFrames, + required double totalSeconds, + }); + + /// One rendered frame, top-to-bottom RGBA, shown for [durationSeconds]. + Future addFrame(Uint8List rgba, double durationSeconds); + + /// Encodes every frame added. Returns the finished video when this sink + /// keeps it in memory (the browser), or null once it is written to its + /// file (desktop). [onProgress] reports 0 to 1. + Future finish({void Function(double fraction)? onProgress}); + + /// Stops the encode in flight; [addFrame] and [finish] then throw + /// `VideoExportCancelled`. + void cancel(); + + /// Releases everything the sink holds. Safe after [finish], after + /// [cancel], and after a failure. + Future close(); +} diff --git a/lib/services/video_export/web_codecs_video_encoder.dart b/lib/services/video_export/web_codecs_video_encoder.dart new file mode 100644 index 00000000..4f1adfd5 --- /dev/null +++ b/lib/services/video_export/web_codecs_video_encoder.dart @@ -0,0 +1,609 @@ +import 'dart:async'; +import 'dart:js_interop'; +import 'dart:js_interop_unsafe'; +import 'dart:math' as math; +import 'dart:typed_data'; + +import 'package:icarus/services/video_export/mp4_muxer.dart'; +import 'package:icarus/services/video_export/video_export_errors.dart'; + +/// Encodes rendered frames into an H.264 .mp4 in the browser, using WebCodecs +/// `VideoEncoder` and [Mp4H264Muxer]. The web counterpart of the desktop +/// ffmpeg pipeline, which browsers cannot run. +/// +/// The output is constant frame rate: each [addFrame] is repeated at `fps` +/// for its duration. A repeat reuses the frame's pixels, so a held page costs +/// one RGBA upload however long it is shown. +class WebCodecsMp4Encoder { + /// Keyframe spacing ceiling, so players can seek within long holds. + static const _maxKeyFrameIntervalSeconds = 2; + + /// Encoder queue depth at which [addFrame] waits for the encoder to catch + /// up, so a fast renderer cannot pile up frames in memory. + static const _maxEncodeQueueSize = 8; + + /// Whether this browser can encode H.264 at this size, frame rate, and + /// bitrate. False when WebCodecs is missing entirely. + static Future isSupported({ + required int width, + required int height, + required int fps, + int? bitrate, + }) async => + await _supportedConfig( + width: width, + height: height, + fps: fps, + bitrate: bitrate, + ) != + null; + + _VideoEncoder? _encoder; + _OffscreenCanvas? _scaleCanvas; + _Canvas2D? _scaleContext; + late int _inputWidth; + late int _inputHeight; + late int _outputWidth; + late int _outputHeight; + late int _fps; + + /// Frames handed to the encoder so far; frame i is shown at i / fps. + int _framesEncoded = 0; + int _framesSinceKeyFrame = 0; + + /// Sum of every [addFrame] duration, so rounding never accumulates. + int _plannedMicros = 0; + + final List<_EncodedSample> _samples = []; + Uint8List? _avcDecoderConfig; + Mp4ColorInfo? _color; + Completer? _dequeued; + + /// Why the encoder failed, reported by the next call. + String? _failure; + bool _cancelled = false; + bool _finished = false; + + /// Configures the encoder. Frames arrive at [inputWidth] x [inputHeight] + /// and are scaled to [outputWidth] x [outputHeight] when those differ. + /// A null [bitrate] encodes at a high variable bitrate (see + /// [_defaultBitrate]); otherwise the bitrate is held constant. + Future start({ + required int inputWidth, + required int inputHeight, + required int outputWidth, + required int outputHeight, + required int fps, + int? bitrate, + }) async { + if (_encoder != null) throw StateError('The encoder is already started.'); + if (fps <= 0) throw ArgumentError.value(fps, 'fps', 'must be positive'); + _inputWidth = inputWidth; + _inputHeight = inputHeight; + _outputWidth = outputWidth; + _outputHeight = outputHeight; + _fps = fps; + + final config = await _supportedConfig( + width: outputWidth, + height: outputHeight, + fps: fps, + bitrate: bitrate, + ); + if (_cancelled) throw VideoExportCancelled(); + if (config == null) { + throw VideoExportException( + 'This browser cannot encode H.264 video at ' + '${outputWidth}x$outputHeight, $fps fps.', + ); + } + + if (outputWidth != inputWidth || outputHeight != inputHeight) { + final canvas = _OffscreenCanvas(outputWidth, outputHeight); + final context = canvas.getContext('2d'); + if (context == null) { + throw VideoExportException('This browser cannot scale video frames.'); + } + context.imageSmoothingQuality = 'high'; + _scaleCanvas = canvas; + _scaleContext = context; + } + + final encoder = _VideoEncoder( + _VideoEncoderInit( + output: _onChunk.toJS, + error: _onError.toJS, + ), + ); + encoder.ondequeue = ((JSAny? _) => _wakeWaiter()).toJS; + _encoder = encoder; + try { + encoder.configure(config); + } on Object catch (error) { + _closeEncoder(); + throw VideoExportException('Could not start the video encoder: $error'); + } + } + + /// Adds one rendered frame (top-to-bottom RGBA, inputWidth * inputHeight * 4 + /// bytes) shown for [duration]. + /// + /// The frame is repeated for round(duration * fps) ticks, at least one, + /// measured against the running total so remainders carry forward and the + /// video's length matches the sum of the durations. + Future addFrame(Uint8List rgba, {required Duration duration}) async { + final encoder = _checkRunning(); + if (rgba.length != _inputWidth * _inputHeight * 4) { + throw ArgumentError( + 'Expected ${_inputWidth * _inputHeight * 4} RGBA bytes, ' + 'got ${rgba.length}.', + ); + } + if (duration.isNegative) { + throw ArgumentError.value(duration, 'duration', 'must not be negative'); + } + _plannedMicros += duration.inMicroseconds; + final plannedFrames = + (_plannedMicros * _fps / Duration.microsecondsPerSecond).round(); + final repeats = math.max(1, plannedFrames - _framesEncoded); + + final base = _baseFrame(rgba); + try { + for (var i = 0; i < repeats; i++) { + while (encoder.encodeQueueSize > _maxEncodeQueueSize) { + final dequeued = _dequeued ??= Completer(); + await dequeued.future; + _checkRunning(); + } + final index = _framesEncoded; + final timestamp = _timestampOf(index); + final frame = _VideoFrame( + base, + _VideoFrameInit( + timestamp: timestamp, + duration: _timestampOf(index + 1) - timestamp, + ), + ); + // A held page (a run longer than one frame) opens on a keyframe so it + // starts crisp and seeks cleanly. Single-frame runs are transition + // frames; keying each of them would spend the bitrate on I-frames. + final keyFrame = index == 0 || + (i == 0 && repeats > 1) || + _framesSinceKeyFrame >= _maxKeyFrameIntervalSeconds * _fps; + try { + encoder.encode(frame, _EncodeOptions(keyFrame: keyFrame)); + } on Object catch (error) { + _failure ??= '$error'; + _checkRunning(); + } finally { + frame.close(); + } + _framesEncoded++; + _framesSinceKeyFrame = keyFrame ? 1 : _framesSinceKeyFrame + 1; + } + } finally { + base.close(); + } + _checkRunning(); + } + + /// Flushes the encoder and returns the finished .mp4 bytes. + Future finish() async { + final encoder = _checkRunning(); + try { + await encoder.flush().toDart; + } on Object catch (error) { + _failure ??= '$error'; + } + _checkRunning(); + _finished = true; + _closeEncoder(); + + final config = _avcDecoderConfig; + if (_samples.isEmpty || config == null) { + throw VideoExportException('The video encoder produced no video.'); + } + final ticksPerFrame = _timescale ~/ _fps; + final muxer = Mp4H264Muxer( + width: _outputWidth, + height: _outputHeight, + timescale: _timescale, + ); + for (var i = 0; i < _samples.length; i++) { + final sample = _samples[i]; + final nextFrame = + i + 1 < _samples.length ? _samples[i + 1].frame : _framesEncoded; + // The MP4 has no composition offsets, so chunks must arrive in + // presentation order. Encoders that reorder (B-frames) fail loudly + // rather than produce a video that plays frames out of order. + if (nextFrame <= sample.frame) { + throw VideoExportException( + 'The video encoder returned frames out of order.', + ); + } + muxer.addSample( + sample.bytes, + duration: (nextFrame - sample.frame) * ticksPerFrame, + isKeyFrame: sample.isKeyFrame, + ); + } + return muxer.finish(avcDecoderConfig: config, color: _color); + } + + /// Stops encoding and releases the encoder. Safe to call at any time and + /// more than once; later [addFrame] and [finish] calls throw + /// [VideoExportCancelled]. + void cancel() { + _cancelled = true; + _closeEncoder(); + _wakeWaiter(); + } + + /// Media ticks per second. 1000 ticks per frame keeps every frame duration + /// an exact integer. + int get _timescale => _fps * 1000; + + int _timestampOf(int frame) => + (frame * Duration.microsecondsPerSecond / _fps).round(); + + _VideoFrame _baseFrame(Uint8List rgba) { + final raw = _VideoFrame( + rgba.toJS, + _VideoFrameBufferInit( + format: 'RGBA', + codedWidth: _inputWidth, + codedHeight: _inputHeight, + timestamp: 0, + ), + ); + final context = _scaleContext; + if (context == null) return raw; + try { + context.drawImage(raw, 0, 0, _outputWidth, _outputHeight); + } finally { + raw.close(); + } + return _VideoFrame(_scaleCanvas!, _VideoFrameInit(timestamp: 0)); + } + + _VideoEncoder _checkRunning() { + if (_cancelled) throw VideoExportCancelled(); + final failure = _failure; + if (failure != null) { + _closeEncoder(); + throw VideoExportException('The video encoder failed: $failure'); + } + final encoder = _encoder; + if (encoder == null || _finished) { + throw StateError( + _finished ? 'The video is already finished.' : 'Call start() first.', + ); + } + return encoder; + } + + void _onChunk(_EncodedVideoChunk chunk, _ChunkMetadata? metadata) { + try { + final description = metadata?.decoderConfig?.description; + if (description != null) { + final config = _copyBufferSource(description); + final previous = _avcDecoderConfig; + if (previous != null && !_sameBytes(previous, config)) { + // The MP4 carries one avcC, which every sample must decode with. + throw StateError('The encoder changed its H.264 parameter sets.'); + } + _avcDecoderConfig = config; + } + if (metadata?.decoderConfig?.colorSpace case final colorSpace?) { + _color = _mp4Color(colorSpace); + } + final bytes = _Uint8Array(chunk.byteLength); + chunk.copyTo(bytes); + _samples.add( + _EncodedSample( + frame: + (chunk.timestamp * _fps / Duration.microsecondsPerSecond).round(), + bytes: bytes.toDart, + isKeyFrame: chunk.type == 'key', + ), + ); + } on Object catch (error) { + _failure ??= '$error'; + _closeEncoder(); + _wakeWaiter(); + } + } + + void _onError(JSAny? error) { + _failure ??= _jsString(error); + _wakeWaiter(); + } + + void _wakeWaiter() { + final waiter = _dequeued; + _dequeued = null; + if (waiter != null && !waiter.isCompleted) waiter.complete(); + } + + void _closeEncoder() { + final encoder = _encoder; + if (encoder == null || encoder.state == 'closed') return; + try { + encoder.close(); + } on Object { + // Already closed by an encoder error. + } + } + + /// Picks a bitrate for [bitrate] == null: 8 Mbps at 1080p30, the desktop + /// Max preset's H.264 rate, scaled by pixel count. 60 fps gets 1.5x rather + /// than 2x: consecutive frames differ less, and held pages cost almost + /// nothing either way. + static int _defaultBitrate(int width, int height, int fps) { + final pixelScale = width * height / (1920 * 1080); + final rateScale = fps > 30 ? 1.5 : 1.0; + return math.max(1000000, (8000000 * pixelScale * rateScale).round()); + } + + /// The first H.264 profile this browser accepts at this size, or null. + /// High needs a hardware encoder in Chrome; its software encoder (OpenH264) + /// only does Constrained Baseline, so fall back through Main to that. + static Future<_VideoEncoderConfig?> _supportedConfig({ + required int width, + required int height, + required int fps, + int? bitrate, + }) async { + if (!globalContext.has('VideoEncoder')) return null; + final targetBitrate = bitrate ?? _defaultBitrate(width, height, fps); + final level = _h264Level(width, height, fps, targetBitrate); + if (level == null) return null; + for (final profile in _h264Profiles) { + final config = _VideoEncoderConfig( + codec: 'avc1.$profile${level.toRadixString(16).padLeft(2, '0')}', + width: width, + height: height, + bitrate: targetBitrate, + bitrateMode: bitrate != null ? 'constant' : 'variable', + framerate: fps, + avc: _AvcEncoderConfig(format: 'avc'), + latencyMode: 'quality', + ); + try { + final support = await _VideoEncoder.isConfigSupported(config).toDart; + if (support.supported ?? false) return config; + } on Object { + // A rejected (TypeError) config means this profile is not usable. + } + } + return null; + } + + /// profile_idc and constraint flags, in preference order: High, Main, + /// Constrained Baseline. + static const _h264Profiles = ['6400', '4d00', '42e0']; + + /// The lowest H.264 level whose frame size, macroblock rate, and bitrate + /// limits hold this stream, or null when none does. + static int? _h264Level(int width, int height, int fps, int bitrate) { + final macroblocks = ((width + 15) ~/ 16) * ((height + 15) ~/ 16); + final macroblocksPerSecond = macroblocks * fps; + // (level_idc, MaxFS, MaxMBPS, MaxBR in kbit/s for Baseline/Main; High + // allows 1.25x, so these limits hold for every profile above.) + const levels = [ + (30, 1620, 40500, 10000), + (31, 3600, 108000, 14000), + (32, 5120, 216000, 20000), + (40, 8192, 245760, 20000), + (41, 8192, 245760, 50000), + (42, 8704, 522240, 50000), + (50, 22080, 589824, 135000), + (51, 36864, 983040, 240000), + (52, 36864, 2073600, 240000), + ]; + for (final (idc, maxFs, maxMbps, maxBrKbps) in levels) { + if (macroblocks <= maxFs && + macroblocksPerSecond <= maxMbps && + bitrate <= maxBrKbps * 1000) { + return idc; + } + } + return null; + } +} + +class _EncodedSample { + _EncodedSample({ + required this.frame, + required this.bytes, + required this.isKeyFrame, + }); + + /// Index of the frame this chunk encodes, from its timestamp. + final int frame; + final Uint8List bytes; + final bool isKeyFrame; +} + +bool _sameBytes(Uint8List a, Uint8List b) { + if (a.length != b.length) return false; + for (var i = 0; i < a.length; i++) { + if (a[i] != b[i]) return false; + } + return true; +} + +/// Copies a WebCodecs BufferSource (an ArrayBuffer or a view of one). +Uint8List _copyBufferSource(JSObject source) { + final bytes = source.isA() + ? _Uint8Array.view(source, 0, (source as _ArrayBuffer).byteLength) + : _Uint8Array.view( + (source as _ArrayBufferView).buffer, + source.byteOffset, + source.byteLength, + ); + return Uint8List.fromList(bytes.toDart); +} + +@JS('String') +external String _jsString(JSAny? value); + +@JS('VideoEncoder') +extension type _VideoEncoder._(JSObject _) implements JSObject { + external factory _VideoEncoder(_VideoEncoderInit init); + + external static JSPromise<_VideoEncoderSupport> isConfigSupported( + _VideoEncoderConfig config, + ); + + external String get state; + external int get encodeQueueSize; + external set ondequeue(JSFunction? handler); + + external void configure(_VideoEncoderConfig config); + external void encode(_VideoFrame frame, _EncodeOptions options); + external JSPromise flush(); + external void close(); +} + +extension type _VideoEncoderInit._(JSObject _) implements JSObject { + external factory _VideoEncoderInit({JSFunction output, JSFunction error}); +} + +extension type _VideoEncoderConfig._(JSObject _) implements JSObject { + external factory _VideoEncoderConfig({ + String codec, + int width, + int height, + int bitrate, + String bitrateMode, + num framerate, + _AvcEncoderConfig avc, + String latencyMode, + }); +} + +extension type _AvcEncoderConfig._(JSObject _) implements JSObject { + external factory _AvcEncoderConfig({String format}); +} + +extension type _VideoEncoderSupport._(JSObject _) implements JSObject { + external bool? get supported; +} + +extension type _EncodeOptions._(JSObject _) implements JSObject { + external factory _EncodeOptions({bool keyFrame}); +} + +@JS('VideoFrame') +extension type _VideoFrame._(JSObject _) implements JSObject { + /// [source] is a BufferSource with a [_VideoFrameBufferInit], or an image + /// source (VideoFrame, OffscreenCanvas) with a [_VideoFrameInit]. + external factory _VideoFrame(JSObject source, JSObject init); + + external void close(); +} + +extension type _VideoFrameBufferInit._(JSObject _) implements JSObject { + external factory _VideoFrameBufferInit({ + String format, + int codedWidth, + int codedHeight, + int timestamp, + }); +} + +extension type _VideoFrameInit._(JSObject _) implements JSObject { + external factory _VideoFrameInit({int timestamp, int duration}); +} + +extension type _EncodedVideoChunk._(JSObject _) implements JSObject { + external String get type; + external int get timestamp; + external int get byteLength; + external void copyTo(_Uint8Array destination); +} + +extension type _ChunkMetadata._(JSObject _) implements JSObject { + external _DecoderConfig? get decoderConfig; +} + +extension type _DecoderConfig._(JSObject _) implements JSObject { + external JSObject? get description; + external _VideoColorSpace? get colorSpace; +} + +extension type _VideoColorSpace._(JSObject _) implements JSObject { + external String? get primaries; + external String? get transfer; + external String? get matrix; + external bool? get fullRange; +} + +/// The colour the encoder says its samples carry, as MP4 code points. Hardware +/// encoders can write full-range samples without a colour description in the +/// bitstream; the container then carries it (see [Mp4ColorInfo]). Names the +/// table does not know are "unspecified" (2). +Mp4ColorInfo _mp4Color(_VideoColorSpace colorSpace) => Mp4ColorInfo( + primaries: switch (colorSpace.primaries) { + 'bt709' => 1, + 'bt470bg' => 5, + 'smpte170m' => 6, + 'bt2020' => 9, + 'smpte432' => 12, + _ => 2, + }, + transfer: switch (colorSpace.transfer) { + 'bt709' => 1, + 'smpte170m' => 6, + 'linear' => 8, + 'iec61966-2-1' => 13, + 'pq' => 16, + 'hlg' => 18, + _ => 2, + }, + matrix: switch (colorSpace.matrix) { + 'rgb' => 0, + 'bt709' => 1, + 'bt470bg' => 5, + 'smpte170m' => 6, + 'bt2020-ncl' => 9, + _ => 2, + }, + fullRange: colorSpace.fullRange ?? false, + ); + +@JS('OffscreenCanvas') +extension type _OffscreenCanvas._(JSObject _) implements JSObject { + external factory _OffscreenCanvas(int width, int height); + + external _Canvas2D? getContext(String contextId); +} + +extension type _Canvas2D._(JSObject _) implements JSObject { + external set imageSmoothingQuality(String value); + + external void drawImage( + JSObject image, + num dx, + num dy, + num dWidth, + num dHeight, + ); +} + +extension type _ArrayBuffer._(JSObject _) implements JSObject { + external int get byteLength; +} + +extension type _ArrayBufferView._(JSObject _) implements JSObject { + external JSObject get buffer; + external int get byteOffset; + external int get byteLength; +} + +@JS('Uint8Array') +extension type _Uint8Array._(JSUint8Array _) implements JSUint8Array { + external factory _Uint8Array(int length); + external factory _Uint8Array.view(JSObject buffer, int offset, int length); +} diff --git a/lib/strategy/strategy_import_export.dart b/lib/strategy/strategy_import_export.dart index b666e082..5f23a826 100644 --- a/lib/strategy/strategy_import_export.dart +++ b/lib/strategy/strategy_import_export.dart @@ -2427,7 +2427,7 @@ class StrategyImportExportService { .read(convexStrategyRepositoryProvider) .fetchFullSnapshot(strategyId); await _ensureRemoteAssetsCached(snapshot); - final strategy = _strategyDataFromRemoteSnapshot(snapshot); + final strategy = strategyDataFromRemoteSnapshot(snapshot); final outputFile = await FilePicker.platform.saveFile( type: FileType.custom, dialogTitle: 'Please select an output file:', @@ -2454,13 +2454,8 @@ class StrategyImportExportService { await zipStrategy(id: id, outputFilePath: outputFile); } - @visibleForTesting - static StrategyData strategyDataFromRemoteSnapshotForTest( - RemoteFullStrategySnapshot snapshot, - ) => - _strategyDataFromRemoteSnapshot(snapshot); - - static StrategyData _strategyDataFromRemoteSnapshot( + /// A cloud strategy as the library models it, from the server's full copy. + static StrategyData strategyDataFromRemoteSnapshot( RemoteFullStrategySnapshot snapshot, ) { final pages = []; diff --git a/lib/widgets/dialogs/export_video_dialog.dart b/lib/widgets/dialogs/export_video_dialog.dart index fff0116b..80069230 100644 --- a/lib/widgets/dialogs/export_video_dialog.dart +++ b/lib/widgets/dialogs/export_video_dialog.dart @@ -1,26 +1,29 @@ import 'dart:async'; import 'package:file_picker/file_picker.dart'; +import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; import 'package:flutter_riverpod/flutter_riverpod.dart'; -import 'package:hive_ce/hive.dart'; import 'package:icarus/const/coordinate_system.dart'; -import 'package:icarus/const/hive_boxes.dart'; import 'package:icarus/const/settings.dart'; import 'package:icarus/const/sort_index_order.dart'; import 'package:icarus/providers/drawing_provider.dart'; import 'package:icarus/providers/map_provider.dart'; import 'package:icarus/providers/screenshot_provider.dart'; -import 'package:icarus/providers/strategy_page.dart'; import 'package:icarus/providers/strategy_provider.dart'; import 'package:icarus/providers/user_preferences_provider.dart'; import 'package:icarus/providers/view_cone_geometry_provider.dart'; import 'package:icarus/providers/navigation_geometry_provider.dart'; import 'package:icarus/page_transition/navigation_geometry_map.dart'; import 'package:icarus/services/analytics_service.dart'; +import 'package:icarus/screenshot/capture_images.dart'; +import 'package:icarus/services/video_export/browser_video_sink.dart'; import 'package:icarus/services/video_export/ffmpeg_video_encoder.dart'; +import 'package:icarus/services/video_export/ffmpeg_video_sink.dart'; +import 'package:icarus/services/video_export/video_export_source.dart'; import 'package:icarus/services/video_export/video_export_quality.dart'; import 'package:icarus/services/video_export/video_exporter.dart'; +import 'package:icarus/services/video_export/video_frame_sink.dart'; import 'package:icarus/strategy/strategy_import_export.dart'; import 'package:icarus/view_cone/vision_geometry.dart'; import 'package:shadcn_ui/shadcn_ui.dart'; @@ -42,25 +45,33 @@ class _ExportVideoDialogState extends ConsumerState { /// descendant (same idiom as UploadImageDialog). static const double _contentWidth = _dialogWidth - 48; - List _pages = const []; + List _pages = const []; final Set _selectedPageIds = {}; double _stepDurationSeconds = 3.0; VideoExportQuality _quality = VideoExportQuality.social; + /// Saving, syncing, and loading images, before any frame renders. + bool _preparing = false; + bool _cancelRequested = false; VideoExporter? _exporter; double _progress = 0; String _progressLabel = ''; bool _exportRunning = false; - bool get _isExporting => _exporter != null; + bool get _isExporting => _preparing || _exporter != null; @override void dispose() { // Closing the dialog mid-export must not leave frames rendering and - // ffmpeg encoding in the background. - _exporter?.cancel(); + // the encoder running in the background. + _cancel(); super.dispose(); } + void _cancel() { + _cancelRequested = true; + _exporter?.cancel(); + } + @override void initState() { super.initState(); @@ -68,10 +79,7 @@ class _ExportVideoDialogState extends ConsumerState { .read(appPreferencesProvider) .videoExportStepDurationSeconds .clamp(1.0, 15.0); - final doc = Hive.box( - HiveBoxNames.strategiesBox, - ).get(ref.read(strategyProvider).id); - _pages = [...?doc?.pages]..sortBySortIndex((item) => item.sortIndex); + _pages = videoExportPageChoices(ref); _selectedPageIds.addAll(_pages.map((p) => p.id)); } @@ -91,7 +99,8 @@ class _ExportVideoDialogState extends ConsumerState { // The root container outlives this dialog, so cleanup after a // mid-export dismissal can still reach the live providers. final container = ProviderScope.containerOf(context, listen: false); - + final quality = _quality; + final pageIds = {..._selectedPageIds}; final stepDuration = Duration( milliseconds: (_stepDurationSeconds * 1000).round(), ); @@ -100,83 +109,114 @@ class _ExportVideoDialogState extends ConsumerState { .read(appPreferencesProvider.notifier) .setVideoExportStepDurationSeconds(_stepDurationSeconds), ); - - final ffmpegBinary = await FfmpegVideoEncoder.resolveBinary(); - if (ffmpegBinary == null) { - Settings.showToast( - message: - 'Video encoder not found. Reinstalling Icarus should fix this.', - backgroundColor: Settings.tacticalVioletTheme.destructive, + final fileName = + "${sanitizeStrategyFileName(ref.read(strategyProvider).stratName ?? "new video")}.mp4"; + + // Where the video goes is settled first, so a dismissed save dialog or + // a browser that cannot encode video costs nothing. + final VideoFrameSink sink; + if (kIsWeb) { + final totalSeconds = VideoExporter.plannedDurationSeconds( + pageCount: pageIds.length, + stepSeconds: stepDuration.inMilliseconds / 1000, + fps: quality.fps, ); - return; - } - - final selectedOutputPath = await FilePicker.platform.saveFile( - type: FileType.custom, - dialogTitle: 'Please select an output file:', - fileName: - "${sanitizeStrategyFileName(ref.read(strategyProvider).stratName ?? "new video")}.mp4", - allowedExtensions: ['mp4'], - ); - if (selectedOutputPath == null || !mounted) return; - final outputPath = ensureMp4Extension(selectedOutputPath); - - final strategyId = ref.read(strategyProvider).id; - await ref.read(strategyProvider.notifier).forceSaveNow(strategyId); - final doc = Hive.box( - HiveBoxNames.strategiesBox, - ).get(strategyId); - if (doc == null || !mounted) return; - - // Resolve pages from the freshly saved document — the dialog's initial - // snapshot may predate unsaved edits on the active page. - final selectedPages = ([...doc.pages] - ..sortBySortIndex((item) => item.sortIndex)) - .where((p) => _selectedPageIds.contains(p.id)) - .toList(); - if (selectedPages.isEmpty) return; - - final mapState = ref.read(mapProvider); - final requireNavigation = - ref.read(worldGeometryEnabledProvider(mapState.currentMap)); - VisionGeometryMap? geometry; - NavigationGeometryMap? navigation; - try { - if (requireNavigation) { - navigation = await ref - .read(navigationGeometryProvider(mapState.currentMap).future); - } else { - geometry = await ref - .read(viewConeGeometryProvider(mapState.currentMap).future); + if (!await browserCanEncodeVideo( + quality, + totalSeconds: totalSeconds, + )) { + Settings.showToast( + message: 'This browser cannot export video. Try the latest Chrome ' + 'or Edge, or the desktop app.', + backgroundColor: Settings.tacticalVioletTheme.destructive, + ); + return; + } + sink = createBrowserVideoSink(quality); + } else { + final ffmpegBinary = await FfmpegVideoEncoder.resolveBinary(); + if (ffmpegBinary == null) { + Settings.showToast( + message: + 'Video encoder not found. Reinstalling Icarus should fix this.', + backgroundColor: Settings.tacticalVioletTheme.destructive, + ); + return; } - } on Object { - // The provider reports the failure. The exporter rejects movement on - // enabled maps when their required navigation data is unavailable. + final selectedOutputPath = await FilePicker.platform.saveFile( + type: FileType.custom, + dialogTitle: 'Please select an output file:', + fileName: fileName, + allowedExtensions: ['mp4'], + ); + if (selectedOutputPath == null) return; + sink = FfmpegVideoSink( + binary: ffmpegBinary, + outputPath: ensureMp4Extension(selectedOutputPath), + quality: quality, + ); } if (!mounted) return; - final exporter = VideoExporter( - strategy: doc, - strategyState: ref.read(strategyProvider), - mapState: mapState, - geometry: geometry, - navigation: navigation, - requireNavigation: requireNavigation, - ); setState(() { - _exporter = exporter; + _preparing = true; + _cancelRequested = false; _progress = 0; - _progressLabel = 'Preparing'; + _progressLabel = 'Saving your changes'; }); - - CoordinateSystem.instance.setIsScreenshot(true); + CaptureImages? images; try { - await exporter.export( + final source = await loadVideoExportSource( + ref, + pageIds: pageIds, + isCancelled: () => _cancelRequested, + ); + images = source.images; + if (_cancelRequested) throw VideoExportCancelled(); + final selectedPages = ([...source.strategy.pages] + ..sortBySortIndex((item) => item.sortIndex)) + .where((p) => pageIds.contains(p.id)) + .toList(); + + final mapState = ref.read(mapProvider); + final requireNavigation = + ref.read(worldGeometryEnabledProvider(mapState.currentMap)); + VisionGeometryMap? geometry; + NavigationGeometryMap? navigation; + try { + if (requireNavigation) { + navigation = await ref + .read(navigationGeometryProvider(mapState.currentMap).future); + } else { + geometry = await ref + .read(viewConeGeometryProvider(mapState.currentMap).future); + } + } on Object { + // The provider reports the failure. The exporter rejects movement on + // enabled maps when their required navigation data is unavailable. + } + if (_cancelRequested) throw VideoExportCancelled(); + + final exporter = VideoExporter( + strategy: source.strategy, + strategyState: ref.read(strategyProvider), + mapState: mapState, + geometry: geometry, + imageSources: images.sources, + navigation: navigation, + requireNavigation: requireNavigation, + ); + setState(() { + _exporter = exporter; + _progressLabel = 'Preparing'; + }); + + CoordinateSystem.instance.setIsScreenshot(true); + final video = await exporter.export( pages: selectedPages, stepDuration: stepDuration, - ffmpegBinary: ffmpegBinary, - outputPath: outputPath, - quality: _quality, + sink: sink, + quality: quality, onProgress: (fraction, label) { if (!mounted) return; setState(() { @@ -185,6 +225,18 @@ class _ExportVideoDialogState extends ConsumerState { }); }, ); + // A cancel that arrived while the export was wrapping up still wins. + if (_cancelRequested) throw VideoExportCancelled(); + // The browser sink hands the finished video back to be downloaded; + // the desktop one has already written it where the user chose. + if (video != null) { + await FilePicker.platform.saveFile( + type: FileType.custom, + fileName: fileName, + allowedExtensions: ['mp4'], + bytes: video, + ); + } unawaited( AnalyticsService.instance.capture( 'content_exported', @@ -192,8 +244,8 @@ class _ExportVideoDialogState extends ConsumerState { 'content_type': 'video', 'page_count': selectedPages.length, 'step_duration_seconds': _stepDurationSeconds, - 'quality': _quality.name, - 'fps': _quality.fps, + 'quality': quality.name, + 'fps': quality.fps, }, ), ); @@ -209,12 +261,41 @@ class _ExportVideoDialogState extends ConsumerState { message: 'Video export cancelled.', backgroundColor: Settings.tacticalVioletTheme.primary, ); - } on Object catch (error) { + } on VideoExportNotSynced catch (error) { + Settings.showToast( + message: error.userMessage, + backgroundColor: Settings.tacticalVioletTheme.destructive, + ); + } on VideoExportPagesChanged catch (error) { Settings.showToast( - message: 'Video export failed: $error', + message: error.userMessage, backgroundColor: Settings.tacticalVioletTheme.destructive, ); + if (mounted) { + setState(() { + _pages = videoExportPageChoices(ref); + _selectedPageIds.retainAll({for (final page in _pages) page.id}); + }); + } + } on CaptureImagesUnavailable catch (error) { + Settings.showToast( + message: error.userMessage, + backgroundColor: Settings.tacticalVioletTheme.destructive, + ); + } on Object catch (error) { + Settings.showToast( + // Closing the dialog mid-preparation can fail the work it cut + // short; the user asked to stop, so that is what they hear. + message: _cancelRequested + ? 'Video export cancelled.' + : 'Video export failed: $error', + backgroundColor: _cancelRequested + ? Settings.tacticalVioletTheme.primary + : Settings.tacticalVioletTheme.destructive, + ); } finally { + await sink.close(); + images?.release(); CoordinateSystem.instance.setIsScreenshot(false); container.read(screenshotProvider.notifier).setIsScreenShot(false); container @@ -222,6 +303,7 @@ class _ExportVideoDialogState extends ConsumerState { .rebuildAllPaths(CoordinateSystem.instance); if (mounted) { setState(() { + _preparing = false; _exporter = null; }); } @@ -271,7 +353,7 @@ class _ExportVideoDialogState extends ConsumerState { title: const Text('Exporting video'), actions: [ ShadButton.destructive( - onPressed: () => _exporter?.cancel(), + onPressed: _cancel, child: const Text('Cancel export'), ), ], @@ -300,6 +382,17 @@ class _ExportVideoDialogState extends ConsumerState { ), const SizedBox(height: 12), ShadProgress(value: _progress, minHeight: 8), + // A hidden tab stops painting frames, which stalls the render. + if (kIsWeb) ...[ + const SizedBox(height: 12), + Text( + 'Keep this tab open until the video downloads.', + style: TextStyle( + fontSize: 12, + color: colors.mutedForeground, + ), + ), + ], ], ), ), @@ -579,7 +672,7 @@ class _PagesPanel extends StatelessWidget { required this.onToggle, }); - final List pages; + final List pages; final Set selectedPageIds; final ValueChanged onToggle; diff --git a/test/collab_sync_models_test.dart b/test/collab_sync_models_test.dart index e56eab51..12f5f455 100644 --- a/test/collab_sync_models_test.dart +++ b/test/collab_sync_models_test.dart @@ -485,7 +485,7 @@ void main() { ); expect( - () => StrategyImportExportService.strategyDataFromRemoteSnapshotForTest( + () => StrategyImportExportService.strategyDataFromRemoteSnapshot( snapshot), throwsA( isA().having( diff --git a/test/mp4_box_test_support.dart b/test/mp4_box_test_support.dart new file mode 100644 index 00000000..35afcedc --- /dev/null +++ b/test/mp4_box_test_support.dart @@ -0,0 +1,169 @@ +import 'dart:typed_data'; + +/// One parsed ISO-BMFF box. +class Mp4Box { + Mp4Box(this.type, this.start, this.size, this.headerSize); + + final String type; + final int start; + final int size; + final int headerSize; + + int get end => start + size; + + Uint8List payload(Uint8List file) => + file.sublist(start + headerSize, start + size); +} + +/// Reads back the boxes an MP4 writer produced, for assertions in tests. +/// Pure Dart, so the VM and browser tests share it. +class Mp4Boxes { + Mp4Boxes(this.file) : _data = ByteData.sublistView(file); + + final Uint8List file; + final ByteData _data; + + static const _containers = {'moov', 'trak', 'mdia', 'minf', 'dinf', 'stbl'}; + + List get topLevelTypes => + [for (final box in _parse(0, file.length)) box.type]; + + List _parse(int start, int end) { + final boxes = []; + var offset = start; + while (offset < end) { + var size = _data.getUint32(offset); + final type = String.fromCharCodes(file.sublist(offset + 4, offset + 8)); + var headerSize = 8; + if (size == 1) { + size = _data.getUint32(offset + 8) * 0x100000000 + + _data.getUint32(offset + 12); + headerSize = 16; + } + if (size < headerSize || offset + size > end) { + throw FormatException('Bad $type box size $size at $offset'); + } + boxes.add(Mp4Box(type, offset, size, headerSize)); + offset += size; + } + return boxes; + } + + /// Follows [path] from the top level, e.g. `['moov', 'trak', 'tkhd']`. + Mp4Box find(List path) { + var boxes = _parse(0, file.length); + late Mp4Box found; + for (var i = 0; i < path.length; i++) { + found = boxes.firstWhere( + (box) => box.type == path[i], + orElse: () => throw StateError('No ${path.take(i + 1).join('/')}'), + ); + if (i + 1 < path.length) { + if (!_containers.contains(found.type)) { + throw StateError('${found.type} is not a container'); + } + boxes = _parse(found.start + found.headerSize, found.end); + } + } + return found; + } + + Mp4Box? _maybeFind(List path) { + try { + return find(path); + } on StateError { + return null; + } + } + + static const _stbl = ['moov', 'trak', 'mdia', 'minf', 'stbl']; + + /// Box payload offset past the version/flags word of a full box. + int _fullBoxBody(Mp4Box box) => box.start + box.headerSize + 4; + + List _u32Table(Mp4Box box, {required int skip, int stride = 1}) { + final body = _fullBoxBody(box); + final count = _data.getUint32(body + skip * 4); + final first = body + (skip + 1) * 4; + return [ + for (var i = 0; i < count * stride; i++) _data.getUint32(first + i * 4), + ]; + } + + List chunkOffsets() => _u32Table(find([..._stbl, 'stco']), skip: 0); + + List sampleSizes() { + final stsz = find([..._stbl, 'stsz']); + final fixed = _data.getUint32(_fullBoxBody(stsz)); + if (fixed != 0) throw UnimplementedError('Fixed-size samples'); + return _u32Table(stsz, skip: 1); + } + + /// `[sample_count, sample_delta]` runs. + List> stts() { + final flat = _u32Table(find([..._stbl, 'stts']), skip: 0, stride: 2); + return [ + for (var i = 0; i < flat.length; i += 2) [flat[i], flat[i + 1]], + ]; + } + + List sampleDurations() => [ + for (final run in stts()) + for (var i = 0; i < run[0]; i++) run[1], + ]; + + /// 1-based keyframe numbers, or null when stss is absent (all sync). + List? syncSamples() { + final stss = _maybeFind([..._stbl, 'stss']); + return stss == null ? null : _u32Table(stss, skip: 0); + } + + int samplesPerChunk() { + final stsc = find([..._stbl, 'stsc']); + final body = _fullBoxBody(stsc); + if (_data.getUint32(body) != 1) throw UnimplementedError('Several runs'); + return _data.getUint32(body + 8); + } + + (int, int) _timescaleAndDuration(List path) { + final box = find(path); + final body = _fullBoxBody(box); + final version = file[box.start + box.headerSize]; + if (version == 1) { + return ( + _data.getUint32(body + 16), + _data.getUint32(body + 20) * 0x100000000 + _data.getUint32(body + 24), + ); + } + return (_data.getUint32(body + 8), _data.getUint32(body + 12)); + } + + int mediaTimescale() => + _timescaleAndDuration(['moov', 'trak', 'mdia', 'mdhd']).$1; + + int mediaDuration() => + _timescaleAndDuration(['moov', 'trak', 'mdia', 'mdhd']).$2; + + int movieDuration() => _timescaleAndDuration(['moov', 'mvhd']).$2; + + /// tkhd width and height, whole pixels of the 16.16 values. + (int, int) trackSize() { + final tkhd = find(['moov', 'trak', 'tkhd']); + return ( + _data.getUint32(tkhd.end - 8) ~/ 0x10000, + _data.getUint32(tkhd.end - 4) ~/ 0x10000, + ); + } + + /// The single entry inside stsd. + Mp4Box sampleEntry() { + final stsd = find([..._stbl, 'stsd']); + final entries = _parse(_fullBoxBody(stsd) + 4, stsd.end); + return entries.single; + } + + /// Child boxes of a box whose own fields take [entryHeaderSize] bytes after + /// its header (78 for a visual sample entry). + List childrenOf(Mp4Box box, {required int entryHeaderSize}) => + _parse(box.start + box.headerSize + entryHeaderSize, box.end); +} diff --git a/test/mp4_muxer_test.dart b/test/mp4_muxer_test.dart new file mode 100644 index 00000000..861aeca5 --- /dev/null +++ b/test/mp4_muxer_test.dart @@ -0,0 +1,352 @@ +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:icarus/services/video_export/mp4_muxer.dart'; + +import 'mp4_box_test_support.dart'; + +void main() { + group('Mp4H264Muxer box structure', () { + // Fake AVCC samples: the muxer never looks inside them. + Uint8List sample(int seed, int length) => Uint8List.fromList( + List.generate(length, (i) => (seed * 31 + i) & 0xFF)); + final avcC = Uint8List.fromList( + [1, 0x64, 0x00, 0x28, 0xFF, 0xE1, 0, 2, 0x67, 0x64, 1, 0, 2, 0x68, 0xEE], + ); + + test('lays out ftyp, moov, mdat with the chunk offset on sample one', () { + final muxer = Mp4H264Muxer(width: 1920, height: 1080); + final samples = [ + sample(1, 100), + sample(2, 37), + sample(3, 51), + sample(4, 9), + sample(5, 64), + ]; + final durations = [270000, 3000, 3000, 1500, 3000]; + final keys = [true, false, false, true, false]; + for (var i = 0; i < samples.length; i++) { + muxer.addSample(samples[i], + duration: durations[i], isKeyFrame: keys[i]); + } + final file = muxer.finish(avcDecoderConfig: avcC); + final mp4 = Mp4Boxes(file); + + expect(mp4.topLevelTypes, ['ftyp', 'moov', 'mdat']); + + final ftyp = mp4.find(['ftyp']); + expect(String.fromCharCodes(ftyp.payload(file).sublist(0, 4)), 'isom'); + + final offsets = mp4.chunkOffsets(); + expect(offsets, hasLength(1)); + final mdat = mp4.find(['mdat']); + expect(offsets.single, mdat.start + mdat.headerSize); + var cursor = offsets.single; + final sizes = mp4.sampleSizes(); + expect(sizes, [for (final s in samples) s.length]); + for (var i = 0; i < samples.length; i++) { + expect(file.sublist(cursor, cursor + sizes[i]), samples[i]); + cursor += sizes[i]; + } + expect(cursor, file.length); + + expect(mp4.sampleDurations(), durations); + expect(mp4.stts(), [ + [1, 270000], + [2, 3000], + [1, 1500], + [1, 3000], + ]); + expect(mp4.syncSamples(), [1, 4]); + expect(mp4.mediaTimescale(), 90000); + expect(mp4.mediaDuration(), 280500); + expect(mp4.movieDuration(), 280500); + expect(mp4.trackSize(), (1920, 1080)); + expect(mp4.samplesPerChunk(), samples.length); + expect( + mp4.find(['moov', 'trak', 'mdia', 'hdlr']).payload(file).sublist(8, 12), + 'vide'.codeUnits, + ); + + final avc1 = mp4.sampleEntry(); + expect(avc1.type, 'avc1'); + final avcCBox = mp4.childrenOf(avc1, entryHeaderSize: 78).single; + expect(avcCBox.type, 'avcC'); + expect(avcCBox.payload(file), avcC); + }); + + test('describes the samples\' colour after avcC when told it', () { + final muxer = Mp4H264Muxer(width: 640, height: 360) + ..addSample(sample(1, 10), duration: 1000, isKeyFrame: true); + final file = muxer.finish( + avcDecoderConfig: avcC, + color: const Mp4ColorInfo( + primaries: 1, + transfer: 13, + matrix: 1, + fullRange: true, + ), + ); + final mp4 = Mp4Boxes(file); + final children = mp4.childrenOf(mp4.sampleEntry(), entryHeaderSize: 78); + expect([for (final box in children) box.type], ['avcC', 'colr']); + expect( + children.last.payload(file), + [...'nclx'.codeUnits, 0, 1, 0, 13, 0, 1, 0x80], + ); + + final undescribed = Mp4Boxes( + (Mp4H264Muxer(width: 640, height: 360) + ..addSample(sample(1, 10), duration: 1000, isKeyFrame: true)) + .finish(avcDecoderConfig: avcC), + ); + expect( + [ + for (final box in undescribed.childrenOf( + undescribed.sampleEntry(), + entryHeaderSize: 78, + )) + box.type, + ], + ['avcC'], + ); + }); + + test('omits stss when every sample is a keyframe', () { + final muxer = Mp4H264Muxer(width: 640, height: 360, timescale: 30000) + ..addSample(sample(1, 10), duration: 1000, isKeyFrame: true) + ..addSample(sample(2, 10), duration: 1000, isKeyFrame: true); + final mp4 = Mp4Boxes(muxer.finish(avcDecoderConfig: avcC)); + expect(mp4.syncSamples(), isNull); + expect(mp4.stts(), [ + [2, 1000], + ]); + }); + + test('refuses an empty file and a non-keyframe start', () { + expect( + () => Mp4H264Muxer(width: 640, height: 360) + .finish(avcDecoderConfig: avcC), + throwsStateError, + ); + expect( + () => Mp4H264Muxer(width: 640, height: 360) + .addSample(sample(1, 10), duration: 1, isKeyFrame: false), + throwsStateError, + ); + expect( + () => Mp4H264Muxer(width: 640, height: 360) + .addSample(sample(1, 10), duration: 0, isKeyFrame: true), + throwsArgumentError, + ); + }); + }); + + group('Mp4H264Muxer with a real H.264 stream', () { + final ffmpegMissing = _ffmpegMissingReason(); + + test( + 'produces a file ffprobe and ffmpeg read back exactly', + () async { + final dir = await Directory.systemTemp.createTemp('icarus_mp4_muxer_'); + addTearDown(() => dir.delete(recursive: true)); + final annexBPath = '${dir.path}${Platform.pathSeparator}in.h264'; + final outPath = '${dir.path}${Platform.pathSeparator}out.mp4'; + + // No B-frames (the muxer writes no composition offsets), an access + // unit delimiter before every frame so the stream splits cleanly, and + // an IDR every 5 frames so stss has several entries. + final encode = await Process.run('ffmpeg', [ + '-v', 'error', '-y', // + '-f', 'lavfi', '-i', 'testsrc=size=320x240:rate=10', + '-frames:v', '20', + '-c:v', 'libx264', '-pix_fmt', 'yuv420p', + '-x264-params', 'aud=1:bframes=0:keyint=5:min-keyint=5:scenecut=0', + '-bsf:v', 'h264_mp4toannexb', + '-f', 'h264', annexBPath, + ]); + expect(encode.exitCode, 0, reason: '${encode.stderr}'); + + final stream = _AnnexBStream.parse( + await File(annexBPath).readAsBytes(), + ); + expect(stream.accessUnits, hasLength(20)); + + final muxer = Mp4H264Muxer(width: 320, height: 240); + // First frame held for 3 s, the rest at 1/10 s: 3 + 19 * 0.1 = 4.9 s. + for (var i = 0; i < stream.accessUnits.length; i++) { + final unit = stream.accessUnits[i]; + muxer.addSample( + unit.avcc, + duration: i == 0 ? 270000 : 9000, + isKeyFrame: unit.isKeyFrame, + ); + } + // libx264 writes no colour description; the container says the + // samples are full range, and players must believe it. + final bytes = muxer.finish( + avcDecoderConfig: stream.avcDecoderConfig, + color: const Mp4ColorInfo( + primaries: 1, + transfer: 1, + matrix: 1, + fullRange: true, + ), + ); + await File(outPath).writeAsBytes(bytes); + + expect(Mp4Boxes(bytes).syncSamples(), [1, 6, 11, 16]); + + final probe = await Process.run('ffprobe', [ + '-v', 'error', // + '-count_frames', + '-show_streams', '-show_format', + outPath, + ]); + expect(probe.exitCode, 0, reason: '${probe.stderr}'); + final fields = _probeFields(probe.stdout as String); + expect(fields['codec_name'], 'h264'); + expect(fields['color_range'], 'pc'); + expect(fields['color_space'], 'bt709'); + expect(fields['width'], '320'); + expect(fields['height'], '240'); + expect(fields['nb_read_frames'], '20'); + expect(fields['nb_frames'], '20'); + expect(double.parse(fields['format.duration']!), closeTo(4.9, 0.001)); + expect(double.parse(fields['stream.duration']!), closeTo(4.9, 0.001)); + + final decode = await Process.run('ffmpeg', [ + '-v', 'error', '-i', outPath, '-f', 'null', '-', // + ]); + expect(decode.exitCode, 0); + expect((decode.stderr as String).trim(), isEmpty); + }, + skip: ffmpegMissing, + timeout: const Timeout(Duration(minutes: 2)), + ); + }); +} + +/// Null when ffmpeg (with libx264) and ffprobe both run, otherwise why the +/// test is skipped. +String? _ffmpegMissingReason() { + for (final tool in ['ffmpeg', 'ffprobe']) { + try { + final result = Process.runSync(tool, ['-version']); + if (result.exitCode != 0) return '$tool -version failed'; + } on ProcessException { + return '$tool is not on PATH'; + } + } + final encoders = Process.runSync('ffmpeg', ['-hide_banner', '-encoders']); + if (!'${encoders.stdout}'.contains('libx264')) { + return 'ffmpeg has no libx264 encoder'; + } + return null; +} + +/// Parses ffprobe's default output into `key` (stream fields) plus +/// `stream.duration` / `format.duration`, which share a name. +Map _probeFields(String output) { + final fields = {}; + var section = ''; + for (final raw in output.split(RegExp(r'\r?\n'))) { + final line = raw.trim(); + if (line == '[STREAM]') section = 'stream'; + if (line == '[FORMAT]') section = 'format'; + final eq = line.indexOf('='); + if (eq <= 0) continue; + final key = line.substring(0, eq); + final value = line.substring(eq + 1); + if (key == 'duration') { + fields['$section.duration'] = value; + } else { + fields.putIfAbsent(key, () => value); + } + } + return fields; +} + +class _AccessUnit { + _AccessUnit(this.avcc, this.isKeyFrame); + final Uint8List avcc; + final bool isKeyFrame; +} + +/// An Annex-B H.264 stream split on its access unit delimiters. +class _AnnexBStream { + _AnnexBStream(this.accessUnits, this.avcDecoderConfig); + + final List<_AccessUnit> accessUnits; + final Uint8List avcDecoderConfig; + + static _AnnexBStream parse(Uint8List bytes) { + final nals = _splitNalUnits(bytes); + Uint8List? sps; + Uint8List? pps; + final units = <_AccessUnit>[]; + List? current; + var currentIsKey = false; + + void close() { + final nalsInUnit = current; + if (nalsInUnit == null || nalsInUnit.isEmpty) return; + final out = BytesBuilder(); + for (final nal in nalsInUnit) { + out + ..add((ByteData(4)..setUint32(0, nal.length)).buffer.asUint8List()) + ..add(nal); + } + units.add(_AccessUnit(out.takeBytes(), currentIsKey)); + } + + for (final nal in nals) { + final type = nal[0] & 0x1F; + switch (type) { + case 9: // access unit delimiter + close(); + current = []; + currentIsKey = false; + case 7: + sps ??= nal; + case 8: + pps ??= nal; + default: + if (type == 5) currentIsKey = true; + (current ??= []).add(nal); + } + } + close(); + + final s = sps!; + final p = pps!; + final config = BytesBuilder() + ..add([1, s[1], s[2], s[3], 0xFF, 0xE1]) + ..add([s.length >> 8, s.length & 0xFF]) + ..add(s) + ..add([1, p.length >> 8, p.length & 0xFF]) + ..add(p); + return _AnnexBStream(units, config.takeBytes()); + } + + static List _splitNalUnits(Uint8List bytes) { + final starts = []; // index of the first byte after each start code + for (var i = 0; i + 2 < bytes.length; i++) { + if (bytes[i] == 0 && bytes[i + 1] == 0 && bytes[i + 2] == 1) { + starts.add(i + 3); + i += 2; + } + } + final nals = []; + for (var n = 0; n < starts.length; n++) { + var end = n + 1 < starts.length ? starts[n + 1] - 3 : bytes.length; + // A 4-byte start code leaves a zero byte on the previous NAL. + while (end > starts[n] && bytes[end - 1] == 0) { + end--; + } + nals.add(Uint8List.sublistView(bytes, starts[n], end)); + } + return nals; + } +} diff --git a/test/video_export_navigation_failure_test.dart b/test/video_export_navigation_failure_test.dart index 2a207800..d0184143 100644 --- a/test/video_export_navigation_failure_test.dart +++ b/test/video_export_navigation_failure_test.dart @@ -9,6 +9,7 @@ import 'package:icarus/providers/strategy_page.dart'; import 'package:icarus/providers/strategy_provider.dart'; import 'package:icarus/providers/strategy_settings_provider.dart'; import 'package:icarus/services/video_export/ffmpeg_video_encoder.dart'; +import 'package:icarus/services/video_export/ffmpeg_video_sink.dart'; import 'package:icarus/services/video_export/video_export_quality.dart'; import 'package:icarus/services/video_export/video_exporter.dart'; @@ -66,8 +67,11 @@ void main() { exporter.export( pages: pages, stepDuration: const Duration(seconds: 1), - ffmpegBinary: 'must-not-run', - outputPath: 'must-not-create.mp4', + sink: FfmpegVideoSink( + binary: 'must-not-run', + outputPath: 'must-not-create.mp4', + quality: VideoExportQuality.social, + ), quality: VideoExportQuality.social, onProgress: (_, label) { startedPreparing = true; diff --git a/test/video_export_source_test.dart b/test/video_export_source_test.dart new file mode 100644 index 00000000..9d6e5412 --- /dev/null +++ b/test/video_export_source_test.dart @@ -0,0 +1,546 @@ +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:flutter/material.dart'; +import 'package:flutter_riverpod/flutter_riverpod.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:hive_ce/hive.dart'; +import 'package:http/http.dart' as http; +import 'package:http/testing.dart'; +import 'package:icarus/collab/cloud_media_models.dart'; +import 'package:icarus/collab/collab_models.dart'; +import 'package:icarus/collab/convex_strategy_repository.dart'; +import 'package:icarus/const/hive_boxes.dart'; +import 'package:icarus/const/maps.dart'; +import 'package:icarus/const/placed_classes.dart'; +import 'package:icarus/hive/hive_registration.dart'; +import 'package:icarus/providers/collab/remote_strategy_snapshot_provider.dart'; +import 'package:icarus/providers/share_link_provider.dart'; +import 'package:icarus/providers/strategy_image_source.dart'; +import 'package:icarus/providers/strategy_page.dart'; +import 'package:icarus/providers/strategy_provider.dart'; +import 'package:icarus/providers/collab/cloud_media_upload_queue_provider.dart'; +import 'package:icarus/providers/collab/strategy_op_queue_provider.dart'; +import 'package:icarus/providers/strategy_save_state_provider.dart'; +import 'package:icarus/services/video_export/video_export_errors.dart'; +import 'package:icarus/providers/strategy_settings_provider.dart'; +import 'package:icarus/services/video_export/video_export_source.dart'; +import 'package:icarus/strategy/strategy_page_models.dart'; +import 'package:image/image.dart' as img; +import 'package:path/path.dart' as path; + +const _strategyId = 'strategy-1'; +const _imageUrl = 'https://media.example.com/image-1.png'; + +final _png = Uint8List.fromList( + img.encodePng( + img.fill(img.Image(width: 4, height: 4), color: img.ColorRgb8(1, 2, 3)), + ), +); + +PlacedImage _image(String id) => PlacedImage( + position: const Offset(500, 500), + id: id, + aspectRatio: 1, + scale: 100, + fileExtension: '.png', + ); + +class _OpenStrategy extends StrategyProvider { + _OpenStrategy(this.source, {this.storageDirectory}); + + final StrategySource source; + final String? storageDirectory; + int saves = 0; + + @override + StrategyState build() => StrategyState( + strategyId: _strategyId, + strategyName: 'Strategy', + source: source, + storageDirectory: storageDirectory, + isOpen: true, + ); + + @override + Future forceSaveNow(String id) async => saves++; +} + +/// This device's op queue, set by the test. +class _Queue extends StrategyOpQueueNotifier { + _Queue(this.initial); + + final StrategyOpQueueState initial; + var flushes = 0; + + @override + StrategyOpQueueState build() => initial; + + @override + Future flushNow() async => flushes++; + + /// Everything this device queued has landed. + void settle() => state = const StrategyOpQueueState(durableLoaded: true); +} + +class _NoUploads extends CloudMediaUploadQueueNotifier { + @override + CloudMediaUploadQueueState build() => + const CloudMediaUploadQueueState(jobs: [], isProcessing: false); +} + +class _Repository extends Fake implements ConvexStrategyRepository { + int fetches = 0; + final shareTokens = []; + + @override + Future fetchFullSnapshot( + String strategyPublicId, { + String? shareToken, + }) async { + fetches++; + shareTokens.add(shareToken); + final now = DateTime.utc(2026); + RemotePage page(String id, int sortIndex) => RemotePage( + publicId: id, + strategyPublicId: _strategyId, + name: 'Page $id', + sortIndex: sortIndex, + isAttack: true, + revision: 1, + createdAt: now, + updatedAt: now, + ); + RemoteFullPage fullPage(String id, int sortIndex) => RemoteFullPage( + page: page(id, sortIndex), + content: + RemotePageContent(revision: 1, createdAt: now, updatedAt: now), + ); + return RemoteFullStrategySnapshot( + header: RemoteStrategyHeader( + publicId: _strategyId, + name: 'Strategy', + mapData: Maps.mapNames[MapValue.ascent]!, + revision: 1, + createdAt: now, + updatedAt: now, + ), + pages: [fullPage('b', 1), fullPage('a', 0)], + elementsByPage: { + for (final (pageId, imageId) in [('a', 'image-a'), ('b', 'image-b')]) + pageId: [ + RemoteElement( + publicId: imageId, + strategyPublicId: _strategyId, + pagePublicId: pageId, + elementType: 'image', + payload: cloudElementPayload( + kind: 'image', + data: cloudImagePayloadFromPlacedImage(_image(imageId)), + ), + sortIndex: 0, + revision: 1, + deleted: false, + ), + ], + }, + lineupsByPage: const {}, + assetsById: { + for (final imageId in ['image-a', 'image-b']) + imageId: RemoteImageAsset( + publicId: imageId, + fileExtension: '.png', + width: 4, + height: 4, + url: '$_imageUrl?$imageId', + legacyStoragePath: null, + uploadStatus: 'active', + ), + }, + ); + } +} + +Future _pumpRef( + WidgetTester tester, List overrides) async { + late WidgetRef ref; + await tester.pumpWidget(ProviderScope( + overrides: overrides, + child: Consumer(builder: (context, widgetRef, _) { + ref = widgetRef; + return const SizedBox.shrink(); + }), + )); + return ref; +} + +void main() { + group('a cloud strategy', () { + List cloud( + _Queue queue, + _Repository repository, { + List more = const [], + }) => + [ + strategyProvider + .overrideWith(() => _OpenStrategy(StrategySource.cloud)), + strategyOpQueueProvider.overrideWith(() => queue), + cloudMediaUploadQueueProvider.overrideWith(_NoUploads.new), + convexStrategyRepositoryProvider.overrideWithValue(repository), + ...more, + ]; + + testWidgets('waits for queued work, then reads the whole strategy', + (tester) async { + final queue = _Queue(const StrategyOpQueueState( + durableLoaded: true, + isFlushing: true, + )); + final repository = _Repository(); + final ref = await _pumpRef(tester, cloud(queue, repository)); + final requested = []; + + final source = await tester.runAsync( + () => http.runWithClient( + () async { + final loading = loadVideoExportSource(ref, pageIds: {'a'}); + await Future.delayed(const Duration(milliseconds: 250)); + // The edits were pushed, but nothing is read until they land. + expect(queue.flushes, 1); + expect(repository.fetches, 0); + queue.settle(); + return loading; + }, + () => MockClient((request) async { + requested.add(request.url.query); + return http.Response.bytes(_png, 200); + }), + ), + ); + addTearDown(source!.images.release); + + expect(repository.fetches, 1); + expect(source.strategy.id, _strategyId); + expect(source.strategy.mapData, MapValue.ascent); + expect( + [for (final page in source.strategy.pages) page.id], + containsAll(['a', 'b']), + ); + // Only the exported page's image is fetched and decoded. + expect(requested, ['image-a']); + expect(source.images.sources.keys, ['image-a']); + expect(source.images.sources['image-a'], isA()); + }); + + testWidgets( + 'with nothing to send, reads at once and leaves the save chip alone', + (tester) async { + final repository = _Repository(); + final ref = await _pumpRef( + tester, + cloud( + _Queue(const StrategyOpQueueState(durableLoaded: true)), + repository, + ), + ); + + final source = await tester.runAsync( + () => http.runWithClient( + () => loadVideoExportSource(ref, pageIds: {'a'}), + () => MockClient((_) async => http.Response.bytes(_png, 200)), + ), + ); + addTearDown(source!.images.release); + + expect(repository.fetches, 1); + // A save that sends nothing must not leave the chip saying syncing. + final saveState = ref.read(strategySaveStateProvider); + expect(saveState.hasPendingCloudSync, isFalse); + expect(saveState.isDirty, isFalse); + }); + + testWidgets('stops when this device has not synced in time', + (tester) async { + final repository = _Repository(); + final ref = await _pumpRef( + tester, + cloud( + _Queue(const StrategyOpQueueState( + durableLoaded: true, + isFlushing: true, + )), + repository, + ), + ); + + Object? error; + loadVideoExportSource(ref, pageIds: {'a'}).then( + (_) {}, + onError: (Object caught) => error = caught, + ); + await tester.pump(videoExportSyncTimeout + const Duration(seconds: 1)); + + expect(error, isA()); + expect(repository.fetches, 0); + }); + + testWidgets('stops at once when work waits for review', (tester) async { + final repository = _Repository(); + final ref = await _pumpRef( + tester, + cloud( + _Queue(const StrategyOpQueueState( + durableLoaded: true, + hasDurabilityFailure: true, + )), + repository, + ), + ); + + Object? error; + loadVideoExportSource(ref, pageIds: {'a'}).then( + (_) {}, + onError: (Object caught) => error = caught, + ); + await tester.pump(const Duration(milliseconds: 1)); + + expect(error, isA()); + }); + + testWidgets('stops waiting when the export is cancelled', (tester) async { + final repository = _Repository(); + final ref = await _pumpRef( + tester, + cloud( + _Queue(const StrategyOpQueueState( + durableLoaded: true, + isFlushing: true, + )), + repository, + ), + ); + var cancelled = false; + + Object? error; + loadVideoExportSource( + ref, + pageIds: {'a'}, + isCancelled: () => cancelled, + ).then((_) {}, onError: (Object caught) => error = caught); + await tester.pump(const Duration(seconds: 1)); + cancelled = true; + await tester.pump(const Duration(milliseconds: 200)); + + expect(error, isA()); + expect(repository.fetches, 0); + }); + + testWidgets('stops between images when the export is cancelled', + (tester) async { + final ref = await _pumpRef( + tester, + cloud( + _Queue(const StrategyOpQueueState(durableLoaded: true)), + _Repository(), + ), + ); + var cancelled = false; + final requested = []; + + Object? error; + await tester.runAsync(() async { + try { + await http.runWithClient( + () => loadVideoExportSource( + ref, + pageIds: {'a', 'b'}, + isCancelled: () => cancelled, + ), + () => MockClient((request) async { + requested.add(request.url.query); + // The user cancels while the first image downloads. + cancelled = true; + return http.Response.bytes(_png, 200); + }), + ); + } catch (caught) { + error = caught; + } + }); + + // Downloads start together; the cancel stops the export before the + // next image is decoded and held. + expect(error, isA()); + expect(requested, isNotEmpty); + }); + + testWidgets('refuses a selection naming a page that no longer exists', + (tester) async { + final repository = _Repository(); + final ref = await _pumpRef( + tester, + cloud( + _Queue(const StrategyOpQueueState(durableLoaded: true)), + repository, + ), + ); + + Object? error; + await tester.runAsync(() async { + try { + await loadVideoExportSource(ref, pageIds: {'a', 'deleted'}); + } catch (caught) { + error = caught; + } + }); + + expect(error, isA()); + }); + + testWidgets('reads through the link a signed-out reader opened', + (tester) async { + final repository = _Repository(); + final ref = await _pumpRef( + tester, + cloud( + _Queue(const StrategyOpQueueState(durableLoaded: true)), + repository, + more: [ + shareLinkViewProvider.overrideWith( + (ref) => (strategyPublicId: _strategyId, token: 'link-token'), + ), + ], + ), + ); + + final source = await tester.runAsync( + () => http.runWithClient( + () => loadVideoExportSource(ref, pageIds: {'a'}), + () => MockClient((_) async => http.Response.bytes(_png, 200)), + ), + ); + addTearDown(source!.images.release); + + expect(repository.shareTokens, ['link-token']); + }); + + testWidgets('offers the pages the editor lists, in order', (tester) async { + final ref = await _pumpRef(tester, [ + strategyProvider + .overrideWith(() => _OpenStrategy(StrategySource.cloud)), + remoteEditorSnapshotProvider.overrideWith(_TwoPageSnapshot.new), + ]); + await tester.runAsync( + () => ProviderScope.containerOf(tester.element(find.byType(Consumer))) + .read(remoteEditorSnapshotProvider.future), + ); + + expect(videoExportPageChoices(ref), [ + (id: 'first', name: 'First'), + (id: 'second', name: 'Second'), + ]); + }); + }); + + group('a local strategy', () { + late Directory dir; + + setUpAll(() async { + dir = await Directory.systemTemp.createTemp('icarus_video_source'); + Hive.init(dir.path); + registerIcarusAdapters(Hive); + await Hive.openBox(HiveBoxNames.strategiesBox); + }); + + tearDownAll(() async { + await Hive.close(); + await dir.delete(recursive: true); + }); + + testWidgets('is read back from the library with its image files', + (tester) async { + final storage = path.join(dir.path, 'storage'); + await tester.runAsync(() async { + final file = File(path.join(storage, 'images', 'image-a.png')); + await file.parent.create(recursive: true); + await file.writeAsBytes(_png); + await Hive.box(HiveBoxNames.strategiesBox).put( + _strategyId, + StrategyData( + id: _strategyId, + name: 'Strategy', + mapData: MapValue.bind, + versionNumber: 1, + lastEdited: DateTime(2026), + folderID: null, + pages: [ + StrategyPage( + id: 'a', + name: 'A', + drawingData: const [], + agentData: const [], + abilityData: const [], + textData: const [], + imageData: [_image('image-a'), _image('image-missing')], + utilityData: const [], + sortIndex: 0, + isAttack: true, + settings: StrategySettings(), + ), + ], + ), + ); + }); + final strategy = _OpenStrategy( + StrategySource.local, + storageDirectory: storage, + ); + final ref = await _pumpRef(tester, [ + strategyProvider.overrideWith(() => strategy), + ]); + + final source = await tester.runAsync( + () => loadVideoExportSource(ref, pageIds: {'a'}), + ); + addTearDown(source!.images.release); + + expect(strategy.saves, 1); + expect(source.strategy.mapData, MapValue.bind); + expect(source.images.sources['image-a'], isA()); + // A file the editor cannot find is exported as it shows: unavailable. + expect(source.images.sources['image-missing'], isA()); + }); + }); +} + +class _TwoPageSnapshot extends RemoteEditorSnapshotNotifier { + @override + Future build() async { + final now = DateTime.utc(2026); + RemotePage page(String id, String name, int sortIndex) => RemotePage( + publicId: id, + strategyPublicId: _strategyId, + name: name, + sortIndex: sortIndex, + isAttack: true, + revision: 1, + createdAt: now, + updatedAt: now, + ); + return RemoteEditorSnapshot( + shell: RemoteStrategyShell( + header: RemoteStrategyHeader( + publicId: _strategyId, + name: 'Strategy', + mapData: Maps.mapNames[MapValue.ascent]!, + revision: 1, + createdAt: now, + updatedAt: now, + role: 'owner', + ), + pages: [page('second', 'Second', 1), page('first', 'First', 0)], + ), + activePage: null, + ); + } +} diff --git a/test/video_exporter_sink_test.dart b/test/video_exporter_sink_test.dart new file mode 100644 index 00000000..6f42c06d --- /dev/null +++ b/test/video_exporter_sink_test.dart @@ -0,0 +1,359 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:hive_ce/hive.dart'; +import 'package:icarus/const/coordinate_system.dart'; +import 'package:icarus/const/hive_boxes.dart'; +import 'package:icarus/const/image_scale_policy.dart'; +import 'package:icarus/const/maps.dart'; +import 'package:icarus/const/placed_classes.dart'; +import 'package:icarus/hive/hive_registration.dart'; +import 'package:icarus/providers/auth_provider.dart'; +import 'package:icarus/providers/map_provider.dart'; +import 'package:icarus/providers/strategy_image_source.dart'; +import 'package:icarus/providers/strategy_page.dart'; +import 'package:icarus/providers/strategy_provider.dart'; +import 'package:icarus/providers/strategy_settings_provider.dart'; +import 'package:icarus/providers/user_preferences_provider.dart'; +import 'package:icarus/services/video_export/ffmpeg_video_sink.dart'; +import 'package:icarus/services/video_export/video_export_quality.dart'; +import 'package:icarus/services/video_export/video_exporter.dart'; +import 'package:icarus/services/video_export/video_frame_sink.dart'; +import 'package:icarus/strategy/strategy_page_models.dart'; +import 'package:image/image.dart' as img; +import 'package:supabase_flutter/supabase_flutter.dart' show AuthState, Session; + +/// Pure magenta, a colour no map or marker draws. +final _magentaPng = Uint8List.fromList( + img.encodePng( + img.fill( + img.Image(width: 64, height: 36), + color: img.ColorRgb8(255, 0, 255), + ), + ), +); + +/// Records every call an export makes to the app's one Convex client. +class _RecordingConvexAuth extends Fake implements AuthProviderConvexApi { + final calls = []; + + @override + Stream get authState => const Stream.empty(); + + @override + bool get isAuthenticated => true; + + @override + Future setAuthWithRefresh({ + required Future Function() fetchToken, + void Function(bool isAuthenticated)? onAuthChange, + }) async { + calls.add('setAuthWithRefresh'); + throw UnimplementedError(); + } + + @override + Future clearAuth() async => calls.add('clearAuth'); +} + +class _NoSupabase extends Fake implements AuthProviderSupabaseApi { + @override + Session? get currentSession => null; + + @override + Stream get onAuthStateChange => const Stream.empty(); +} + +class _RecordingSink implements VideoFrameSink { + final video = Uint8List.fromList([7, 7, 7]); + final frames = <(Uint8List, double)>[]; + ({int width, int height, int totalFrames, double totalSeconds})? started; + var closed = 0; + + @override + Future start({ + required int width, + required int height, + required int totalFrames, + required double totalSeconds, + }) async { + started = ( + width: width, + height: height, + totalFrames: totalFrames, + totalSeconds: totalSeconds, + ); + } + + @override + Future addFrame(Uint8List rgba, double durationSeconds) async => + frames.add((rgba, durationSeconds)); + + @override + Future finish({void Function(double)? onProgress}) async => video; + + @override + void cancel() {} + + @override + Future close() async => closed++; +} + +StrategyPage _page(String id, int sortIndex, Offset imageAt) => StrategyPage( + id: id, + name: id, + drawingData: const [], + agentData: const [], + abilityData: const [], + textData: const [], + imageData: [ + PlacedImage( + position: imageAt, + id: 'image-1', + aspectRatio: 64 / 36, + scale: ImageScalePolicy.defaultWidth, + fileExtension: '.png', + sizeVersion: PlacedImage.currentSizeVersion, + ), + ], + utilityData: const [], + sortIndex: sortIndex, + isAttack: true, + settings: StrategySettings(), + ); + +VideoExporter _exporter(List pages) { + return VideoExporter( + strategy: StrategyData( + id: 'cloud-strategy', + name: 'Cloud strat', + mapData: MapValue.ascent, + versionNumber: 1, + lastEdited: DateTime(2026), + folderID: null, + pages: pages, + ), + strategyState: const StrategyState( + strategyId: 'cloud-strategy', + strategyName: 'Cloud strat', + source: StrategySource.cloud, + storageDirectory: null, + isOpen: true, + ), + mapState: MapState(currentMap: MapValue.ascent, isAttack: true), + geometry: null, + imageSources: {'image-1': ImageBytes(_magentaPng)}, + ); +} + +/// Runs the export on real time with the app's frames still coming, as in +/// the app: the offscreen tree's layout builders wait for the app's next +/// frame. +Future _exportWithFrames( + WidgetTester tester, + VideoExporter exporter, + List pages, + VideoFrameSink sink, +) async { + Object? outcome; + var done = false; + await tester.runAsync(() async { + unawaited( + exporter + .export( + pages: pages, + stepDuration: const Duration(seconds: 2), + sink: sink, + quality: VideoExportQuality.social, + ) + .then((video) => outcome = video, + onError: (Object error) => outcome = error) + .whenComplete(() => done = true), + ); + }); + while (!done) { + await tester.runAsync( + () => Future.delayed(const Duration(milliseconds: 16)), + ); + await tester.pump(); + } + return outcome; +} + +int _magentaPixels(Uint8List rgba) { + var count = 0; + for (var i = 0; i < rgba.length; i += 4) { + if (rgba[i] > 240 && rgba[i + 1] < 20 && rgba[i + 2] > 240) count++; + } + return count; +} + +/// A PATH ffmpeg, the development fallback of FfmpegVideoEncoder, with the +/// H.264 encoder the export asks for on this platform. +final String? _ffmpeg = () { + final ffmpeg = _onPath('ffmpeg'); + if (ffmpeg == null) return null; + final encoders = Process.runSync(ffmpeg, ['-hide_banner', '-encoders']); + final h264 = Platform.isWindows ? 'h264_mf' : 'libx264'; + return '${encoders.stdout}'.contains(h264) ? ffmpeg : null; +}(); +final String? _ffprobe = _onPath('ffprobe'); + +String? _onPath(String tool) { + try { + final result = Process.runSync( + Platform.isWindows ? 'where' : 'which', + [tool], + ); + if (result.exitCode != 0) return null; + return const LineSplitter().convert(result.stdout as String).first.trim(); + } on ProcessException { + return null; + } +} + +void main() { + late Directory hiveDir; + + setUpAll(() async { + hiveDir = await Directory.systemTemp.createTemp('icarus_video_sink'); + Hive.init(hiveDir.path); + registerIcarusAdapters(Hive); + await Hive.openBox(HiveBoxNames.strategiesBox); + await Hive.openBox(HiveBoxNames.mapThemeProfilesBox); + await Hive.openBox(HiveBoxNames.appPreferencesBox); + await MapThemeProfilesProvider.bootstrap(); + }); + + tearDownAll(() async { + await Hive.close(); + await hiveDir.delete(recursive: true); + }); + + testWidgets( + 'renders every page and transition frame into the sink, images and all', + (tester) async { + CoordinateSystem(playAreaSize: const Size(1600, 900)); + final pages = [ + _page('one', 0, const Offset(400, 400)), + _page('two', 1, const Offset(700, 600)), + ]; + final exporter = _exporter(pages); + final sink = _RecordingSink(); + + final outcome = await _exportWithFrames(tester, exporter, pages, sink); + + expect(outcome, same(sink.video)); + final transitionFrames = VideoExporter.transitionFrameCountFor(30); + expect(sink.started, ( + width: 1920, + height: 1080, + totalFrames: 2 + transitionFrames, + totalSeconds: VideoExporter.plannedDurationSeconds( + pageCount: 2, + stepSeconds: 2, + fps: 30, + ), + )); + expect(sink.frames, hasLength(2 + transitionFrames)); + expect(sink.frames.first.$2, 2); + expect(sink.frames.last.$2, 2); + for (final (_, duration) in sink.frames.skip(1).take(transitionFrames)) { + expect(duration, closeTo(1 / 30, 1e-9)); + } + for (final (rgba, _) in sink.frames) { + expect(rgba.length, 1920 * 1080 * 4); + } + expect(_magentaPixels(sink.frames.first.$1), greaterThan(500)); + expect(_magentaPixels(sink.frames.last.$1), greaterThan(500)); + expect(sink.closed, 1); + expect(CoordinateSystem.instance.isScreenshot, isFalse); + }); + + testWidgets('an export never touches the app\'s cloud session', + (tester) async { + CoordinateSystem(playAreaSize: const Size(1600, 900)); + final convex = _RecordingConvexAuth(); + AuthProvider.debugConvexApi = convex; + AuthProvider.debugSupabaseApi = _NoSupabase(); + addTearDown(AuthProvider.resetTestOverrides); + final pages = [ + _page('one', 0, const Offset(400, 400)), + _page('two', 1, const Offset(700, 600)), + ]; + + final outcome = await _exportWithFrames( + tester, + _exporter(pages), + pages, + _RecordingSink(), + ); + await tester.runAsync( + () => Future.delayed(const Duration(milliseconds: 50)), + ); + + expect(outcome, isA()); + expect(convex.calls, isEmpty); + }); + + testWidgets('desktop encodes the frames into a playable .mp4 with ffmpeg', + (tester) async { + CoordinateSystem(playAreaSize: const Size(1600, 900)); + final pages = [ + _page('one', 0, const Offset(400, 400)), + _page('two', 1, const Offset(700, 600)), + ]; + final outputDir = (await tester.runAsync( + () => Directory.systemTemp.createTemp('icarus_video_out'), + ))!; + addTearDown(() => outputDir.deleteSync(recursive: true)); + final outputPath = '${outputDir.path}${Platform.pathSeparator}out.mp4'; + + final outcome = await _exportWithFrames( + tester, + _exporter(pages), + pages, + FfmpegVideoSink( + binary: _ffmpeg!, + outputPath: outputPath, + quality: VideoExportQuality.social, + ), + ); + + expect(outcome, isNull); + final probe = (await tester.runAsync( + () => Process.run(_ffprobe!, [ + '-v', + 'error', + '-show_entries', + 'stream=codec_name,width,height:format=duration', + '-of', + 'default=noprint_wrappers=1', + outputPath, + ]), + ))!; + final report = probe.stdout as String; + expect(report, contains('codec_name=h264')); + expect(report, contains('width=1920')); + expect(report, contains('height=1080')); + final seconds = double.parse( + RegExp(r'duration=([\d.]+)').firstMatch(report)!.group(1)!, + ); + expect( + seconds, + closeTo( + VideoExporter.plannedDurationSeconds( + pageCount: 2, + stepSeconds: 2, + fps: 30, + ), + 0.1, + ), + ); + // Needs a PATH ffmpeg and ffprobe, as local development has. + }, skip: _ffmpeg == null || _ffprobe == null); +} diff --git a/test/web/web_codecs_video_encoder_test.dart b/test/web/web_codecs_video_encoder_test.dart new file mode 100644 index 00000000..16df4458 --- /dev/null +++ b/test/web/web_codecs_video_encoder_test.dart @@ -0,0 +1,290 @@ +@TestOn('browser') +library; + +import 'dart:async'; +import 'dart:js_interop'; +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:icarus/services/video_export/video_export_errors.dart'; +import 'package:icarus/services/video_export/web_codecs_video_encoder.dart'; + +import '../mp4_box_test_support.dart'; + +// Run with: flutter test --platform chrome test/web/web_codecs_video_encoder_test.dart +// Needs a Chrome or Edge build with H.264 (Chromium builds lack it); point +// CHROME_EXECUTABLE at it when Chrome is not installed. + +void main() { + test('Chrome can encode H.264 at every export size', () async { + for (final (width, height, fps) in [ + (1920, 1080, 60), + (1920, 1080, 30), + (1280, 720, 30), + (640, 360, 30), + ]) { + expect( + await WebCodecsMp4Encoder.isSupported( + width: width, + height: height, + fps: fps, + ), + isTrue, + reason: '${width}x$height@$fps', + ); + } + expect( + await WebCodecsMp4Encoder.isSupported( + width: 1920, + height: 1080, + fps: 30, + bitrate: 2000000, + ), + isTrue, + ); + }); + + test('encodes held frames into an MP4 the browser plays back', () async { + final encoder = WebCodecsMp4Encoder(); + await encoder.start( + inputWidth: 640, + inputHeight: 360, + outputWidth: 640, + outputHeight: 360, + fps: 30, + ); + await encoder.addFrame( + _solid(640, 360, 255, 0, 0), + duration: const Duration(seconds: 1), + ); + await encoder.addFrame( + _solid(640, 360, 0, 255, 0), + duration: const Duration(milliseconds: 500), + ); + await encoder.addFrame( + _solid(640, 360, 0, 0, 255), + duration: const Duration(seconds: 1), + ); + final mp4 = await encoder.finish(); + + final boxes = Mp4Boxes(mp4); + expect(boxes.topLevelTypes, ['ftyp', 'moov', 'mdat']); + expect(boxes.sampleSizes(), hasLength(75)); + expect(boxes.mediaTimescale(), 30000); + expect(boxes.mediaDuration(), 75000); + expect(boxes.sampleDurations().toSet(), {1000}); + // Each held page opens on a keyframe. + expect(boxes.syncSamples() ?? [for (var i = 1; i <= 75; i++) i], + containsAll([1, 31, 46])); + expect(boxes.trackSize(), (640, 360)); + + final video = await _loadVideo(mp4); + expect(video.videoWidth, 640); + expect(video.videoHeight, 360); + expect(video.duration, closeTo(2.5, 0.05)); + expect(await _colourAt(video, 0.5), 'red'); + expect(await _colourAt(video, 1.25), 'green'); + expect(await _colourAt(video, 2.2), 'blue'); + }); + + test('scales 1280x720 frames down to a 640x360 video', () async { + final encoder = WebCodecsMp4Encoder(); + await encoder.start( + inputWidth: 1280, + inputHeight: 720, + outputWidth: 640, + outputHeight: 360, + fps: 30, + bitrate: 1000000, + ); + await encoder.addFrame( + _solid(1280, 720, 0, 255, 0), + duration: const Duration(milliseconds: 500), + ); + await encoder.addFrame( + _solid(1280, 720, 0, 0, 255), + duration: const Duration(milliseconds: 500), + ); + final mp4 = await encoder.finish(); + + final boxes = Mp4Boxes(mp4); + expect(boxes.sampleSizes(), hasLength(30)); + expect(boxes.trackSize(), (640, 360)); + + final video = await _loadVideo(mp4); + expect(video.videoWidth, 640); + expect(video.videoHeight, 360); + expect(video.duration, closeTo(1.0, 0.05)); + expect(await _colourAt(video, 0.25), 'green'); + expect(await _colourAt(video, 0.75), 'blue'); + }); + + test('carries fractional frame remainders forward', () async { + final encoder = WebCodecsMp4Encoder(); + await encoder.start( + inputWidth: 64, + inputHeight: 64, + outputWidth: 64, + outputHeight: 64, + fps: 30, + ); + // 50 ms is 1.5 frames at 30 fps: the runs are 2, 1, 2 frames so the + // total stays round(150 ms * 30) = 5 frames (rounded half up). + for (var i = 0; i < 3; i++) { + await encoder.addFrame( + _solid(64, 64, 80 * i, 0, 0), + duration: const Duration(milliseconds: 50), + ); + } + final boxes = Mp4Boxes(await encoder.finish()); + expect(boxes.sampleSizes(), hasLength(5)); + expect(boxes.mediaDuration(), 5000); + }); + + test('cancel is idempotent and later calls report the cancel', () async { + final encoder = WebCodecsMp4Encoder(); + await encoder.start( + inputWidth: 64, + inputHeight: 64, + outputWidth: 64, + outputHeight: 64, + fps: 30, + ); + encoder + ..cancel() + ..cancel(); + await expectLater( + encoder.addFrame(_solid(64, 64, 0, 0, 0), duration: Duration.zero), + throwsA(isA()), + ); + }); +} + +Uint8List _solid(int width, int height, int r, int g, int b) { + final bytes = Uint8List(width * height * 4); + for (var i = 0; i < bytes.length; i += 4) { + bytes[i] = r; + bytes[i + 1] = g; + bytes[i + 2] = b; + bytes[i + 3] = 255; + } + return bytes; +} + +Future<_VideoElement> _loadVideo(Uint8List mp4) async { + final url = _Url.createObjectURL( + _Blob([mp4.toJS].toJS, _BlobOptions(type: 'video/mp4')), + ); + addTearDown(() => _Url.revokeObjectURL(url)); + final video = _document.createElement('video') as _VideoElement + ..muted = true + ..preload = 'auto'; + final loaded = _nextEvent(video, 'loadeddata'); + video.src = url; + await loaded; + return video; +} + +/// Seeks [video] to [seconds] and names the dominant colour at its centre. +Future _colourAt(_VideoElement video, double seconds) async { + final seeked = _nextEvent(video, 'seeked'); + video.currentTime = seconds; + await seeked; + final canvas = _document.createElement('canvas') as _Canvas + ..width = video.videoWidth + ..height = video.videoHeight; + final context = canvas.getContext('2d'); + context.drawImage(video, 0, 0); + final pixel = context + .getImageData(video.videoWidth ~/ 2, video.videoHeight ~/ 2, 1, 1) + .data + .toDart; + final (r, g, b) = (pixel[0], pixel[1], pixel[2]); + if (r > 180 && g < 80 && b < 80) return 'red'; + if (g > 180 && r < 80 && b < 80) return 'green'; + if (b > 180 && r < 80 && g < 80) return 'blue'; + return 'rgb($r, $g, $b)'; +} + +/// Completes on [video]'s next [type] event, or fails on its error event. +Future _nextEvent(_VideoElement video, String type) { + final done = Completer(); + late JSFunction onEvent; + late JSFunction onError; + void detach() { + video + ..removeEventListener(type, onEvent) + ..removeEventListener('error', onError); + } + + onEvent = ((JSAny? _) { + detach(); + if (!done.isCompleted) done.complete(); + }).toJS; + onError = ((JSAny? _) { + detach(); + if (!done.isCompleted) { + done.completeError( + StateError('The video element failed: code ${video.error?.code}'), + ); + } + }).toJS; + video + ..addEventListener(type, onEvent) + ..addEventListener('error', onError); + return done.future.timeout(const Duration(seconds: 20)); +} + +@JS('document') +external _Document get _document; + +extension type _Document._(JSObject _) implements JSObject { + external JSObject createElement(String tag); +} + +extension type _VideoElement._(JSObject _) implements JSObject { + external set src(String value); + external set muted(bool value); + external set preload(String value); + external set currentTime(num value); + external int get videoWidth; + external int get videoHeight; + external double get duration; + external _MediaError? get error; + external void addEventListener(String type, JSFunction listener); + external void removeEventListener(String type, JSFunction listener); +} + +extension type _MediaError._(JSObject _) implements JSObject { + external int get code; +} + +extension type _Canvas._(JSObject _) implements JSObject { + external set width(int value); + external set height(int value); + external _Context2D getContext(String contextId); +} + +extension type _Context2D._(JSObject _) implements JSObject { + external void drawImage(JSObject image, num dx, num dy); + external _ImageData getImageData(int x, int y, int width, int height); +} + +extension type _ImageData._(JSObject _) implements JSObject { + external JSUint8ClampedArray get data; +} + +@JS('Blob') +extension type _Blob._(JSObject _) implements JSObject { + external factory _Blob(JSArray parts, _BlobOptions options); +} + +extension type _BlobOptions._(JSObject _) implements JSObject { + external factory _BlobOptions({String type}); +} + +@JS('URL') +extension type _Url._(JSObject _) implements JSObject { + external static String createObjectURL(_Blob blob); + external static void revokeObjectURL(String url); +} diff --git a/test/widgets/web_beta_library_test.dart b/test/widgets/web_beta_library_test.dart index c6151725..84ddf531 100644 --- a/test/widgets/web_beta_library_test.dart +++ b/test/widgets/web_beta_library_test.dart @@ -272,7 +272,7 @@ void main() { expect(find.text('Desktop-only for now'), findsOneWidget); expect( find.text( - 'Export · Import · Video export · Drag and drop', + 'Export · Import · Drag and drop', ), findsOneWidget, );