diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 547e1e35..53d5e196 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -269,7 +269,7 @@ jobs: env: RUSTFLAGS: --cfg fframes_docsrs RUSTDOCFLAGS: --cfg docsrs -D warnings - run: cargo doc --no-deps -p fframes-media -p fframes --features fframes/cli,fframes/captions,fframes/compile-time-svgtree,fframes/exif,fframes/3d + run: cargo doc --no-deps -p fframes-media -p fframes --features fframes/cli,fframes/captions,fframes/srt,fframes/compile-time-svgtree,fframes/exif,fframes/3d release: # A push to main publishes a nightly, a v* tag publishes that version (scripts/release-version.sh) @@ -362,7 +362,7 @@ jobs: cargo clippy $PACKAGES --tests -- -D warnings - name: Unit tests - run: cargo test -p fframes -p fframes-media -p webvtt-parser -p fframes-media-dir-macro --features fframes/captions + run: cargo test -p fframes -p fframes-media -p webvtt-parser -p fframes-media-dir-macro --features fframes/captions,fframes/srt - name: Render a video run: cargo test -p e2e-tests --test render_smoke -- --nocapture diff --git a/e2e/snapshots/captions/7.8s.png b/e2e/snapshots/captions/7.8s.png new file mode 100644 index 00000000..18de67e7 Binary files /dev/null and b/e2e/snapshots/captions/7.8s.png differ diff --git a/examples/captions/Cargo.toml b/examples/captions/Cargo.toml index f519d762..a68530c6 100644 --- a/examples/captions/Cargo.toml +++ b/examples/captions/Cargo.toml @@ -5,7 +5,7 @@ edition = "2024" publish = false [dependencies] -fframes = { workspace = true, features = ["cli", "captions", "compile-time-svgtree"] } +fframes = { workspace = true, features = ["cli", "captions", "srt", "compile-time-svgtree"] } serde = { version = "1", features = ["derive"] } [features] diff --git a/examples/captions/media/outro.srt b/examples/captions/media/outro.srt new file mode 100644 index 00000000..3d35f61b --- /dev/null +++ b/examples/captions/media/outro.srt @@ -0,0 +1,4 @@ +1 +00:00:07,650 --> 00:00:09,000 +The same captions also load +from a SubRip (.srt) file. diff --git a/examples/captions/src/lib.rs b/examples/captions/src/lib.rs index 3ba205e1..31297b6f 100644 --- a/examples/captions/src/lib.rs +++ b/examples/captions/src/lib.rs @@ -1,4 +1,5 @@ -//! A small captions example: one media track, four presets and a typed word highlight. +//! A small captions example: one media track, four presets and a typed word highlight, +//! then a closing line loaded from an SRT file through the same media path. use fframes::captions::{CaptionMode, CaptionRect, Captions}; use fframes::{AudioMap, Duration, FFramesContext, Frame, Svgr, Video, include_media_dir, svgr}; @@ -13,6 +14,7 @@ enum Highlight { pub struct CaptionVideo { modes: [Captions; 4], + outro: Captions, } impl Default for CaptionVideo { @@ -33,6 +35,11 @@ impl Default for CaptionVideo { .upcoming_fill("#64748b") .outline("#0c1423", 2.0) }), + outro: Captions::from_media("outro.srt", CaptionRect::new(1010.0, 40.0, 820.0, 130.0)) + .font("DM Sans", 44) + .font_weight(500) + .active_fill("#f4f7fb") + .outline("#0c1423", 2.0), } } } @@ -43,7 +50,7 @@ impl Video for CaptionVideo { const HEIGHT: usize = 1080; fn duration(&self) -> Duration<'_> { - Duration::Seconds(8.0) + Duration::Seconds(9.0) } fn audio(&self) -> AudioMap<'_> { @@ -72,6 +79,7 @@ impl Video for CaptionVideo { let karaoke = self.modes[1].render(&mut frame, ctx); let reveal = self.modes[2].render(&mut frame, ctx); let word = self.modes[3].render(&mut frame, ctx); + let outro = self.outro.render(&mut frame, ctx); svgr!( @@ -85,7 +93,7 @@ impl Video for CaptionVideo { "WORD" - {highlight}{karaoke}{reveal}{word} + {highlight}{karaoke}{reveal}{word}{outro} ) } diff --git a/fframes-editor-controller/src/wasm_editor.rs b/fframes-editor-controller/src/wasm_editor.rs index fe16f6d6..24a8be6e 100644 --- a/fframes-editor-controller/src/wasm_editor.rs +++ b/fframes-editor-controller/src/wasm_editor.rs @@ -185,7 +185,7 @@ impl + 'static> WasmEditor Result { use fframes::media::FFramesSubtitles; diff --git a/fframes-editor/src/services/mediaLoader.bs.js b/fframes-editor/src/services/mediaLoader.bs.js index 7b018767..6639921e 100644 --- a/fframes-editor/src/services/mediaLoader.bs.js +++ b/fframes-editor/src/services/mediaLoader.bs.js @@ -155,6 +155,7 @@ function processDynamicMedia(imports, wasmController, options) { case "ttf" : tmp = resolveFont; break; + case "srt" : case "vtt" : tmp = resolveSubtitles; break; diff --git a/fframes-editor/src/services/mediaLoader.res b/fframes-editor/src/services/mediaLoader.res index b3815afd..7fab679e 100644 --- a/fframes-editor/src/services/mediaLoader.res +++ b/fframes-editor/src/services/mediaLoader.res @@ -178,7 +178,7 @@ let processDynamicMedia = ( } else { switch name->Js.String.toLowerCase->Js.String.split(".")->Utils.Array.last { | Some("mp3" | "aac" | "ogg" | "wav" | "pcm" | "flac") => Some(resolveAudio) - | Some("vtt") => Some(resolveSubtitles) + | Some("vtt" | "srt") => Some(resolveSubtitles) | Some("json") => Some(resolveJson) | Some("ttf" | "otf") => Some(resolveFont) | Some("png" | "jpg" | "jpeg") => Some(resolveImage) diff --git a/fframes-media/Cargo.toml b/fframes-media/Cargo.toml index 5319e758..613b9cc2 100644 --- a/fframes-media/Cargo.toml +++ b/fframes-media/Cargo.toml @@ -54,6 +54,8 @@ ffmpeg-sys-fframes = { workspace = true, features = ["build", "static"] } # by default we enable only platform specific encoders that # would be always populated by the platform sysroot itself default = [] +# SubRip (`.srt`) subtitles: `CaptionTrack::from_srt` and detection in `CaptionTrack::parse` +srt = [] # Preserve prebuilt downloads and omit host CPU flags when building from source. build-portable = ["ffmpeg-sys-fframes/build-portable"] # enable various encoder & decoders libraries diff --git a/fframes-media/src/lib.rs b/fframes-media/src/lib.rs index d2469aec..6ec23d36 100644 --- a/fframes-media/src/lib.rs +++ b/fframes-media/src/lib.rs @@ -4,6 +4,9 @@ mod error; mod font; mod images; mod raw_file; +// Detection is always compiled so that SRT input without the feature gets a clear error. +#[cfg_attr(not(feature = "srt"), allow(dead_code))] +mod srt; pub mod subtitles; mod subtitles_data; mod video_types; diff --git a/fframes-media/src/srt.rs b/fframes-media/src/srt.rs new file mode 100644 index 00000000..d5c7b999 --- /dev/null +++ b/fframes-media/src/srt.rs @@ -0,0 +1,325 @@ +//! `SubRip` (`.srt`) cue parsing for [`CaptionTrack::from_srt`](crate::CaptionTrack::from_srt). +//! +//! A cue is an optional number line, a `00:00:01,000 --> 00:00:02,500` timing line and one or +//! more text lines, ended by a blank line. Text is kept raw, like `WebVTT` payloads: inline +//! tags such as `` are left for a renderer to interpret. + +use crate::CaptionError; + +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct SrtCue { + /// The cue number as written. `SubRip` numbers are not guaranteed to be unique. + pub number: Option, + pub start: f64, + pub end: f64, + pub text: String, +} + +/// Whether a document without a `WEBVTT` header or JSON object looks like `SubRip`: +/// its first line is a cue number or a timing line. +pub(crate) fn looks_like_srt(content: &str) -> bool { + lines(content) + .map(|(_, line)| line.trim()) + .find(|line| !line.is_empty()) + .is_some_and(|line| is_cue_number(line) || line.contains("-->")) +} + +/// Parse every cue in document order. Errors name the 1-based line they occur on. +/// Cues without text are dropped, as `WebVTT` spacer cues are. +pub(crate) fn parse(content: &str) -> Result, CaptionError> { + let content = content.strip_prefix('\u{feff}').unwrap_or(content); + let mut lines = lines(content).peekable(); + let mut cues = Vec::new(); + while let Some((line_number, line)) = lines.next() { + let line = line.trim(); + if line.is_empty() { + continue; + } + let (number, timing_line_number, timing) = if line.contains("-->") { + (None, line_number, line) + } else if is_cue_number(line) { + match lines.next() { + Some((next_number, next)) if next.contains("-->") => { + (Some(line.to_owned()), next_number, next.trim()) + } + _ => { + return Err(error( + line_number + 1, + "expected a timing line such as `00:00:01,000 --> 00:00:02,000` after the cue number", + )); + } + } + } else { + return Err(error( + line_number, + format!("expected a cue number or a timing line, found `{line}`"), + )); + }; + let (start, end) = parse_timing(timing, timing_line_number)?; + + let mut text = String::new(); + while let Some((_, text_line)) = lines.next_if(|(_, line)| !line.trim().is_empty()) { + if !text.is_empty() { + text.push('\n'); + } + text.push_str(text_line.trim_end()); + } + if !text.is_empty() { + cues.push(SrtCue { + number, + start, + end, + text, + }); + } + } + Ok(cues) +} + +/// Split on `\n`, `\r\n` or a lone `\r`, numbering lines from 1. +fn lines(content: &str) -> impl Iterator { + let mut rest = Some(content); + std::iter::from_fn(move || { + let current = rest?; + if let Some(index) = current.find(['\r', '\n']) { + let skip = if current[index..].starts_with("\r\n") { + 2 + } else { + 1 + }; + rest = Some(¤t[index + skip..]); + Some(¤t[..index]) + } else { + rest = None; + Some(current) + } + }) + .enumerate() + .map(|(index, line)| (index + 1, line)) +} + +fn is_cue_number(line: &str) -> bool { + !line.is_empty() && line.bytes().all(|byte| byte.is_ascii_digit()) +} + +fn parse_timing(line: &str, line_number: usize) -> Result<(f64, f64), CaptionError> { + let (start, rest) = line + .split_once("-->") + .expect("timing lines contain an arrow"); + // Anything after the end time is legacy SubRip positioning (`X1:… Y1:…`), ignored here. + let end = rest.split_whitespace().next().unwrap_or_default(); + let start = parse_timestamp(start.trim(), line_number)?; + let end = parse_timestamp(end, line_number)?; + if end < start { + return Err(error(line_number, "cue ends before it starts")); + } + Ok((start, end)) +} + +/// `HH:MM:SS,mmm`. Also accepts `.` before the fraction, one to three fraction digits, +/// any number of hour digits, and `MM:SS,mmm` without hours. +fn parse_timestamp(value: &str, line_number: usize) -> Result { + let invalid = || { + error( + line_number, + format!("invalid timestamp `{value}`, expected `HH:MM:SS,mmm`"), + ) + }; + let (clock, fraction) = value.split_once([',', '.']).ok_or_else(invalid)?; + if fraction.is_empty() + || fraction.len() > 3 + || !fraction.bytes().all(|byte| byte.is_ascii_digit()) + { + return Err(invalid()); + } + let fraction_milliseconds = + fraction.parse::().map_err(|_| invalid())? * 10u64.pow(3 - fraction.len() as u32); + + let parts = clock.split(':').collect::>(); + let (hours, minutes, seconds) = match parts.as_slice() { + [hours, minutes, seconds] => (*hours, *minutes, *seconds), + [minutes, seconds] => ("0", *minutes, *seconds), + _ => return Err(invalid()), + }; + let number = |part: &str, two_digits: bool| -> Result { + if part.is_empty() + || (two_digits && part.len() != 2) + || !part.bytes().all(|byte| byte.is_ascii_digit()) + { + return Err(invalid()); + } + part.parse().map_err(|_| invalid()) + }; + let hours = number(hours, false)?; + let minutes = number(minutes, true)?; + let seconds = number(seconds, true)?; + if minutes >= 60 || seconds >= 60 { + return Err(invalid()); + } + let milliseconds = hours + .checked_mul(3_600_000) + .and_then(|value| { + value.checked_add((minutes * 60 + seconds) * 1000 + fraction_milliseconds) + }) + .ok_or_else(invalid)?; + Ok(milliseconds as f64 / 1000.0) +} + +fn error(line: usize, message: impl Into) -> CaptionError { + CaptionError::invalid(format!("SRT line {line}"), message) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn cue(number: Option<&str>, start: f64, end: f64, text: &str) -> SrtCue { + SrtCue { + number: number.map(str::to_owned), + start, + end, + text: text.to_owned(), + } + } + + fn error_message(content: &str) -> String { + parse(content).unwrap_err().to_string() + } + + #[test] + fn parses_numbered_cues_and_multiline_text() { + let cues = parse( + "1\n00:00:01,000 --> 00:00:02,500\nFirst line\nSecond line\n\n\ + 2\n00:00:03,000 --> 00:00:04,000\nNext\n", + ) + .unwrap(); + assert_eq!( + cues, + [ + cue(Some("1"), 1.0, 2.5, "First line\nSecond line"), + cue(Some("2"), 3.0, 4.0, "Next"), + ] + ); + } + + #[test] + fn handles_bom_crlf_lone_cr_and_trailing_whitespace() { + let expected = [ + cue(Some("1"), 0.5, 1.0, "Hello\nworld"), + cue(Some("2"), 2.0, 3.0, "Again"), + ]; + for document in [ + "\u{feff}1\r\n00:00:00,500 --> 00:00:01,000\r\nHello \r\nworld\r\n\r\n2\r\n00:00:02,000 --> 00:00:03,000\r\nAgain\r\n", + "1\r00:00:00,500 --> 00:00:01,000\rHello\rworld\r\r2\r00:00:02,000 --> 00:00:03,000\rAgain", + "\n\n1\n00:00:00,500 --> 00:00:01,000\nHello\nworld\n \t\n\n\n2\n00:00:02,000 --> 00:00:03,000\nAgain\n\n\n", + ] { + assert_eq!(parse(document).unwrap(), expected, "{document:?}"); + } + } + + #[test] + fn keeps_overlapping_and_out_of_order_cues_in_document_order() { + let cues = parse( + "1\n00:00:01,000 --> 00:00:05,000\nNarrator\n\n\ + 2\n00:00:02,000 --> 00:00:03,000\nOverlaps\n\n\ + 3\n00:00:00,000 --> 00:00:00,800\nEarlier\n", + ) + .unwrap(); + assert_eq!( + cues.iter().map(|cue| cue.start).collect::>(), + [1.0, 2.0, 0.0] + ); + } + + #[test] + fn accepts_common_timestamp_variants() { + let cues = parse( + "00:00:01.250 --> 00:00:02,5 X1:10 X2:20 Y1:30 Y2:40\nNo number, dot, short fraction\n\n\ + 7\n01:02:03,004 --> 101:00:00,000\nLong hours\n\n\ + 8\n00:05,000 --> 00:06,000\nNo hours\n", + ) + .unwrap(); + assert_eq!( + cues[0], + cue(None, 1.25, 2.5, "No number, dot, short fraction") + ); + assert_eq!(cues[1].start, 3723.004); + assert_eq!(cues[1].end, 363_600.0); + assert_eq!((cues[2].start, cues[2].end), (5.0, 6.0)); + } + + #[test] + fn keeps_raw_markup_and_numeric_text_lines() { + let cues = parse("1\n00:00:01,000 --> 00:00:02,000\nThe year\n1984\n").unwrap(); + assert_eq!(cues, [cue(Some("1"), 1.0, 2.0, "The year\n1984")]); + } + + #[test] + fn drops_cues_without_text_and_allows_instant_cues() { + let cues = parse( + "1\n00:00:01,000 --> 00:00:02,000\n\n\ + 2\n00:00:03,000 --> 00:00:03,000\nInstant\n", + ) + .unwrap(); + assert_eq!(cues, [cue(Some("2"), 3.0, 3.0, "Instant")]); + assert_eq!(parse("").unwrap(), []); + assert_eq!(parse("\u{feff}\r\n\r\n").unwrap(), []); + } + + #[test] + fn rejects_malformed_lines_with_their_line_number() { + for (document, expected) in [ + ( + "Hello\n", + "SRT line 1: expected a cue number or a timing line", + ), + ("1\nHello\n", "SRT line 2: expected a timing line"), + ("1\n", "SRT line 2: expected a timing line"), + ( + "1\n\n00:00:01,000 --> 00:00:02,000\nx\n", + "SRT line 2: expected a timing line", + ), + ( + "1\n00:00:01 --> 00:00:02,000\nx\n", + "SRT line 2: invalid timestamp `00:00:01`", + ), + ( + "1\n00:00:01,000 -->\nx\n", + "SRT line 2: invalid timestamp ``", + ), + ( + "1\n00:00:01,0000 --> 00:00:02,000\nx\n", + "invalid timestamp", + ), + ( + "1\n00:61:01,000 --> 00:62:02,000\nx\n", + "invalid timestamp `00:61:01,000`", + ), + ("1\n00:00:1,000 --> 00:00:02,000\nx\n", "invalid timestamp"), + ( + "1\n-00:00:01,000 --> 00:00:02,000\nx\n", + "invalid timestamp", + ), + ("1\n0:0:0:1,000 --> 00:00:02,000\nx\n", "invalid timestamp"), + ( + "1\n00:00:05,000 --> 00:00:02,000\nx\n", + "SRT line 2: cue ends before it starts", + ), + ( + "1\n00:00:01,000 --> 00:00:02,000\nOne\n\nstray text after a blank line\n", + "SRT line 5: expected a cue number or a timing line, found `stray text after a blank line`", + ), + ] { + let message = error_message(document); + assert!(message.contains(expected), "{document:?}: {message}"); + } + } + + #[test] + fn detects_srt_by_its_first_line() { + assert!(looks_like_srt("\n\n1\n00:00:01,000 --> 00:00:02,000\nx")); + assert!(looks_like_srt("00:00:01,000 --> 00:00:02,000\nx")); + assert!(!looks_like_srt("Hello\n00:00:01,000 --> 00:00:02,000")); + assert!(!looks_like_srt("")); + } +} diff --git a/fframes-media/src/subtitles.rs b/fframes-media/src/subtitles.rs index 1de0d68f..1909bdbe 100644 --- a/fframes-media/src/subtitles.rs +++ b/fframes-media/src/subtitles.rs @@ -1,6 +1,7 @@ -//! Subtitle data and cue lookup for `WebVTT` and ordinary timed-word JSON. +//! Subtitle data and cue lookup for `WebVTT`, `SubRip` (SRT) and ordinary timed-word JSON. //! -//! [`Subtitles::parse`] detects the input format from its content. Both formats +//! [`Subtitles::parse`] detects the input format from its content; `SubRip` needs the +//! `srt` feature. All formats //! share owned, validated cue data and the same media-loading and frame APIs. //! Metadata decodes lazily on first typed access and is cached by application type. //! Legacy frame helpers keep inclusive cue ends; [`CaptionTrack::active_cues`] @@ -14,7 +15,7 @@ pub use crate::subtitles_data::{ CaptionCue, CaptionError, CaptionMetadata, CaptionTrack, CaptionWord, WordState, }; -/// Compatibility name for [`CaptionTrack`], the owned VTT/JSON subtitle model. +/// Compatibility name for [`CaptionTrack`], the owned VTT/SRT/JSON subtitle model. pub type Subtitles = CaptionTrack; /// Compatibility name for subtitle loading errors. pub type SubtitlesError = CaptionError; diff --git a/fframes-media/src/subtitles_data.rs b/fframes-media/src/subtitles_data.rs index 34fef04d..6e580350 100644 --- a/fframes-media/src/subtitles_data.rs +++ b/fframes-media/src/subtitles_data.rs @@ -389,7 +389,7 @@ pub struct CaptionCue { /// Application-defined cue data. Interpretation and validation belong to the renderer. #[serde(default, skip_serializing_if = "Map::is_empty")] pub metadata: CaptionMetadata, - /// The original `WebVTT` identifier. It is not required to be unique. + /// The original `WebVTT` identifier or SRT cue number. It is not required to be unique. #[serde(skip)] pub name: Option, /// Raw `WebVTT` cue note, when present. @@ -539,9 +539,25 @@ pub struct CaptionTrack { #[serde(skip)] pub style: Option, #[serde(skip)] - source_vtt: bool, + source: CaptionSource, } +/// The document format a track was parsed from. Subtitle formats permit +/// instantaneous cues, which the canonical JSON representation rejects. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum CaptionSource { + Json, + Vtt, + #[cfg(feature = "srt")] + Srt, +} + +#[cfg(feature = "srt")] +const UNKNOWN_FORMAT: &str = + "expected a WEBVTT document, a SubRip (SRT) document or a JSON object with a cues array"; +#[cfg(not(feature = "srt"))] +const UNKNOWN_FORMAT: &str = "expected a WEBVTT document or a JSON object with a cues array"; + #[derive(Deserialize)] #[serde(deny_unknown_fields)] struct TrackInput { @@ -606,19 +622,19 @@ impl<'de> Deserialize<'de> for CaptionTrack { impl CaptionTrack { /// Validate and own a sequence of cues. An empty track is valid. pub fn new(cues: impl IntoIterator) -> Result { - Self::from_cues(cues, false) + Self::from_cues(cues, CaptionSource::Json) } fn from_cues( cues: impl IntoIterator, - source_vtt: bool, + source: CaptionSource, ) -> Result { let mut track = Self { cues: cues.into_iter().collect::>().into(), identity: Arc::new(()), slugs: HashMap::new(), style: None, - source_vtt, + source, }; track.validate()?; for cue in Arc::make_mut(&mut track.cues) { @@ -627,20 +643,27 @@ impl CaptionTrack { Ok(track) } - /// Parse ordinary JSON or `WebVTT`. The format is detected from its contents, - /// independent of the filename. All parsed data is owned. + /// Parse ordinary JSON, `WebVTT` or, with the `srt` feature, `SubRip`. The format is + /// detected from its contents, independent of the filename. All parsed data is owned. pub fn parse(content: &str) -> Result { - let content = normalize_input(content); - if content.starts_with("WEBVTT") { - Self::from_vtt(content) - } else if content.starts_with('{') { - Self::from_json(content) - } else { - Err(CaptionError::invalid( + let normalized = normalize_input(content); + if normalized.starts_with("WEBVTT") { + return Self::from_vtt(normalized); + } + if normalized.starts_with('{') { + return Self::from_json(normalized); + } + if crate::srt::looks_like_srt(normalized) { + // The original text keeps SRT error line numbers accurate. + #[cfg(feature = "srt")] + return Self::from_srt(content); + #[cfg(not(feature = "srt"))] + return Err(CaptionError::invalid( "format", - "expected a WEBVTT document or a JSON object with a cues array", - )) + "SubRip (SRT) subtitles require the `srt` feature of fframes", + )); } + Err(CaptionError::invalid("format", UNKNOWN_FORMAT)) } /// Classify and load a JSON document with a single textual JSON parse. @@ -684,7 +707,7 @@ impl CaptionTrack { } /// Serialize the canonical JSON representation with indented formatting. - /// Imported zero-duration VTT cues need a duration (or removal) before JSON export. + /// Imported zero-duration VTT or SRT cues need a duration (or removal) before JSON export. pub fn to_json_pretty(&self) -> Result { self.validate_cues(false)?; Ok(serde_json::to_string_pretty(self)?) @@ -719,16 +742,52 @@ impl CaptionTrack { imported.cue_settings = cue.cue_settings; cues.push(imported); } - let mut track = Self::from_cues(cues, true)?; + let mut track = Self::from_cues(cues, CaptionSource::Vtt)?; track.slugs = slugs; track.style = style; Ok(track) } + /// Import `SubRip` (`.srt`) cues as plain-text cues (`srt` feature). Multi-line cue text keeps its + /// line breaks, inline tags such as `` stay raw, and the cue number is kept in + /// [`CaptionCue::name`]. Cues are ordered by start time (stable for equal starts) + /// and may overlap. Empty cues are dropped; instantaneous cues behave as in `WebVTT`. + /// Malformed lines fail with their line number. Word timestamps are never inferred. + /// + /// ``` + /// use fframes_media::CaptionTrack; + /// + /// let track = CaptionTrack::from_srt("1\n00:00:01,000 --> 00:00:02,500\nHello\nworld\n")?; + /// assert_eq!(track.cues()[0].phrase(), "Hello\nworld"); + /// assert_eq!((track.cues()[0].start, track.cues()[0].end), (1.0, 2.5)); + /// assert!(track.is_srt()); + /// # Ok::<(), fframes_media::CaptionError>(()) + /// ``` + #[cfg(feature = "srt")] + pub fn from_srt(srt: &str) -> Result { + let mut cues = crate::srt::parse(srt)? + .into_iter() + .map(|cue| { + let mut imported = CaptionCue::text(cue.start, cue.end, cue.text); + imported.name = cue.number; + imported + }) + .collect::>(); + cues.sort_by(|a, b| a.start.total_cmp(&b.start)); + Self::from_cues(cues, CaptionSource::Srt) + } + /// Whether this track was parsed from `WebVTT`. Raw payload markup/settings /// remain available for a renderer to interpret explicitly. pub fn is_vtt(&self) -> bool { - self.source_vtt + self.source == CaptionSource::Vtt + } + + /// Whether this track was parsed from `SubRip` (SRT). Raw inline markup remains + /// available for a renderer to interpret explicitly. + #[cfg(feature = "srt")] + pub fn is_srt(&self) -> bool { + self.source == CaptionSource::Srt } /// The ordered, validated cues. Modify through a new track so validation stays intact. @@ -753,7 +812,7 @@ impl CaptionTrack { /// Validate timing, content, ordering and optional identifiers. pub fn validate(&self) -> Result<(), CaptionError> { - self.validate_cues(self.source_vtt) + self.validate_cues(self.source != CaptionSource::Json) } fn validate_cues(&self, allow_instant_vtt: bool) -> Result<(), CaptionError> { diff --git a/fframes-media/tests/subtitles.rs b/fframes-media/tests/subtitles.rs index a076114e..d4dc6c8c 100644 --- a/fframes-media/tests/subtitles.rs +++ b/fframes-media/tests/subtitles.rs @@ -266,3 +266,68 @@ fn metadata_edits_detach_the_cache_without_changing_other_consumers() { ); assert!(!std::ptr::eq(original.cues(), edited.cues())); } + +#[cfg(feature = "srt")] +#[test] +fn srt_is_detected_from_content_and_shares_the_caption_model() { + use fframes_media::FFramesSubtitles; + let srt = "\u{feff}\r\n1\r\n00:00:02,000 --> 00:00:04,000\r\nSecond\r\n\r\n\ + 2\r\n00:00:00,500 --> 00:00:03,000\r\nFirst\r\ntwo lines\r\n\r\n\ + 3\r\n00:00:05,000 --> 00:00:05,000\r\nInstant\r\n"; + let track = Subtitles::parse(srt).unwrap(); + assert!(track.is_srt()); + assert!(!track.is_vtt()); + + // Ordered by start time; overlaps stay simultaneous. + let phrases = track.get_whole_text(); + assert_eq!(phrases, ["First\ntwo lines", "Second", "Instant"]); + assert_eq!(track.cues()[0].name.as_deref(), Some("2")); + assert_eq!(track.active_cues(2.5).count(), 2); + assert_eq!(track.get_cue_by_time(2500).unwrap().1.phrase(), "Second"); + assert!(track.get_cue_by_time(499).is_none()); + + // A zero-duration cue matches WebVTT: inclusive legacy lookup, never active. + assert_eq!(track.get_cue_by_time(5000).unwrap().1.phrase(), "Instant"); + assert_eq!(track.active_cues(5.0).count(), 0); + assert!(track.to_json_pretty().is_err()); + + assert_eq!( + Subtitles::parse("00:00:01,000 --> 00:00:02,000\nNo number\n") + .unwrap() + .cues()[0] + .phrase(), + "No number" + ); + assert_eq!(Subtitles::from_srt("").unwrap().cues(), []); +} + +#[cfg(feature = "srt")] +#[test] +fn malformed_srt_reports_the_original_line() { + // Leading blank lines and a BOM must not shift the reported line. + let error = Subtitles::parse("\u{feff}\n\n1\n00:00:01 --> 00:00:02,000\nx\n") + .unwrap_err() + .to_string(); + assert!( + error.starts_with("SRT line 4: invalid timestamp"), + "{error}" + ); + + let error = Subtitles::parse("Just some text\n") + .unwrap_err() + .to_string(); + assert!(error.contains("SubRip"), "{error}"); +} + +#[cfg(not(feature = "srt"))] +#[test] +fn srt_is_rejected_without_the_feature() { + let error = Subtitles::parse("1\n00:00:01,000 --> 00:00:02,000\nHello\n") + .unwrap_err() + .to_string(); + assert!(error.contains("require the `srt` feature"), "{error}"); + let error = Subtitles::parse("Just some text\n") + .unwrap_err() + .to_string(); + assert!(error.contains("expected a WEBVTT document"), "{error}"); +} diff --git a/fframes/Cargo.toml b/fframes/Cargo.toml index 7de4f70b..4445a12e 100644 --- a/fframes/Cargo.toml +++ b/fframes/Cargo.toml @@ -71,6 +71,8 @@ cpu_renderer = ["svgr"] cli = ["cpu_renderer", "dep:clap", "dep:serde_json"] # Declarative SVG caption rendering. Subtitle data/parsing is part of fframes-media. captions = [] +# SubRip (`.srt`) subtitle files, loaded and rendered like `.vtt` +srt = ["fframes-media/srt", "fframes-media-dir-macro/srt"] exif = ["fframes-media/exif", "fframes-media-dir-macro/exif"] compile-time-svgtree = ["svgr-macro/compile-time-svgtree"] # `fframes::Lottie` and `frame.play_lottie`: Lottie `.json` files in the media folder diff --git a/fframes/src/captions/fixtures-srt/empty.srt b/fframes/src/captions/fixtures-srt/empty.srt new file mode 100644 index 00000000..e69de29b diff --git a/fframes/src/captions/fixtures-srt/lines.srt b/fframes/src/captions/fixtures-srt/lines.srt new file mode 100644 index 00000000..e75e56fb --- /dev/null +++ b/fframes/src/captions/fixtures-srt/lines.srt @@ -0,0 +1,4 @@ +1 +00:00:00,000 --> 00:00:01,000 +SubRip phrase +on two lines diff --git a/fframes/src/captions/tests.rs b/fframes/src/captions/tests.rs index 1a44b304..3a1a77d6 100644 --- a/fframes/src/captions/tests.rs +++ b/fframes/src/captions/tests.rs @@ -288,3 +288,29 @@ fn runtime_and_embedded_subtitles_use_the_same_provider_contract() { cloned.resolve_subtitles("speech.json").unwrap().cues() )); } + +// `include_media_dir!` defines module-level helpers, so a second media folder needs its own module. +#[cfg(feature = "srt")] +mod srt_media { + use crate::MediaProvider; + + crate::include_media_dir!(struct EmbeddedSrt, "fframes/src/captions/fixtures-srt"); + + #[test] + fn srt_media_loads_like_vtt_when_embedded_or_read_at_runtime() { + let embedded = EmbeddedSrt::new().unwrap(); + let directory = crate::MediaDirectory::read_folder(concat!( + env!("CARGO_MANIFEST_DIR"), + "/src/captions/fixtures-srt" + )) + .unwrap(); + let runtime = directory.process_media_source().unwrap(); + for provider in [&embedded as &dyn MediaProvider<'_>, &runtime] { + let srt = provider.resolve_subtitles("lines.srt").unwrap(); + assert!(srt.is_srt()); + assert_eq!(srt.cues()[0].phrase(), "SubRip phrase\non two lines"); + let empty = provider.resolve_subtitles("empty.srt").unwrap(); + assert!(empty.is_srt() && empty.cues().is_empty()); + } + } +} diff --git a/fframes/src/media_provider.rs b/fframes/src/media_provider.rs index a43e0e2b..0088908b 100644 --- a/fframes/src/media_provider.rs +++ b/fframes/src/media_provider.rs @@ -60,7 +60,7 @@ pub trait MediaProvider<'a>: Send + Sync + Debug { !self.get_all_video_data().is_empty() } - /// All loaded subtitles, whether parsed from VTT or JSON. + /// All loaded subtitles, whether parsed from VTT, SRT or JSON. fn get_all_subtitle_data(&self) -> Vec<(&media::Subtitles, &str)> { Vec::new() } diff --git a/fframes/src/renderer/media_directory.rs b/fframes/src/renderer/media_directory.rs index b99c2568..77fbcea8 100644 --- a/fframes/src/renderer/media_directory.rs +++ b/fframes/src/renderer/media_directory.rs @@ -112,6 +112,8 @@ impl MediaDirectory { Some("json") => Some( "JSON is not recognized as subtitles; animations require the lottie feature", ), + Some("srt") if cfg!(feature = "srt") => None, + Some("srt") => Some("SubRip subtitles require the srt feature"), Some( "jpg" | "jpeg" | "png" | "vtt" | "mp3" | "wav" | "flac" | "aac" | "pcm" | "ogg" | "mp2" | "m4a" | "mp4" | "webm" | "mkv" | "avi" | "mov" | "flv" @@ -135,7 +137,9 @@ impl MediaDirectory { json } else { DirectoryMedia::Raw(match extension.as_deref() { - Some("jpg" | "jpeg" | "png" | "vtt") => RawMediaFile::Data(fs::read(&path)?), + Some("jpg" | "jpeg" | "png" | "vtt" | "srt") => { + RawMediaFile::Data(fs::read(&path)?) + } _ => RawMediaFile::Stream(path.clone()), }) }; @@ -201,7 +205,7 @@ impl MediaDirectory { crate::AudioData::Preloaded(audio_data), ); } - ("vtt", RawMediaFile::Data(bytes)) => { + ("vtt" | "srt", RawMediaFile::Data(bytes)) => { let subtitles = parse_subtitles(path, bytes)?; subtitles_hash .lock()? @@ -312,6 +316,18 @@ fn parse_subtitles(path: &Path, bytes: &[u8]) -> FFramesRendererResult TokenStream { let fframes_crate_ident = resolve_fframes_crate_ident(); @@ -294,6 +295,11 @@ impl MediaVariant { "ttf" | "ttc" | "otf" | "otc" => Some(MediaVariant::Font), "jpeg" | "png" | "jpg" => Some(MediaVariant::Image), "vtt" => Some(MediaVariant::Subtitles), + "srt" if cfg!(feature = "srt") => Some(MediaVariant::Subtitles), + "srt" => panic!( + "{}: SubRip subtitles require the `srt` feature of fframes.", + path.display() + ), "json" if is_subtitle_json(file_name, path) => Some(MediaVariant::Subtitles), "json" if cfg!(feature = "lottie") => Some(MediaVariant::Lottie), "json" => panic!( @@ -309,6 +315,15 @@ impl MediaVariant { } } +fn is_empty_srt(path: &Path, content: &str) -> bool { + cfg!(feature = "srt") + && path + .extension() + .and_then(|extension| extension.to_str()) + .is_some_and(|extension| extension.eq_ignore_ascii_case("srt")) + && content.trim_start_matches('\u{feff}').trim().is_empty() +} + fn is_subtitle_json(file_name: &str, path: &Path) -> bool { let lower = file_name.to_ascii_lowercase(); if matches!(lower.as_str(), "subtitles.json" | "captions.json") @@ -418,9 +433,15 @@ impl MediaFile { // same parser used by runtime loading, without a second cue schema. let file_str = std::fs::read_to_string(&self.path).unwrap(); let file_str_literal = Literal::string(&file_str); + // An empty `.srt` is an empty track; content detection cannot tell its format. + let parse = if is_empty_srt(&self.path, &file_str) { + quote! { from_srt } + } else { + quote! { parse } + }; quote! { - #fframes_crate_ident::media::Subtitles::parse( + #fframes_crate_ident::media::Subtitles::#parse( #file_str_literal ).map_err(#fframes_crate_ident::media::FFramesMediaError::from)? } diff --git a/scripts/e2e.mjs b/scripts/e2e.mjs index 937410a0..030f3462 100644 --- a/scripts/e2e.mjs +++ b/scripts/e2e.mjs @@ -48,8 +48,8 @@ const allExamples = [ { dir: "anamorphic-video", pkg: "anamorphic-video", bin: "anamorphic-video", frames: "50%" }, { dir: "audio-announce", pkg: "audio-announce-example", bin: "audio_announce_example_bin", frames: "5s,30s" }, { dir: "beta", pkg: "beta-example", bin: "beta_example_bin", frames: "CodeDemoScene@50%,IphoneScene@50%,BetaExamples@50%" }, - // Preset layouts, typed highlights, wrapping and simultaneous cues. - { dir: "captions", pkg: "captions-example", bin: "captions-example", frames: "1.5s,5.2s,7s" }, + // Preset layouts, typed highlights, wrapping, simultaneous cues and a two-line SRT cue. + { dir: "captions", pkg: "captions-example", bin: "captions-example", frames: "1.5s,5.2s,7s,7.8s" }, { dir: "conference-splash-screen", pkg: "conference-splash-screen-example", bin: "conference_splash_screen_bin", frames: "SponsorScene@50%,SpeakerScene@50%" }, { dir: "fframes-intro", pkg: "fframes-intro", bin: "fframes-intro", frames: "CodeScene@50%,ShaderScene@2.6s,BenchmarkScene@12.4s,OutroScene@7s", skia: true }, { dir: "hello-world", pkg: "hello-world-example", bin: "hello_world_example_bin", frames: "5s,20s" }, diff --git a/skills/fframes-video/references/captions.md b/skills/fframes-video/references/captions.md index fdf30895..17487f0d 100644 --- a/skills/fframes-video/references/captions.md +++ b/skills/fframes-video/references/captions.md @@ -5,10 +5,11 @@ need a caption track. ## The data -A caption track is a WebVTT file or a JSON file in `media/`. `CaptionTrack::parse` detects -the format from the content; both become the same owned `CaptionTrack` (`Subtitles` is an -alias). Use VTT when you only have phrases. Use JSON when you have word timings or want -per-word settings: +A caption track is a WebVTT, SubRip (`.srt`, with the `srt` feature) or JSON file in `media/`. `CaptionTrack::parse` +detects the format from the content; all three become the same owned `CaptionTrack` +(`Subtitles` is an alias). Use VTT or SRT when you only have phrases (SRT cue text keeps its +line breaks, and inline tags such as `` stay raw). Use JSON when you have word timings or +want per-word settings: ```json {