Skip to content

Commit df1a3d7

Browse files
Zachary Guglinclaude
authored andcommitted
layers LA4c, LA4d, CC2f: footage that moves, is cut or fades is keyed in every export, each editor placing it as raea drew it
The manifest says what of a video a box it's in leaves showing (clip) and its object-position; layers.json keeps each footage clip's box, clip, object-position and opacity at the frames they change (keys). One reading of a picture (plan::picture: object-fit, object-position, cut only where its box or a box it's in cuts it inside the frame) places it in every export: - the FCP7 XML keyframes Basic Motion, crop and opacity, written per editor (NAME.premiere.xml, NAME.resolve.xml): Premiere's crop in Basic Motion, Resolve's as FCP7's Crop mirrored for a phone's take stored turned; Scale and Center each in that editor's clip at Scale 100 (Premiere: the media as described, a phone's take as stored; Resolve: the media fitted to the timeline), read from what each did with what it was given - the CapCut draft keys position, scale and opacity (common_keyframes); its crop holds as the first frame shows it, and a crop that changes over the clip (a split screen's seam) is said: CapCut can't show it - keys a straight line between gives to 0.15 px are left out; a clip at another speed is placed at its own, and said Also: a layer's ProRes movie that opens on an opaque frame keeps one sample description with alpha (VideoToolbox otherwise starts a second, which Premiere won't import and Resolve shows opaque); a sound's clip ends where its file does; the stack's duplicate code folded (tools/dupes.max 59 -> 58 near). Checked on 11-17: Premiere Pro 26.2.2 and DaVinci Resolve 21.2.0 on the Telluride test (iPhone takes, a keyframed split close) and the positioning test (horizontal and phone takes, 1080 and 4K compositions); CapCut 9.5 on Telluride; LA3a composites the fixtures and the positioning test back as the render at 33-49 dB. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 3147884 commit df1a3d7

19 files changed

Lines changed: 1046 additions & 198 deletions

File tree

‎apps/capcut/src/from_layers.rs‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,9 @@ pub fn track_name(layer: Layer, name: &str) -> String {
2020
}
2121
}
2222

23-
/// The timeline. `dir` is where layers.json is: its movies' paths are relative to it.
24-
pub fn timeline(l: &Layers, dir: &Path) -> Timeline {
23+
/// The timeline, and what it can't carry. `dir` is where layers.json is: its movies' paths are relative to it.
24+
pub fn timeline(l: &Layers, dir: &Path) -> (Timeline, Vec<String>) {
25+
let mut notes = Vec::new();
2526
let media: Vec<Media> = l
2627
.files
2728
.iter()
@@ -47,10 +48,19 @@ pub fn timeline(l: &Layers, dir: &Path) -> Timeline {
4748
Layer::Footage => {
4849
let m = &media[c.file];
4950
// (a picture stretched to its box is noted by the layers export already, for every editor)
50-
let (place, _) = plan::place(&c.r#box.unwrap_or([0.0, 0.0, l.width as f64, l.height as f64]), c.fit.as_deref().unwrap_or("fill"), (m.width as f64).max(1.0), (m.height as f64).max(1.0), l.width as f64, l.height as f64, c.opacity);
51+
let (media, canvas) = (((m.width as f64).max(1.0), (m.height as f64).max(1.0)), (l.width as f64, l.height as f64));
52+
let bx = c.r#box.unwrap_or([0.0, 0.0, canvas.0, canvas.1]);
53+
let fit = c.fit.as_deref().unwrap_or("fill");
54+
let (picture, _) = raea_app_layers::plan::picture(&bx, &c.clip, fit, &c.position, media, canvas);
55+
let place = plan::place(&picture, media, canvas, c.opacity);
56+
let (keys, cut_changes) = plan::keys(c, fit, picture.kept, media, canvas);
57+
if cut_changes {
58+
let name = Path::new(&l.files[c.file].path).file_name().map_or_else(String::new, |n| n.to_string_lossy().into_owned());
59+
notes.push(format!("{name} (frames {}-{}): what of it is cut changes over its run, and CapCut's crop holds: cut as its first frame shows it throughout, its moves and fades keyed", c.start, c.end() - 1));
60+
}
5161
// its own sound at its gain, or silent: the composition doesn't play its sound there
5262
let own = c.link.as_deref().and_then(|id| all().find(|a| a.id == id));
53-
Clip { volume: own.map_or(0.0, |a| 10f64.powf(a.gain_db / 20.0)), place, ..base }
63+
Clip { volume: own.map_or(0.0, |a| 10f64.powf(a.gain_db / 20.0)), place, keys, ..base }
5464
}
5565
Layer::Graphics | Layer::Captions | Layer::Under | Layer::Backing => Clip { volume: 0.0, place: Place::centred(l.width, l.height), ..base },
5666
}
@@ -68,5 +78,5 @@ pub fn timeline(l: &Layers, dir: &Path) -> Timeline {
6878
main.name.clear();
6979
}
7080
video.extend(audio);
71-
Timeline { width: l.width, height: l.height, fps: l.fps, media, tracks: video }
81+
(Timeline { width: l.width, height: l.height, fps: l.fps, media, tracks: video }, notes)
7282
}

‎apps/capcut/src/lib.rs‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ mod tests;
1212

1313
use std::path::{Path, PathBuf};
1414

15+
use raea_app_layers::many;
1516
use raea_io_capcut::Kind;
1617
use raea_runtime_bundle::flag;
1718

@@ -78,13 +79,17 @@ fn export(args: &[String]) -> R<String> {
7879
let reference = work.join(&layers.reference);
7980

8081
// 2. its tracks as CapCut's, and the draft
81-
let tl = from_layers::timeline(&layers, &work);
82+
let (tl, carried) = from_layers::timeline(&layers, &work);
8283
let mut notes = layers.notes.clone();
84+
notes.extend(carried);
8385
let markers = raea_io_capcut::seed(&root);
8486
let written = raea_io_capcut::write(&tl, &root, &name, &markers, media::cover(&reference).as_deref())?;
8587
notes.extend(written.warnings.iter().cloned());
8688
// the reference render in the draft's folder, on no track, to set beside what CapCut plays (LA4b)
87-
std::fs::copy(&reference, written.dir.join("reference.mp4")).map_err(|e| format!("{}: {e}", written.dir.join("reference.mp4").display()))?;
89+
let beside = written.dir.join("reference.mp4");
90+
if let Err(e) = std::fs::copy(&reference, &beside) {
91+
return Err(format!("{}: {e}", beside.display()));
92+
}
8893
let listed = if !in_store || args.iter().any(|a| a == "--no-register") {
8994
"not entered in CapCut's project list".to_string()
9095
} else {
@@ -119,11 +124,6 @@ fn export(args: &[String]) -> R<String> {
119124
Ok(said)
120125
}
121126

122-
/// "1 clip", "4 clips"
123-
fn many(k: usize, what: &str) -> String {
124-
if k == 1 { format!("1 {what}") } else { format!("{k} {what}s") }
125-
}
126-
127127
/// Where CapCut's project list is copied before a draft is entered in it: the person's own folder, ~/.raea/capcut/backups
128128
/// ($RAEA_CAPCUT_BACKUPS; %LOCALAPPDATA%\raea\capcut\backups on Windows, which sets no HOME). Never raea's home, which
129129
/// is its install or, from a checkout, the source tree (11-11: a list's backup landed in the repository).

‎apps/capcut/src/plan.rs‎

Lines changed: 41 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,34 +1,46 @@
11
//! Where a picture sits in CapCut's terms (docs/capcut.feature CC2a). The manifest is read by raea-app-layers, the same
2-
//! for every editor; this is CapCut's own.
2+
//! for every editor, and so is where a picture is drawn and what of it is kept (raea_app_layers::plan::picture); this
3+
//! is CapCut's own.
34
4-
use raea_app_layers::plan::full;
5+
use raea_app_layers::model::Clip;
6+
use raea_app_layers::plan::{self, Picture};
57
use raea_io_capcut::Place;
68

7-
/// Where a picture of `mw` x `mh` sits to show as it does in its box with its object-fit, in CapCut's terms (scale 1:
8-
/// fitted inside the canvas; a crop is fitted as the part it keeps). A warning when CapCut can't show it so.
9-
pub fn place(bx: &[f64; 4], fit: &str, mw: f64, mh: f64, w: f64, h: f64, opacity: f64) -> (Place, Option<String>) {
10-
let (bw, bh) = (bx[2].max(1.0), bx[3].max(1.0));
11-
let (x, y) = (bx[0] + bw / 2.0, bx[1] + bh / 2.0);
12-
let fit0 = (w / mw).min(h / mh);
13-
let mut warn = None;
14-
let contain = (bw / mw).min(bh / mh);
15-
let (scale, crop) = match fit {
16-
"cover" if full(bx, w, h) => ((bw / mw).max(bh / mh) / fit0, [0.0, 0.0, 1.0, 1.0]),
17-
"cover" => {
18-
let s = (bw / mw).max(bh / mh);
19-
let (cw, ch) = (bw / (mw * s), bh / (mh * s));
20-
let fitc = (w / (mw * cw)).min(h / (mh * ch));
21-
(bw / (mw * cw * fitc), [(1.0 - cw) / 2.0, (1.0 - ch) / 2.0, (1.0 + cw) / 2.0, (1.0 + ch) / 2.0])
22-
}
23-
"none" => (1.0 / fit0, [0.0, 0.0, 1.0, 1.0]),
24-
"scale-down" => (contain.min(1.0) / fit0, [0.0, 0.0, 1.0, 1.0]),
25-
"contain" => (contain / fit0, [0.0, 0.0, 1.0, 1.0]),
26-
_ => {
27-
if ((bw / bh) - (mw / mh)).abs() > 0.01 * (mw / mh) {
28-
warn = Some(format!("stretched to its {bw:.0}x{bh:.0} box in raea (object-fit: fill), fitted inside it in CapCut"));
29-
}
30-
(contain / fit0, [0.0, 0.0, 1.0, 1.0])
31-
}
32-
};
33-
(Place { x, y, scale, opacity, crop }, warn)
9+
/// Where a picture of `mw` x `mh` drawn as `p` sits in CapCut's terms on a `w` x `h` canvas: the part of it kept as
10+
/// its crop, which CapCut fits inside the canvas (scale 1), then scaled to the size raea drew it, centred where raea
11+
/// drew that part.
12+
pub fn place(p: &Picture, (mw, mh): (f64, f64), (w, h): (f64, f64), opacity: f64) -> Place {
13+
let [dx, dy, dw, dh] = p.drawn;
14+
let [l, t, r, b] = p.kept;
15+
let (cw, ch) = ((r - l).max(1e-6), (b - t).max(1e-6));
16+
let fit = (w / (mw * cw)).min(h / (mh * ch));
17+
let (x, y) = (dx + dw * (l + r) / 2.0, dy + dh * (t + b) / 2.0);
18+
Place { x, y, scale: dw / (mw * fit), opacity, crop: p.kept }
19+
}
20+
21+
/// A footage clip's keys in CapCut's terms (frames from its first, each its place there; none when it holds), cut
22+
/// throughout as `first` keeps it (its first frame's): CapCut's crop holds, so its moves, scaling and fades are keyed
23+
/// around that cut. Thinned to what a straight line between can't give, to 0.15 px and half a percent of opacity; and
24+
/// whether what is cut changes over its run, which CapCut can't show.
25+
pub fn keys(c: &Clip, fit: &str, first: [f64; 4], (mw, mh): (f64, f64), canvas: (f64, f64)) -> (Vec<(i64, Place)>, bool) {
26+
let mut changes = false;
27+
let all: Vec<(i64, [f64; 4])> = c
28+
.keys
29+
.iter()
30+
.map(|k| {
31+
let (p, _) = plan::picture(&k.r#box, &k.clip, fit, &k.position, (mw, mh), canvas);
32+
changes |= plan::apart(&p.kept, &first) > 1e-3;
33+
let at = place(&Picture { drawn: p.drawn, kept: first }, (mw, mh), canvas, k.opacity);
34+
(k.frame, [at.x, at.y, at.scale, at.opacity])
35+
})
36+
.collect();
37+
if all.is_empty() {
38+
return (Vec::new(), false);
39+
}
40+
// a scale step a 0.15 px change in the width of what's kept, as CapCut fits it inside the canvas
41+
let (cw, ch) = ((first[2] - first[0]).max(1e-6), (first[3] - first[1]).max(1e-6));
42+
let fitted = mw * cw * (canvas.0 / (mw * cw)).min(canvas.1 / (mh * ch));
43+
let kept = plan::thin(&all, &[0.15, 0.15, 0.15 / fitted, 0.005]);
44+
let keys = kept.into_iter().map(|(f, [x, y, scale, opacity])| (f, Place { x, y, scale, opacity, crop: first })).collect();
45+
(keys, changes)
3446
}

‎apps/capcut/src/tests.rs‎

Lines changed: 44 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -10,24 +10,34 @@ use crate::plan::*;
1010

1111
#[test]
1212
fn pictures_sit_as_their_fit_shows_them() {
13-
// a landscape take covering a portrait canvas: scaled past the fitted width until it fills the height
14-
let (p, w) = place(&[0.0, 0.0, 1080.0, 1920.0], "cover", 1920.0, 1080.0, 1080.0, 1920.0, 1.0);
13+
let at = |bx: [f64; 4], fit: &str, media: (f64, f64)| {
14+
let (p, note) = raea_app_layers::plan::picture(&bx, &None, fit, &None, media, (1080.0, 1920.0));
15+
(place(&p, media, (1080.0, 1920.0), 1.0), note)
16+
};
17+
// a landscape take covering a portrait canvas: scaled past the fitted width until it fills the height, uncut (the
18+
// canvas cuts it)
19+
let (p, w) = at([0.0, 0.0, 1080.0, 1920.0], "cover", (1920.0, 1080.0));
1520
assert!(w.is_none() && (p.scale - (1920.0 / 1080.0) / 0.5625).abs() < 1e-9 && p.crop == [0.0, 0.0, 1.0, 1.0]);
1621
assert_eq!((p.x, p.y), (540.0, 960.0));
1722
// a square in a corner, contained: a third of the canvas's width
18-
let (p, _) = place(&[660.0, 200.0, 360.0, 360.0], "contain", 640.0, 640.0, 1080.0, 1920.0, 1.0);
23+
let (p, _) = at([660.0, 200.0, 360.0, 360.0], "contain", (640.0, 640.0));
1924
assert!((p.scale - 1.0 / 3.0).abs() < 1e-9 && (p.x, p.y) == (840.0, 380.0));
20-
// a landscape take covering a square box: cropped to its middle square, which then fits as a square
21-
let (p, _) = place(&[0.0, 0.0, 540.0, 540.0], "cover", 1920.0, 1080.0, 1080.0, 1920.0, 1.0);
25+
// a landscape take covering a square box inside the canvas: cropped to its middle square, which then fits as a square
26+
let (p, _) = at([100.0, 100.0, 540.0, 540.0], "cover", (1920.0, 1080.0));
2227
assert!((p.crop[0] - 0.21875).abs() < 1e-9 && (p.crop[2] - 0.78125).abs() < 1e-9 && p.crop[1] == 0.0);
23-
assert!((p.scale - 0.5).abs() < 1e-9);
28+
assert!((p.scale - 0.5).abs() < 1e-9 && ((p.x - 370.0).abs(), (p.y - 370.0).abs()) < (1e-9, 1e-9));
29+
// the same box in the canvas's corner: cut only on its right, the canvas cutting its left; the part kept sits where
30+
// raea drew it, its left past the canvas
31+
let (p, _) = at([0.0, 0.0, 540.0, 540.0], "cover", (1920.0, 1080.0));
32+
assert!(p.crop[0] == 0.0 && (p.crop[2] - 0.78125).abs() < 1e-9);
33+
assert!((p.x - (-210.0 + 750.0 / 2.0)).abs() < 1e-9 && (p.y - 270.0).abs() < 1e-9);
2434
// stretched in raea: said
25-
assert!(place(&[0.0, 0.0, 500.0, 100.0], "fill", 640.0, 640.0, 1080.0, 1920.0, 1.0).1.is_some());
35+
assert!(at([0.0, 0.0, 500.0, 100.0], "fill", (640.0, 640.0)).1.is_some());
2636
}
2737

2838
#[test]
2939
fn layers_json_goes_bottom_first_with_footage_carrying_its_own_sound() {
30-
let file = |path: &str, kind, w, h| File { path: path.into(), kind, seconds: 3.0, width: w, height: h, has_audio: kind == FileKind::Video, channels: 2, sample_rate: 48_000 };
40+
let file = |path: &str, kind, w, h| File { path: path.into(), kind, seconds: 3.0, width: w, height: h, has_audio: kind == FileKind::Video, channels: 2, sample_rate: 48_000, rotation: 0 };
3141
let clip = |id: &str, f, start, frames| LClip::new(id.into(), f, start, frames, 0.0);
3242
let t = |layer, name: &str, clips| LTrack { layer, name: name.into(), clips };
3343
let v1 = LClip { r#box: Some([0.0, 0.0, 1080.0, 1920.0]), fit: Some("cover".into()), link: Some("a1".into()), ..clip("v1", 0, 0, 90) };
@@ -50,7 +60,7 @@ fn layers_json_goes_bottom_first_with_footage_carrying_its_own_sound() {
5060
],
5161
notes: vec![],
5262
};
53-
let tl = from_layers::timeline(&l, Path::new("/out/edit"));
63+
let (tl, _) = from_layers::timeline(&l, Path::new("/out/edit"));
5464
// bottom first: the footage is CapCut's main track (no name), then the overlay, then the captions; sound after
5565
assert_eq!(tl.tracks.iter().map(|t| t.name.as_str()).collect::<Vec<_>>(), ["", "raea overlay", "raea captions", "sound"]);
5666
// a layer's movie is found beside layers.json; the footage where it is
@@ -68,5 +78,29 @@ fn layers_json_goes_bottom_first_with_footage_carrying_its_own_sound() {
6878
let mut quiet = l.clone();
6979
quiet.tracks[2].clips[0].link = None;
7080
quiet.tracks[3].clips.remove(0);
71-
assert_eq!(from_layers::timeline(&quiet, Path::new("/out/edit")).tracks[0].clips[0].volume, 0.0);
81+
assert_eq!(from_layers::timeline(&quiet, Path::new("/out/edit")).0.tracks[0].clips[0].volume, 0.0);
82+
}
83+
84+
/// CC2f: a footage clip's keys in CapCut's terms, around the cut its first frame shows: a steady move is its two ends,
85+
/// a move inside a card keeps the card's cut, and a cut that changes (the split close) is said
86+
#[test]
87+
fn keyed_footage_is_keyed_around_its_first_cut() {
88+
use raea_app_layers::model::Key;
89+
let canvas = (1080.0, 1920.0);
90+
let key = |frame: i64, bx: [f64; 4], clip: Option<[f64; 4]>| Key { frame, r#box: bx, clip, position: None, opacity: 1.0 };
91+
let first = |c: &LClip| raea_app_layers::plan::picture(&c.r#box.unwrap(), &c.clip, "cover", &None, (1080.0, 1920.0), canvas).0.kept;
92+
// a picture in picture sliding right 10 px a frame: its two ends
93+
let slide = LClip { r#box: Some([100.0, 500.0, 400.0, 400.0]), keys: (0..24).map(|f| key(f, [100.0 + 10.0 * f as f64, 500.0, 400.0, 400.0], None)).collect(), ..LClip::new("v".into(), 0, 0, 24, 0.0) };
94+
let (got, changes) = keys(&slide, "cover", first(&slide), (1080.0, 1920.0), canvas);
95+
assert!(!changes && got.len() == 2 && got[1].0 == 23 && (got[1].1.x - got[0].1.x - 230.0).abs() < 1e-6);
96+
// inside a card that moves with it, cutting its sides: the same cut throughout, keyed as it moves
97+
let card = |f: i64| key(f, [80.0 + 4.0 * f as f64, 300.0, 600.0, 1066.7], Some([140.0 + 4.0 * f as f64, 300.0, 480.0, 1066.7]));
98+
let carded = LClip { r#box: Some(card(0).r#box), clip: card(0).clip, keys: (0..24).map(card).collect(), ..LClip::new("v".into(), 0, 0, 24, 0.0) };
99+
let (got, changes) = keys(&carded, "cover", first(&carded), (1080.0, 1920.0), canvas);
100+
assert!(!changes && got.len() == 2 && got.iter().all(|(_, p)| p.crop[0] > 0.09 && p.crop[2] < 0.91));
101+
// the split close: its box shrinking to the seam changes what's cut, which CapCut's crop can't follow
102+
let split = |f: i64, h: f64| key(f, [0.0, -260.0 * (1920.0 - h) / 1060.0, 1080.0, 1920.0], (h < 1920.0).then_some([0.0, 0.0, 1080.0, h]));
103+
let closing = LClip { r#box: Some(split(0, 1920.0).r#box), keys: vec![split(0, 1920.0), split(12, 860.0)], ..LClip::new("v".into(), 0, 0, 24, 0.0) };
104+
let (got, changes) = keys(&closing, "cover", first(&closing), (1080.0, 1920.0), canvas);
105+
assert!(changes && got.len() == 2 && got[1].1.crop == [0.0, 0.0, 1.0, 1.0]);
72106
}

0 commit comments

Comments
 (0)