Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 44 additions & 5 deletions site/scripts/clip-timeline.mjs
Original file line number Diff line number Diff line change
@@ -1,3 +1,32 @@
// Full-frame mean difference: tolerate cursor/caret flicker and small streamed
// text updates, but retain larger visual changes. Require 1.2 s of near-stasis.
export const freezeDetectFilter = "freezedetect=n=0.003:d=1.2";

/** Parse ffmpeg stderr; an open freeze at EOF extends to the probed duration. */
export function parseFreezeDetect(stderr, duration) {
if (!Number.isFinite(duration) || duration <= 0) throw new Error("Video duration must be positive");
const intervals = [];
let start;
for (const match of stderr.matchAll(/lavfi\.freezedetect\.freeze_(start|end):\s*([^\r\n]*)/g)) {
const value = match[2].trim();
const at = Number(value);
if (!value || !Number.isFinite(at) || at < 0 || at > duration + 0.25) {
throw new Error("Invalid freezedetect timestamp");
}
if (match[1] === "start") {
if (start !== undefined || at < (intervals.at(-1)?.end ?? 0)) throw new Error("Unordered freezedetect start");
start = Math.min(at, duration);
} else {
if (start === undefined || at <= start) throw new Error("Unpaired or unordered freezedetect end");
const end = Math.min(at, duration);
if (end > start) intervals.push({ start, end });
start = undefined;
}
}
if (start !== undefined && start < duration) intervals.push({ start, end: duration });
return intervals;
}

function mergeIntervals(intervals) {
const merged = [];
for (const { start, end } of intervals.sort((a, b) => a.start - b.start)) {
Expand All @@ -8,21 +37,31 @@ function mergeIntervals(intervals) {
return merged;
}

/** Merge concurrent waits, preserving holds at real speed even inside a wait. Frames before from are cut. */
export function clipSegments(waits, duration, from = 0) {
/** Merge waits and freezes, then subtract holds/typing before compressing. */
export function clipSegments(waits, duration, from = 0, freezes = []) {
if (!Number.isFinite(duration) || duration <= 0) throw new Error("Video duration must be positive");
if (!Number.isFinite(from) || from < 0 || from >= duration) throw new Error("Clip start must be within the raw video");
if (!Array.isArray(waits)) throw new Error("Timeline waits must be an array");
const intervals = waits.map(({ kind, start, end }) => {
if (!Array.isArray(freezes)) throw new Error("Frozen intervals must be an array");
const intervals = [...waits, ...freezes.map(({ start, end }) => ({ kind: "freeze", start, end }))].map(({ kind, start, end }) => {
if (!Number.isFinite(start) || !Number.isFinite(end) || start < 0 || end <= start) {
throw new Error("Wait intervals must have finite seconds with 0 <= start < end");
}
if (end > duration + 0.25) throw new Error("Wait interval extends past the raw video");
return { kind, start: Math.max(from, Math.min(start, duration)), end: Math.min(end, duration) };
}).filter(({ start, end }) => end > start);
const holds = mergeIntervals(intervals.filter(interval => interval.kind === "hold"));
// Version 1 recordings start before typing and first mark a wait after Send.
// Protect that whole opening even if caret flicker/typing looks frozen. With
// no wait boundary, conservatively keep it all; explicit typing overrides this.
if (freezes.length && !waits.some(interval => interval.kind === "typing")) {
const end = waits.filter(interval => interval.kind !== "hold")
.reduce((first, interval) => Math.min(first, interval.start), duration);
if (end > from) intervals.push({ kind: "typing", start: from, end: Math.min(end, duration) });
}
const protectedInterval = interval => interval.kind === "hold" || interval.kind === "typing";
const holds = mergeIntervals(intervals.filter(protectedInterval));
const waitsToCompress = [];
for (const { start, end } of mergeIntervals(intervals.filter(interval => interval.kind !== "hold"))) {
for (const { start, end } of mergeIntervals(intervals.filter(interval => !protectedInterval(interval)))) {
let cursor = start;
for (const hold of holds) {
if (hold.end <= cursor) continue;
Expand Down
101 changes: 101 additions & 0 deletions site/scripts/clip-timeline.test.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
import { clipSegments, parseFreezeDetect } from "./clip-timeline.mjs";

const fixture = name => readFileSync(new URL(`./fixtures/freezedetect-${name}.stderr`, import.meta.url), "utf8");

test("freezedetect stderr parses paired intervals, including a freeze starting at zero", () => {
const stderr = fixture("complete");
for (const input of [stderr, stderr.replaceAll("\n", "\r\n")]) {
assert.deepEqual(parseFreezeDetect(input, 12), [{ start: 0, end: 2.4 }, { start: 4.4, end: 9.8 }]);
}
});

test("freezedetect closes an EOF freeze at the probed duration and accepts exponent timestamps", () => {
assert.deepEqual(parseFreezeDetect(fixture("eof"), 10), [{ start: 0.12, end: 2.12 }, { start: 3.24, end: 10 }]);
assert.deepEqual(parseFreezeDetect("[freezedetect] lavfi.freezedetect.freeze_start: 0", 10), [{ start: 0, end: 10 }]);
assert.deepEqual(parseFreezeDetect("[freezedetect] lavfi.freezedetect.freeze_start: 10", 10), []);
});

test("non-freeze stderr and freeze_duration diagnostics do not create intervals", () => {
assert.deepEqual(parseFreezeDetect("", 10), []);
assert.deepEqual(parseFreezeDetect("frame=250 time=00:00:10.00\n[freezedetect] lavfi.freezedetect.freeze_duration: 2\n", 10), []);
});

test("invalid or unordered freezedetect timestamps fail rather than silently over-compress", () => {
const log = events => events.map(([kind, at]) => `[freezedetect] lavfi.freezedetect.freeze_${kind}: ${at}\n`).join("");
for (const events of [
[["start", "NaN"]], [["start", "Infinity"]], [["start", ""]], [["start", -1]], [["start", 11]],
[["end", 4]], [["start", 2], ["start", 3]], [["start", 2], ["end", 1]],
[["start", 2], ["end", 2]], [["start", 2], ["end", 4], ["start", 3]],
]) assert.throws(() => parseFreezeDetect(log(events), 10));
for (const duration of [0, -1, NaN, Infinity]) assert.throws(() => parseFreezeDetect("", duration));
// Permit the same small container/frame rounding tolerance as timeline waits.
assert.deepEqual(parseFreezeDetect(log([["start", 2], ["end", 10.1]]), 10), [{ start: 2, end: 10 }]);
});

test("freezes merge with waits before overlapping holds are subtracted, without mutating inputs", () => {
const waits = Object.freeze([
Object.freeze({ kind: "hold", start: 20, end: 28 }),
Object.freeze({ kind: "model_thinking", start: 4, end: 8 }),
Object.freeze({ kind: "hold", start: 12, end: 16 }),
Object.freeze({ kind: "hold", start: 10, end: 14 }),
]);
const freezes = Object.freeze([Object.freeze({ start: 7, end: 30 }), Object.freeze({ start: 0, end: 5 })]);
assert.deepEqual(clipSegments(waits, 32, 2, freezes), [
{ start: 2, end: 4, seconds: 2 }, // Typing survives a detected freeze.
{ start: 4, end: 10, seconds: 0.6 }, // One merged wait/freeze, not two budgets.
{ start: 10, end: 16, seconds: 6 },
{ start: 16, end: 20, seconds: 0.6 },
{ start: 20, end: 28, seconds: 8 },
{ start: 28, end: 30, seconds: 0.6 },
{ start: 30, end: 32, seconds: 2 },
]);
});

test("a freeze spanning typing, waits, scroll and final dwell cannot shorten protected time", () => {
assert.deepEqual(clipSegments([
{ kind: "model_thinking", start: 5, end: 10 },
{ kind: "hold", start: 18, end: 30 },
], 30, 2, [{ start: 0, end: 30 }]), [
{ start: 2, end: 5, seconds: 3 },
{ start: 5, end: 18, seconds: 0.6 },
{ start: 18, end: 30, seconds: 12 },
]);
// Without a post-typing wait boundary, keep the opening conservatively.
assert.deepEqual(clipSegments([{ kind: "hold", start: 10, end: 20 }], 20, 0, [{ start: 0, end: 20 }]), [
{ start: 0, end: 20, seconds: 20 },
]);
});

test("explicit typing takes priority over freezes and waits, and short remainders keep real speed", () => {
assert.deepEqual(clipSegments([
{ kind: "typing", start: 2, end: 5 },
{ kind: "tool_call", start: 1, end: 10 },
{ kind: "hold", start: 10.2, end: 12 },
], 12, 0, [{ start: 8, end: 10.2 }]), [
{ start: 0, end: 1, seconds: 1 },
{ start: 1, end: 2, seconds: 0.6 },
{ start: 2, end: 5, seconds: 3 },
{ start: 5, end: 10.2, seconds: 0.6 },
{ start: 10.2, end: 12, seconds: 12 - 10.2 },
]);
const segments = clipSegments([
{ kind: "typing", start: 0, end: 3 }, { kind: "hold", start: 3.4, end: 10 },
], 10, 0, [{ start: 0, end: 10 }]);
assert.equal(segments[1].seconds, 3.4 - 3);
});

test("frozen intervals validate and clip at raw video and retained start boundaries", () => {
for (const freezes of [null, [{ start: -1, end: 3 }], [{ start: 2, end: 1 }], [{ start: 0, end: Infinity }], [{ start: 5, end: 11 }]]) {
assert.throws(() => clipSegments([], 10, 0, freezes));
}
assert.deepEqual(clipSegments([{ kind: "model_thinking", start: 1, end: 2 }], 10, 4, [
{ start: 0, end: 6 }, { start: 8, end: 10.1 },
]), [
{ start: 4, end: 6, seconds: 0.6 },
{ start: 6, end: 8, seconds: 2 },
{ start: 8, end: 10, seconds: 0.6 },
]);
});
19 changes: 13 additions & 6 deletions site/scripts/encode-clips.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,17 @@ import { mkdir } from "node:fs/promises";
import { tmpdir } from "node:os";
import { basename, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { clipFilter, clipSegments } from "./clip-timeline.mjs";
import { clipFilter, clipSegments, freezeDetectFilter, parseFreezeDetect } from "./clip-timeline.mjs";

const input = resolve(process.argv[2] ?? fileURLToPath(new URL("../../ui/host/test-results/landing-recordings/", import.meta.url)));
const output = resolve(process.argv[3] ?? fileURLToPath(new URL("../src/assets/landing/", import.meta.url)));
const temporary = mkdtempSync(join(tmpdir(), "fanout-clips-"));
function run(command, args, capture = false) {
const result = spawnSync(command, args, { encoding: "utf8", stdio: capture ? ["ignore", "pipe", "inherit"] : "inherit" });
const result = spawnSync(command, args, { encoding: "utf8", maxBuffer: 16 * 1024 * 1024,
stdio: capture === "stderr" ? ["ignore", "ignore", "pipe"] : capture ? ["ignore", "pipe", "inherit"] : "inherit" });
if (result.error) throw result.error;
if (result.status !== 0) throw new Error(`${command} exited with status ${result.status}`);
return result.stdout;
if (result.status !== 0) throw new Error(`${command} exited with status ${result.status}${capture === "stderr" ? `\n${result.stderr}` : ""}`);
return capture === "stderr" ? result.stderr : result.stdout;
}

try {
Expand All @@ -30,7 +31,13 @@ try {
const raw = join(input, timeline.raw_file);
const probe = JSON.parse(run("ffprobe", ["-v", "error", "-show_entries", "format=duration:stream=width,height", "-of", "json", raw], true));
if (!probe.streams.some(stream => stream.width === 1280 && stream.height === 800)) throw new Error(`${name} must be recorded at 1280 × 800`);
const segments = clipSegments(timeline.waits, Number(probe.format.duration), timeline.start);
const duration = Number(probe.format.duration);
// Decode the raw video, before trimming, so detection and timeline use the
// same clock. Info-level stderr carries freeze_start/end (including EOF).
const detection = run("ffmpeg", ["-hide_banner", "-nostdin", "-nostats", "-loglevel", "info", "-i", raw,
"-map", "0:v:0", "-an", "-vf", freezeDetectFilter, "-f", "null", "-"], "stderr");
const freezes = parseFreezeDetect(detection, duration);
const segments = clipSegments(timeline.waits, duration, timeline.start, freezes);
const seconds = segments.reduce((sum, segment) => sum + segment.seconds, 0);
// Reserve 5% of a decimal 4 MB target for muxing; two passes share this budget.
const bitrate = String(Math.floor(4_000_000 * 8 * 0.95 / seconds));
Expand All @@ -48,7 +55,7 @@ try {
// Reverse only the final second so the PNG is the last encoded frame.
run("ffmpeg", ["-hide_banner", "-loglevel", "warning", "-y", "-sseof", "-1", "-i", join(output, `${name}.mp4`),
"-vf", "reverse", "-frames:v", "1", "-update", "1", join(output, `${name}.png`)]);
console.log(`${name}: ${seconds.toFixed(1)} s, 1280 × 800, 30 fps, approximately 4 MB per format`);
console.log(`${name}: ${seconds.toFixed(1)} s, ${freezes.length} frozen intervals, 1280 × 800, 30 fps, approximately 4 MB per format`);
}
} finally {
rmSync(temporary, { recursive: true, force: true });
Expand Down
9 changes: 9 additions & 0 deletions site/scripts/fixtures/freezedetect-complete.stderr
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
Input #0, matroska,webm, from 'sample.webm':
Duration: 00:00:12.00, start: 0.000000
[Parsed_freezedetect_0 @ 0x1234] lavfi.freezedetect.freeze_start: 0
[Parsed_freezedetect_0 @ 0x1234] lavfi.freezedetect.freeze_duration: 2.4
[Parsed_freezedetect_0 @ 0x1234] lavfi.freezedetect.freeze_end: 2.4
[Parsed_freezedetect_0 @ 0x1234] lavfi.freezedetect.freeze_start: 4.4
[Parsed_freezedetect_0 @ 0x1234] lavfi.freezedetect.freeze_duration: 5.4
[Parsed_freezedetect_0 @ 0x1234] lavfi.freezedetect.freeze_end: 9.8
frame= 300 fps=0.0 q=-0.0 Lsize=N/A time=00:00:12.00 bitrate=N/A
6 changes: 6 additions & 0 deletions site/scripts/fixtures/freezedetect-eof.stderr
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
[Parsed_freezedetect_0 @ 0x5678] lavfi.freezedetect.freeze_start: 1.2e-1
[Parsed_freezedetect_0 @ 0x5678] lavfi.freezedetect.freeze_duration: 2
[Parsed_freezedetect_0 @ 0x5678] lavfi.freezedetect.freeze_end: 2.12
[Parsed_freezedetect_0 @ 0x5678] lavfi.freezedetect.freeze_start: 3.24
[out#0/null @ 0x9876] video:100KiB audio:0KiB
frame= 250 fps=0.0 q=-0.0 Lsize=N/A time=00:00:10.00 bitrate=N/A
119 changes: 119 additions & 0 deletions site/scripts/landing-recording-scroll.test.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { runInNewContext } from "node:vm";
import { revealRecording, waitForConversation } from "../../ui/host/e2e/landing-recording-support.ts";

// Execute the actual page.evaluate callbacks with a virtual layout and clock.
// No browser is launched; iframe growth is represented by moving the answer.
function recordingPage({ view = "chat", answerReady = true } = {}) {
let now = 0, nextId = 0, scrollTop = 0, answerBottom = 1_300;
const pending = new Map(), observers = new Set(), positions = [];
const schedule = (callback, milliseconds) => {
const id = ++nextId;
pending.set(id, { at: now + milliseconds, callback });
return id;
};
class Observer {
constructor(callback) { this.callback = callback; this.targets = new Set(); observers.add(this); }
observe(target) { this.targets.add(target); }
disconnect() { observers.delete(this); }
}
const answer = { getBoundingClientRect: () => ({ bottom: 100 + answerBottom - scrollTop }) };
const markdown = { textContent: "The final answer." };
const content = {};
const scroller = {
clientHeight: 500, clientTop: 0, firstElementChild: content,
get scrollHeight() { return view === "chat" ? answerBottom + 24 : 1_400; },
get scrollTop() { return scrollTop; },
set scrollTop(value) {
scrollTop = Math.max(0, Math.min(value, this.scrollHeight - this.clientHeight));
positions.push({ at: now, top: scrollTop });
},
getBoundingClientRect: () => ({ top: 100 }),
querySelector: selector => selector === "[data-answer-position]" ? answer : answerReady ? markdown : null,
querySelectorAll: () => [answer],
dispatchEvent() {},
};
const environment = {
document: { querySelector: () => scroller, scrollingElement: scroller, fonts: { ready: Promise.resolve() } },
performance: { now: () => now },
requestAnimationFrame: callback => schedule(() => callback(now), 16),
cancelAnimationFrame: id => pending.delete(id),
setTimeout: schedule, clearTimeout: id => pending.delete(id),
ResizeObserver: Observer, MutationObserver: Observer, WheelEvent: class {},
};
const page = {
async evaluate(callback, argument) {
let done = false, error;
const result = runInNewContext(`(${callback.toString()})(argument)`, { ...environment, argument });
result.then(() => { done = true; }, reason => { done = true; error = reason; });
for (let steps = 0; !done && steps < 10_000; steps++) {
// Drain async continuations before advancing the simulated animation clock.
await new Promise(resolve => setImmediate(resolve));
if (done) break;
const next = [...pending].sort((a, b) => a[1].at - b[1].at)[0];
assert.ok(next, "evaluate must resolve or schedule more work");
pending.delete(next[0]);
now = next[1].at;
next[1].callback();
}
assert.ok(done, "evaluate must finish within the virtual clock budget");
if (error) throw error;
return result;
},
};
return {
page, positions, observers, pending,
get now() { return now; },
get expectedEnd() { return answerBottom - scroller.clientHeight; },
get top() { return scrollTop; },
resizeAt(at, bottom) {
schedule(() => {
answerBottom = bottom;
for (const observer of observers) observer.callback();
}, at);
},
renderAnswerAt(at) { schedule(() => { answerReady = true; }, at); },
};
}

test("chat retargets late evidence growth during the scroll and holds the answer end for 4 seconds", async () => {
const recording = recordingPage();
recording.resizeAt(3_000, 2_400);
recording.resizeAt(9_500, 3_100); // Growth during the final dwell must also stay aligned.
await revealRecording(recording.page, "chat");
assert.equal(recording.top, recording.expectedEnd);
assert.ok(recording.positions.some(position => position.at > 8_100 && position.at < 9_500 && position.top === 1_900));
assert.ok(recording.positions.filter(position => position.at >= 9_520).every(position => position.top === 2_600));
// Initial distance sets an 8-second scroll; growth must not restart its duration.
assert.ok(recording.now >= 12_000 && recording.now < 12_150);
});

test("dashboard keeps the down/bottom/up timings and dwells at the top for only 3 seconds", async () => {
const recording = recordingPage({ view: "dashboard" });
await revealRecording(recording.page, "dashboard");
assert.equal(recording.top, 0);
assert.equal(Math.max(...recording.positions.map(position => position.top)), 900);
assert.ok(recording.positions.some(position => position.top > 0 && position.top < 900 && position.at > 7_500));
assert.ok(recording.now >= 12_000 && recording.now < 12_150);
assert.equal(recording.now - recording.positions.at(-1).at, 3_000);
});

test("conversation readiness waits for rendered text and 1.5 seconds after the last resize", async () => {
const recording = recordingPage({ answerReady: false });
recording.renderAnswerAt(1_500);
recording.resizeAt(1_000, 2_000);
recording.resizeAt(2_000, 2_400);
await waitForConversation(recording.page);
assert.ok(recording.now >= 3_500 && recording.now < 3_550);
assert.equal(recording.observers.size, 0);
assert.equal(recording.pending.size, 0);
});

test("conversation readiness fails instead of recording an answer that never renders", async () => {
const recording = recordingPage({ answerReady: false });
await assert.rejects(waitForConversation(recording.page), /Conversation did not stop resizing/);
assert.equal(recording.now, 30_000);
assert.equal(recording.observers.size, 0);
assert.equal(recording.pending.size, 0);
});
Loading
Loading