diff --git a/site/scripts/clip-timeline.mjs b/site/scripts/clip-timeline.mjs index 16319eca6..02d54f174 100644 --- a/site/scripts/clip-timeline.mjs +++ b/site/scripts/clip-timeline.mjs @@ -1,3 +1,32 @@ +// Full-frame mean difference: tolerate cursor/caret flicker and small streamed +// text updates, but retain larger visual changes. Require 1.2 s of near-stasis. +export const freezeDetectFilter = "freezedetect=n=0.003:d=1.2"; + +/** Parse ffmpeg stderr; an open freeze at EOF extends to the probed duration. */ +export function parseFreezeDetect(stderr, duration) { + if (!Number.isFinite(duration) || duration <= 0) throw new Error("Video duration must be positive"); + const intervals = []; + let start; + for (const match of stderr.matchAll(/lavfi\.freezedetect\.freeze_(start|end):\s*([^\r\n]*)/g)) { + const value = match[2].trim(); + const at = Number(value); + if (!value || !Number.isFinite(at) || at < 0 || at > duration + 0.25) { + throw new Error("Invalid freezedetect timestamp"); + } + if (match[1] === "start") { + if (start !== undefined || at < (intervals.at(-1)?.end ?? 0)) throw new Error("Unordered freezedetect start"); + start = Math.min(at, duration); + } else { + if (start === undefined || at <= start) throw new Error("Unpaired or unordered freezedetect end"); + const end = Math.min(at, duration); + if (end > start) intervals.push({ start, end }); + start = undefined; + } + } + if (start !== undefined && start < duration) intervals.push({ start, end: duration }); + return intervals; +} + function mergeIntervals(intervals) { const merged = []; for (const { start, end } of intervals.sort((a, b) => a.start - b.start)) { @@ -8,21 +37,31 @@ function mergeIntervals(intervals) { return merged; } -/** Merge concurrent waits, preserving holds at real speed even inside a wait. Frames before from are cut. */ -export function clipSegments(waits, duration, from = 0) { +/** Merge waits and freezes, then subtract holds/typing before compressing. */ +export function clipSegments(waits, duration, from = 0, freezes = []) { if (!Number.isFinite(duration) || duration <= 0) throw new Error("Video duration must be positive"); if (!Number.isFinite(from) || from < 0 || from >= duration) throw new Error("Clip start must be within the raw video"); if (!Array.isArray(waits)) throw new Error("Timeline waits must be an array"); - const intervals = waits.map(({ kind, start, end }) => { + if (!Array.isArray(freezes)) throw new Error("Frozen intervals must be an array"); + const intervals = [...waits, ...freezes.map(({ start, end }) => ({ kind: "freeze", start, end }))].map(({ kind, start, end }) => { if (!Number.isFinite(start) || !Number.isFinite(end) || start < 0 || end <= start) { throw new Error("Wait intervals must have finite seconds with 0 <= start < end"); } if (end > duration + 0.25) throw new Error("Wait interval extends past the raw video"); return { kind, start: Math.max(from, Math.min(start, duration)), end: Math.min(end, duration) }; }).filter(({ start, end }) => end > start); - const holds = mergeIntervals(intervals.filter(interval => interval.kind === "hold")); + // Version 1 recordings start before typing and first mark a wait after Send. + // Protect that whole opening even if caret flicker/typing looks frozen. With + // no wait boundary, conservatively keep it all; explicit typing overrides this. + if (freezes.length && !waits.some(interval => interval.kind === "typing")) { + const end = waits.filter(interval => interval.kind !== "hold") + .reduce((first, interval) => Math.min(first, interval.start), duration); + if (end > from) intervals.push({ kind: "typing", start: from, end: Math.min(end, duration) }); + } + const protectedInterval = interval => interval.kind === "hold" || interval.kind === "typing"; + const holds = mergeIntervals(intervals.filter(protectedInterval)); const waitsToCompress = []; - for (const { start, end } of mergeIntervals(intervals.filter(interval => interval.kind !== "hold"))) { + for (const { start, end } of mergeIntervals(intervals.filter(interval => !protectedInterval(interval)))) { let cursor = start; for (const hold of holds) { if (hold.end <= cursor) continue; diff --git a/site/scripts/clip-timeline.test.mjs b/site/scripts/clip-timeline.test.mjs new file mode 100644 index 000000000..a133b3a74 --- /dev/null +++ b/site/scripts/clip-timeline.test.mjs @@ -0,0 +1,101 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { clipSegments, parseFreezeDetect } from "./clip-timeline.mjs"; + +const fixture = name => readFileSync(new URL(`./fixtures/freezedetect-${name}.stderr`, import.meta.url), "utf8"); + +test("freezedetect stderr parses paired intervals, including a freeze starting at zero", () => { + const stderr = fixture("complete"); + for (const input of [stderr, stderr.replaceAll("\n", "\r\n")]) { + assert.deepEqual(parseFreezeDetect(input, 12), [{ start: 0, end: 2.4 }, { start: 4.4, end: 9.8 }]); + } +}); + +test("freezedetect closes an EOF freeze at the probed duration and accepts exponent timestamps", () => { + assert.deepEqual(parseFreezeDetect(fixture("eof"), 10), [{ start: 0.12, end: 2.12 }, { start: 3.24, end: 10 }]); + assert.deepEqual(parseFreezeDetect("[freezedetect] lavfi.freezedetect.freeze_start: 0", 10), [{ start: 0, end: 10 }]); + assert.deepEqual(parseFreezeDetect("[freezedetect] lavfi.freezedetect.freeze_start: 10", 10), []); +}); + +test("non-freeze stderr and freeze_duration diagnostics do not create intervals", () => { + assert.deepEqual(parseFreezeDetect("", 10), []); + assert.deepEqual(parseFreezeDetect("frame=250 time=00:00:10.00\n[freezedetect] lavfi.freezedetect.freeze_duration: 2\n", 10), []); +}); + +test("invalid or unordered freezedetect timestamps fail rather than silently over-compress", () => { + const log = events => events.map(([kind, at]) => `[freezedetect] lavfi.freezedetect.freeze_${kind}: ${at}\n`).join(""); + for (const events of [ + [["start", "NaN"]], [["start", "Infinity"]], [["start", ""]], [["start", -1]], [["start", 11]], + [["end", 4]], [["start", 2], ["start", 3]], [["start", 2], ["end", 1]], + [["start", 2], ["end", 2]], [["start", 2], ["end", 4], ["start", 3]], + ]) assert.throws(() => parseFreezeDetect(log(events), 10)); + for (const duration of [0, -1, NaN, Infinity]) assert.throws(() => parseFreezeDetect("", duration)); + // Permit the same small container/frame rounding tolerance as timeline waits. + assert.deepEqual(parseFreezeDetect(log([["start", 2], ["end", 10.1]]), 10), [{ start: 2, end: 10 }]); +}); + +test("freezes merge with waits before overlapping holds are subtracted, without mutating inputs", () => { + const waits = Object.freeze([ + Object.freeze({ kind: "hold", start: 20, end: 28 }), + Object.freeze({ kind: "model_thinking", start: 4, end: 8 }), + Object.freeze({ kind: "hold", start: 12, end: 16 }), + Object.freeze({ kind: "hold", start: 10, end: 14 }), + ]); + const freezes = Object.freeze([Object.freeze({ start: 7, end: 30 }), Object.freeze({ start: 0, end: 5 })]); + assert.deepEqual(clipSegments(waits, 32, 2, freezes), [ + { start: 2, end: 4, seconds: 2 }, // Typing survives a detected freeze. + { start: 4, end: 10, seconds: 0.6 }, // One merged wait/freeze, not two budgets. + { start: 10, end: 16, seconds: 6 }, + { start: 16, end: 20, seconds: 0.6 }, + { start: 20, end: 28, seconds: 8 }, + { start: 28, end: 30, seconds: 0.6 }, + { start: 30, end: 32, seconds: 2 }, + ]); +}); + +test("a freeze spanning typing, waits, scroll and final dwell cannot shorten protected time", () => { + assert.deepEqual(clipSegments([ + { kind: "model_thinking", start: 5, end: 10 }, + { kind: "hold", start: 18, end: 30 }, + ], 30, 2, [{ start: 0, end: 30 }]), [ + { start: 2, end: 5, seconds: 3 }, + { start: 5, end: 18, seconds: 0.6 }, + { start: 18, end: 30, seconds: 12 }, + ]); + // Without a post-typing wait boundary, keep the opening conservatively. + assert.deepEqual(clipSegments([{ kind: "hold", start: 10, end: 20 }], 20, 0, [{ start: 0, end: 20 }]), [ + { start: 0, end: 20, seconds: 20 }, + ]); +}); + +test("explicit typing takes priority over freezes and waits, and short remainders keep real speed", () => { + assert.deepEqual(clipSegments([ + { kind: "typing", start: 2, end: 5 }, + { kind: "tool_call", start: 1, end: 10 }, + { kind: "hold", start: 10.2, end: 12 }, + ], 12, 0, [{ start: 8, end: 10.2 }]), [ + { start: 0, end: 1, seconds: 1 }, + { start: 1, end: 2, seconds: 0.6 }, + { start: 2, end: 5, seconds: 3 }, + { start: 5, end: 10.2, seconds: 0.6 }, + { start: 10.2, end: 12, seconds: 12 - 10.2 }, + ]); + const segments = clipSegments([ + { kind: "typing", start: 0, end: 3 }, { kind: "hold", start: 3.4, end: 10 }, + ], 10, 0, [{ start: 0, end: 10 }]); + assert.equal(segments[1].seconds, 3.4 - 3); +}); + +test("frozen intervals validate and clip at raw video and retained start boundaries", () => { + for (const freezes of [null, [{ start: -1, end: 3 }], [{ start: 2, end: 1 }], [{ start: 0, end: Infinity }], [{ start: 5, end: 11 }]]) { + assert.throws(() => clipSegments([], 10, 0, freezes)); + } + assert.deepEqual(clipSegments([{ kind: "model_thinking", start: 1, end: 2 }], 10, 4, [ + { start: 0, end: 6 }, { start: 8, end: 10.1 }, + ]), [ + { start: 4, end: 6, seconds: 0.6 }, + { start: 6, end: 8, seconds: 2 }, + { start: 8, end: 10, seconds: 0.6 }, + ]); +}); diff --git a/site/scripts/encode-clips.mjs b/site/scripts/encode-clips.mjs index d31d85aa9..060c8407b 100644 --- a/site/scripts/encode-clips.mjs +++ b/site/scripts/encode-clips.mjs @@ -4,16 +4,17 @@ import { mkdir } from "node:fs/promises"; import { tmpdir } from "node:os"; import { basename, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; -import { clipFilter, clipSegments } from "./clip-timeline.mjs"; +import { clipFilter, clipSegments, freezeDetectFilter, parseFreezeDetect } from "./clip-timeline.mjs"; const input = resolve(process.argv[2] ?? fileURLToPath(new URL("../../ui/host/test-results/landing-recordings/", import.meta.url))); const output = resolve(process.argv[3] ?? fileURLToPath(new URL("../src/assets/landing/", import.meta.url))); const temporary = mkdtempSync(join(tmpdir(), "fanout-clips-")); function run(command, args, capture = false) { - const result = spawnSync(command, args, { encoding: "utf8", stdio: capture ? ["ignore", "pipe", "inherit"] : "inherit" }); + const result = spawnSync(command, args, { encoding: "utf8", maxBuffer: 16 * 1024 * 1024, + stdio: capture === "stderr" ? ["ignore", "ignore", "pipe"] : capture ? ["ignore", "pipe", "inherit"] : "inherit" }); if (result.error) throw result.error; - if (result.status !== 0) throw new Error(`${command} exited with status ${result.status}`); - return result.stdout; + if (result.status !== 0) throw new Error(`${command} exited with status ${result.status}${capture === "stderr" ? `\n${result.stderr}` : ""}`); + return capture === "stderr" ? result.stderr : result.stdout; } try { @@ -30,7 +31,13 @@ try { const raw = join(input, timeline.raw_file); const probe = JSON.parse(run("ffprobe", ["-v", "error", "-show_entries", "format=duration:stream=width,height", "-of", "json", raw], true)); if (!probe.streams.some(stream => stream.width === 1280 && stream.height === 800)) throw new Error(`${name} must be recorded at 1280 × 800`); - const segments = clipSegments(timeline.waits, Number(probe.format.duration), timeline.start); + const duration = Number(probe.format.duration); + // Decode the raw video, before trimming, so detection and timeline use the + // same clock. Info-level stderr carries freeze_start/end (including EOF). + const detection = run("ffmpeg", ["-hide_banner", "-nostdin", "-nostats", "-loglevel", "info", "-i", raw, + "-map", "0:v:0", "-an", "-vf", freezeDetectFilter, "-f", "null", "-"], "stderr"); + const freezes = parseFreezeDetect(detection, duration); + const segments = clipSegments(timeline.waits, duration, timeline.start, freezes); const seconds = segments.reduce((sum, segment) => sum + segment.seconds, 0); // Reserve 5% of a decimal 4 MB target for muxing; two passes share this budget. const bitrate = String(Math.floor(4_000_000 * 8 * 0.95 / seconds)); @@ -48,7 +55,7 @@ try { // Reverse only the final second so the PNG is the last encoded frame. run("ffmpeg", ["-hide_banner", "-loglevel", "warning", "-y", "-sseof", "-1", "-i", join(output, `${name}.mp4`), "-vf", "reverse", "-frames:v", "1", "-update", "1", join(output, `${name}.png`)]); - console.log(`${name}: ${seconds.toFixed(1)} s, 1280 × 800, 30 fps, approximately 4 MB per format`); + console.log(`${name}: ${seconds.toFixed(1)} s, ${freezes.length} frozen intervals, 1280 × 800, 30 fps, approximately 4 MB per format`); } } finally { rmSync(temporary, { recursive: true, force: true }); diff --git a/site/scripts/fixtures/freezedetect-complete.stderr b/site/scripts/fixtures/freezedetect-complete.stderr new file mode 100644 index 000000000..c4f59cce3 --- /dev/null +++ b/site/scripts/fixtures/freezedetect-complete.stderr @@ -0,0 +1,9 @@ +Input #0, matroska,webm, from 'sample.webm': + Duration: 00:00:12.00, start: 0.000000 +[Parsed_freezedetect_0 @ 0x1234] lavfi.freezedetect.freeze_start: 0 +[Parsed_freezedetect_0 @ 0x1234] lavfi.freezedetect.freeze_duration: 2.4 +[Parsed_freezedetect_0 @ 0x1234] lavfi.freezedetect.freeze_end: 2.4 +[Parsed_freezedetect_0 @ 0x1234] lavfi.freezedetect.freeze_start: 4.4 +[Parsed_freezedetect_0 @ 0x1234] lavfi.freezedetect.freeze_duration: 5.4 +[Parsed_freezedetect_0 @ 0x1234] lavfi.freezedetect.freeze_end: 9.8 +frame= 300 fps=0.0 q=-0.0 Lsize=N/A time=00:00:12.00 bitrate=N/A diff --git a/site/scripts/fixtures/freezedetect-eof.stderr b/site/scripts/fixtures/freezedetect-eof.stderr new file mode 100644 index 000000000..8a36c2794 --- /dev/null +++ b/site/scripts/fixtures/freezedetect-eof.stderr @@ -0,0 +1,6 @@ +[Parsed_freezedetect_0 @ 0x5678] lavfi.freezedetect.freeze_start: 1.2e-1 +[Parsed_freezedetect_0 @ 0x5678] lavfi.freezedetect.freeze_duration: 2 +[Parsed_freezedetect_0 @ 0x5678] lavfi.freezedetect.freeze_end: 2.12 +[Parsed_freezedetect_0 @ 0x5678] lavfi.freezedetect.freeze_start: 3.24 +[out#0/null @ 0x9876] video:100KiB audio:0KiB +frame= 250 fps=0.0 q=-0.0 Lsize=N/A time=00:00:10.00 bitrate=N/A diff --git a/site/scripts/landing-recording-scroll.test.mjs b/site/scripts/landing-recording-scroll.test.mjs new file mode 100644 index 000000000..00600a0f5 --- /dev/null +++ b/site/scripts/landing-recording-scroll.test.mjs @@ -0,0 +1,119 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { runInNewContext } from "node:vm"; +import { revealRecording, waitForConversation } from "../../ui/host/e2e/landing-recording-support.ts"; + +// Execute the actual page.evaluate callbacks with a virtual layout and clock. +// No browser is launched; iframe growth is represented by moving the answer. +function recordingPage({ view = "chat", answerReady = true } = {}) { + let now = 0, nextId = 0, scrollTop = 0, answerBottom = 1_300; + const pending = new Map(), observers = new Set(), positions = []; + const schedule = (callback, milliseconds) => { + const id = ++nextId; + pending.set(id, { at: now + milliseconds, callback }); + return id; + }; + class Observer { + constructor(callback) { this.callback = callback; this.targets = new Set(); observers.add(this); } + observe(target) { this.targets.add(target); } + disconnect() { observers.delete(this); } + } + const answer = { getBoundingClientRect: () => ({ bottom: 100 + answerBottom - scrollTop }) }; + const markdown = { textContent: "The final answer." }; + const content = {}; + const scroller = { + clientHeight: 500, clientTop: 0, firstElementChild: content, + get scrollHeight() { return view === "chat" ? answerBottom + 24 : 1_400; }, + get scrollTop() { return scrollTop; }, + set scrollTop(value) { + scrollTop = Math.max(0, Math.min(value, this.scrollHeight - this.clientHeight)); + positions.push({ at: now, top: scrollTop }); + }, + getBoundingClientRect: () => ({ top: 100 }), + querySelector: selector => selector === "[data-answer-position]" ? answer : answerReady ? markdown : null, + querySelectorAll: () => [answer], + dispatchEvent() {}, + }; + const environment = { + document: { querySelector: () => scroller, scrollingElement: scroller, fonts: { ready: Promise.resolve() } }, + performance: { now: () => now }, + requestAnimationFrame: callback => schedule(() => callback(now), 16), + cancelAnimationFrame: id => pending.delete(id), + setTimeout: schedule, clearTimeout: id => pending.delete(id), + ResizeObserver: Observer, MutationObserver: Observer, WheelEvent: class {}, + }; + const page = { + async evaluate(callback, argument) { + let done = false, error; + const result = runInNewContext(`(${callback.toString()})(argument)`, { ...environment, argument }); + result.then(() => { done = true; }, reason => { done = true; error = reason; }); + for (let steps = 0; !done && steps < 10_000; steps++) { + // Drain async continuations before advancing the simulated animation clock. + await new Promise(resolve => setImmediate(resolve)); + if (done) break; + const next = [...pending].sort((a, b) => a[1].at - b[1].at)[0]; + assert.ok(next, "evaluate must resolve or schedule more work"); + pending.delete(next[0]); + now = next[1].at; + next[1].callback(); + } + assert.ok(done, "evaluate must finish within the virtual clock budget"); + if (error) throw error; + return result; + }, + }; + return { + page, positions, observers, pending, + get now() { return now; }, + get expectedEnd() { return answerBottom - scroller.clientHeight; }, + get top() { return scrollTop; }, + resizeAt(at, bottom) { + schedule(() => { + answerBottom = bottom; + for (const observer of observers) observer.callback(); + }, at); + }, + renderAnswerAt(at) { schedule(() => { answerReady = true; }, at); }, + }; +} + +test("chat retargets late evidence growth during the scroll and holds the answer end for 4 seconds", async () => { + const recording = recordingPage(); + recording.resizeAt(3_000, 2_400); + recording.resizeAt(9_500, 3_100); // Growth during the final dwell must also stay aligned. + await revealRecording(recording.page, "chat"); + assert.equal(recording.top, recording.expectedEnd); + assert.ok(recording.positions.some(position => position.at > 8_100 && position.at < 9_500 && position.top === 1_900)); + assert.ok(recording.positions.filter(position => position.at >= 9_520).every(position => position.top === 2_600)); + // Initial distance sets an 8-second scroll; growth must not restart its duration. + assert.ok(recording.now >= 12_000 && recording.now < 12_150); +}); + +test("dashboard keeps the down/bottom/up timings and dwells at the top for only 3 seconds", async () => { + const recording = recordingPage({ view: "dashboard" }); + await revealRecording(recording.page, "dashboard"); + assert.equal(recording.top, 0); + assert.equal(Math.max(...recording.positions.map(position => position.top)), 900); + assert.ok(recording.positions.some(position => position.top > 0 && position.top < 900 && position.at > 7_500)); + assert.ok(recording.now >= 12_000 && recording.now < 12_150); + assert.equal(recording.now - recording.positions.at(-1).at, 3_000); +}); + +test("conversation readiness waits for rendered text and 1.5 seconds after the last resize", async () => { + const recording = recordingPage({ answerReady: false }); + recording.renderAnswerAt(1_500); + recording.resizeAt(1_000, 2_000); + recording.resizeAt(2_000, 2_400); + await waitForConversation(recording.page); + assert.ok(recording.now >= 3_500 && recording.now < 3_550); + assert.equal(recording.observers.size, 0); + assert.equal(recording.pending.size, 0); +}); + +test("conversation readiness fails instead of recording an answer that never renders", async () => { + const recording = recordingPage({ answerReady: false }); + await assert.rejects(waitForConversation(recording.page), /Conversation did not stop resizing/); + assert.equal(recording.now, 30_000); + assert.equal(recording.observers.size, 0); + assert.equal(recording.pending.size, 0); +}); diff --git a/site/scripts/landing-recordings.md b/site/scripts/landing-recordings.md index 9abdeb2d8..6fb92d11b 100644 --- a/site/scripts/landing-recordings.md +++ b/site/scripts/landing-recordings.md @@ -29,19 +29,35 @@ bun run e2e landing-recordings.spec.ts --project=1440-light --project=1440-dark Recordings use a 1280 × 800 viewport and raw video at the same size; Playwright video captures CSS pixels, so a higher device scale only pads the frame. This only uses the external instance. It types the configured prompts and waits for the -real run and its evidence views to finish. The chat clip then returns the +real run and every evidence iframe to load and finish rendering. A take fails +when the run shows a failed step, when a chat run has fewer than four evidence +views or any view without rows, or when the saved dashboard has fewer than six +panels or any non-text panel without rows. Before the +reveal, a ResizeObserver waits for 1.5 seconds without conversation size changes, +including individual evidence and answer blocks, with the final answer rendered. +This settling interval is recorded as a compressible `conversation_ready` wait. +The chat clip then returns the conversation pane to the user's question and scrolls through the evidence to the end of the answer over about 8 seconds, followed by a 4-second final dwell. +It measures the `[data-answer-position]` end on every animation frame and keeps +it aligned to the bottom of the conversation viewport throughout the dwell, +retargeting if an evidence view grows. The take fails unless the answer end is +bottom-aligned and its last text line is inside the viewport at the end. The dashboard clip opens the dashboard from its successful `create_dashboard` -result and waits for every saved panel to render. It scrolls from the top +result and records a compressible `dashboard_ready` wait from just before the +Open dashboard click until every saved panel has rendered and the first fold +has settled again. This includes navigation, loading spinners, lazy panel +loading and the setup scroll back to the top. It then scrolls from the top through all panels to the bottom over about 6 seconds, dwells there for 1.5 -seconds, and scrolls back to the top over 1.5 seconds before its 5-second final +seconds, and scrolls back to the top over 1.5 seconds before its 3-second final dwell. The dashboard poster therefore shows the top of the dashboard. Scrolls use awaited `requestAnimationFrame` animation with brief easing at the ends and a steady pace through the middle. Downward durations scale with the scroll distance: two viewport heights take the baseline 8 or 6 seconds, clamped -between that baseline and 12 seconds. It creates real chats and dashboards; +between that baseline and 12 seconds. Aim for encoded clips of 20–35 seconds; +review the actual duration after encoding, since real answer and panel content +vary. It creates real chats and dashboards; no data is mocked or intercepted. Use a dedicated recording account: before each take, the spec deletes all of @@ -53,8 +69,9 @@ Keep a copy before another Playwright run clears `test-results`. Timelines use seconds from page creation, with server event timestamps anchored to receipt of the run's response headers to avoid server clock skew. They list model-thinking and tool-call intervals from the unmodified AG-UI stream. The `waits` array also -includes a `kind: "hold"` interval covering the entire result scroll, including -the dashboard's bottom dwell and return to the top. Timelines contain no session +includes readiness waits and a `kind: "hold"` interval covering the entire result +scroll and final dwell, including the dashboard's bottom dwell and return to the +top. Timelines contain no session credentials or tool result payloads. Raw recordings can contain instance data. From the repository root, with ffmpeg and ffprobe installed: @@ -68,10 +85,29 @@ bash site/scripts/encode-clips.sh /path/to/landing-recordings It encodes every recording present in the input directory. An optional second directory argument writes encoded assets elsewhere for review. -The encoder merges overlapping waits, compresses each wait longer than 0.6 s -to 0.6 s, preserves holds at real speed even when they overlap a wait, and -preserves typing and the final dwell. A hold splits an overlapping wait into -separate compressible portions, each capped at 0.6 s. It writes VP9 WebM and H.264 +Before building segments, the encoder decodes the full raw video through +ffmpeg's `freezedetect=n=0.003:d=1.2`. The normalized full-frame noise tolerance +of `0.003` (about −50.5 dB on ffmpeg's scale) ignores small cursor blinks, +typing-caret flicker, and small streamed text or build-stage updates; the +minimum duration of 1.2 s catches near-static stretches without shortening +every brief pause. Detection runs at the raw frame rate and resolution, with +no resampling or downscaling. Its stderr timestamps share the raw timeline's +clock; an unfinished freeze at EOF ends at the ffprobe duration. + +The encoder merges overlapping or touching frozen intervals and timeline waits, +then subtracts protected intervals. Every remaining portion longer than 0.6 s +becomes exactly 0.6 s; shorter portions keep their original duration. Holds +preserve the entire scroll and final dwell at real speed, even if detection +marks them frozen. A hold splits a merged interval into separate compressible +portions, each capped at 0.6 s. Typing is protected from the clip start through +the first timeline wait, which starts after submission in these recordings. +An explicit `kind: "typing"` interval is also protected and replaces that +inference; without either an explicit typing interval or a wait boundary, the +opening is conservatively retained in full. These protections also override +timeline waits. The encoder reports the detected interval count and planned +duration for each clip. + +It writes VP9 WebM and H.264 MP4 at 1280 × 800 and 30 fps with a two-pass target of about 4 MB per file. It extracts each PNG poster from the final encoded frame. Review the clips and posters, then upload from the repository root with Wrangler logged in to the diff --git a/site/scripts/landing-recordings.test.mjs b/site/scripts/landing-recordings.test.mjs index eed716995..efa037c5e 100644 --- a/site/scripts/landing-recordings.test.mjs +++ b/site/scripts/landing-recordings.test.mjs @@ -6,17 +6,15 @@ import { clipSegments, clipFilter } from "./clip-timeline.mjs"; const stream = events => events.map(event => `data: ${JSON.stringify(event)}\n\n`).join(""); const event = (type, at, data = {}) => ({ type, timestamp: 1_000_000 + at * 1000, ...data }); -test("recording times survive server clock skew and preserve visible reasoning and answer text", () => { +test("recording times survive server clock skew and preserve the answer text", () => { const recorded = recordingTimeline(stream([ - event("RUN_STARTED", 0), event("REASONING_MESSAGE_START", 2), event("REASONING_MESSAGE_CONTENT", 3), - event("REASONING_MESSAGE_END", 4), event("TOOL_CALL_START", 6, { toolCallId: "q", toolCallName: "query_telemetry" }), + event("RUN_STARTED", 0), event("TOOL_CALL_START", 6, { toolCallId: "q", toolCallName: "query_telemetry" }), event("TOOL_CALL_END", 6.1, { toolCallId: "q" }), // Arguments ended; the tool is still running. event("TOOL_CALL_RESULT", 9, { toolCallId: "q", content: "{}" }), event("TEXT_MESSAGE_START", 12), event("TEXT_MESSAGE_CONTENT", 13), event("RUN_FINISHED", 14), ]), 10); assert.deepEqual(recorded.waits, [ - { kind: "model_thinking", start: 10, end: 12 }, - { kind: "model_thinking", start: 14, end: 16 }, + { kind: "model_thinking", start: 10, end: 16 }, { kind: "tool_call", start: 16, end: 19, tool: "query_telemetry" }, { kind: "model_thinking", start: 19, end: 22 }, ]); @@ -108,3 +106,31 @@ test("encoding cuts frames before the clip start", () => { ]); assert.throws(() => clipSegments([], 12, 12), /Clip start must be within the raw video/); }); + +test("dashboard navigation and panel loading compress while the scroll and 3-second final dwell stay real time", () => { + const segments = clipSegments([ + { kind: "model_thinking", start: 9, end: 32 }, + { kind: "conversation_ready", start: 32, end: 36 }, + // Includes clicking Open dashboard, navigation, lazy panels, and return to top. + { kind: "dashboard_ready", start: 36, end: 52 }, + { kind: "hold", start: 52, end: 70 }, // 12 down + 1.5 bottom + 1.5 up + 3 final. + ], 70, 2); + assert.deepEqual(segments, [ + { start: 2, end: 9, seconds: 7 }, + { start: 9, end: 52, seconds: 0.6 }, + { start: 52, end: 70, seconds: 18 }, + ]); + const duration = segments.reduce((sum, segment) => sum + segment.seconds, 0); + assert.ok(duration >= 20 && duration <= 35); +}); + +test("conversation layout settling compresses without shortening the answer scroll or 4-second dwell", () => { + assert.deepEqual(clipSegments([ + { kind: "conversation_ready", start: 10, end: 18 }, + { kind: "hold", start: 18, end: 30 }, // 8-second scroll and 4-second answer dwell. + ], 30), [ + { start: 0, end: 10, seconds: 10 }, + { start: 10, end: 18, seconds: 0.6 }, + { start: 18, end: 30, seconds: 12 }, + ]); +}); diff --git a/site/src/assets/landing/chat-dark.png b/site/src/assets/landing/chat-dark.png index e7c46bc60..895ca124d 100644 Binary files a/site/src/assets/landing/chat-dark.png and b/site/src/assets/landing/chat-dark.png differ diff --git a/site/src/assets/landing/chat-light.png b/site/src/assets/landing/chat-light.png index 7cb8ae8a1..c049d30be 100644 Binary files a/site/src/assets/landing/chat-light.png and b/site/src/assets/landing/chat-light.png differ diff --git a/site/src/assets/landing/dashboard-dark.png b/site/src/assets/landing/dashboard-dark.png index 3af26fcaf..d416ca6f7 100644 Binary files a/site/src/assets/landing/dashboard-dark.png and b/site/src/assets/landing/dashboard-dark.png differ diff --git a/site/src/assets/landing/dashboard-light.png b/site/src/assets/landing/dashboard-light.png index ef2cf7016..0f844e44f 100644 Binary files a/site/src/assets/landing/dashboard-light.png and b/site/src/assets/landing/dashboard-light.png differ diff --git a/site/src/content/docs/index.mdx b/site/src/content/docs/index.mdx index 2dc2112d4..8fa74fc14 100644 --- a/site/src/content/docs/index.mdx +++ b/site/src/content/docs/index.mdx @@ -58,9 +58,9 @@ import ThroughputReplayClient from "../../components/landing/ThroughputReplayCli ## Dashboards from a sentence. -Ask for checkout request rate, errors, p95 latency and slowest dependencies. The agent builds and saves the dashboard. +Ask for checkout's headline stats and trends, p95 latency by operation, slowest dependencies and recent slow traces. The agent builds and saves the dashboard. - + diff --git a/site/src/data/landing-media.json b/site/src/data/landing-media.json index 22a89c5de..13136895b 100644 --- a/site/src/data/landing-media.json +++ b/site/src/data/landing-media.json @@ -1,18 +1,18 @@ { "chat-dark": { - "mp4": "https://assets.fanout.run/landing/chat-dark-060356746cef.mp4", - "webm": "https://assets.fanout.run/landing/chat-dark-3afac098a9b8.webm" + "mp4": "https://assets.fanout.run/landing/chat-dark-847f67c660a2.mp4", + "webm": "https://assets.fanout.run/landing/chat-dark-d8ac7e78a674.webm" }, "chat-light": { - "mp4": "https://assets.fanout.run/landing/chat-light-3ff97d86374f.mp4", - "webm": "https://assets.fanout.run/landing/chat-light-3947b9e88408.webm" + "mp4": "https://assets.fanout.run/landing/chat-light-aed5fb359884.mp4", + "webm": "https://assets.fanout.run/landing/chat-light-4d3bcc09f5de.webm" }, "dashboard-dark": { - "mp4": "https://assets.fanout.run/landing/dashboard-dark-edf139c5a5a1.mp4", - "webm": "https://assets.fanout.run/landing/dashboard-dark-288fcd70d685.webm" + "mp4": "https://assets.fanout.run/landing/dashboard-dark-106bbe5a437d.mp4", + "webm": "https://assets.fanout.run/landing/dashboard-dark-5c4580915a9e.webm" }, "dashboard-light": { - "mp4": "https://assets.fanout.run/landing/dashboard-light-b51d120624a4.mp4", - "webm": "https://assets.fanout.run/landing/dashboard-light-dc0e9675651a.webm" + "mp4": "https://assets.fanout.run/landing/dashboard-light-096a6eaf1685.mp4", + "webm": "https://assets.fanout.run/landing/dashboard-light-3c5456d88c86.webm" } } diff --git a/ui/host/e2e/landing-recording-support.ts b/ui/host/e2e/landing-recording-support.ts index f45d4239f..ced441e3e 100644 --- a/ui/host/e2e/landing-recording-support.ts +++ b/ui/host/e2e/landing-recording-support.ts @@ -7,9 +7,43 @@ export type AgentEvent = { toolCallName?: string; content?: string; }; -export type WaitInterval = { kind: "model_thinking" | "tool_call" | "hold"; start: number; end: number; tool?: string }; +export type WaitInterval = { kind: "model_thinking" | "tool_call" | "conversation_ready" | "dashboard_ready" | "hold"; start: number; end: number; tool?: string }; -/** Show the completed result at reading speed before the recording's final dwell. */ +/** Iframe load and panel readiness are checked by the caller before measuring layout. */ +export async function waitForConversation(page: Page) { + await page.evaluate(async () => { + await document.fonts.ready; + const scroller = document.querySelector(".chat-pane .chat-scroll"); + if (!scroller?.firstElementChild) throw new Error("No conversation content for the landing recording"); + await new Promise((resolve, reject) => { + let changedAt = performance.now(); + let frame: number; + const resized = new ResizeObserver(() => { changedAt = performance.now(); }); + const observe = () => { + for (const element of [scroller, scroller.firstElementChild!, + ...scroller.querySelectorAll("[data-chat-turn], [data-chat-anchor], [data-answer-position], iframe")]) resized.observe(element); + }; + const mutated = new MutationObserver(() => { changedAt = performance.now(); observe(); }); + const cleanup = () => { resized.disconnect(); mutated.disconnect(); cancelAnimationFrame(frame); clearTimeout(timeout); }; + const timeout = setTimeout(() => { + cleanup(); + reject(new Error("Conversation did not stop resizing for 1.5 seconds")); + }, 30_000); + observe(); + mutated.observe(scroller, { childList: true, subtree: true, characterData: true }); + const check = () => { + const answer = scroller.querySelector('[data-answer-position] .chat-markdown:not([data-provisional])'); + if (answer?.textContent?.trim() && performance.now() - changedAt >= 1_500) { + cleanup(); + resolve(); + } else frame = requestAnimationFrame(check); + }; + frame = requestAnimationFrame(check); + }); + }); +} + +/** Show the completed result at reading speed, then hold its final frame. */ export async function revealRecording(page: Page, view: "chat" | "dashboard") { await page.evaluate(async view => { const scroller = view === "chat" @@ -17,14 +51,19 @@ export async function revealRecording(page: Page, view: "chat" | "dashboard") { : document.scrollingElement as HTMLElement | null; if (!scroller) throw new Error(`No ${view} scroll container for the landing recording`); - // Tell the chat's stick-to-bottom hook that the reader is scrolling up. + const answer = view === "chat" ? scroller.querySelector("[data-answer-position]") : null; + if (view === "chat" && !answer) throw new Error("No final answer for the landing recording"); + const answerEnd = () => scroller.scrollTop + answer!.getBoundingClientRect().bottom + - scroller.getBoundingClientRect().top - scroller.clientTop - scroller.clientHeight; + + // Release the question anchor before controlling the recording's scroll. if (view === "chat") scroller.dispatchEvent(new WheelEvent("wheel", { deltaY: -1 })); scroller.scrollTop = 0; await new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(() => resolve()))); - const animate = async (to: number, milliseconds: number) => { + const animate = async (target: () => number, milliseconds: number) => { const from = scroller.scrollTop; - if (to === from) return; + if (target() === from) return; const start = performance.now(); await new Promise(resolve => { const frame = (now: number) => { @@ -34,21 +73,35 @@ export async function revealRecording(page: Page, view: "chat" | "dashboard") { const eased = progress < ramp ? progress * progress / (2 * ramp * (1 - ramp)) : progress > 1 - ramp ? 1 - (1 - progress) ** 2 / (2 * ramp * (1 - ramp)) : (progress - ramp / 2) / (1 - ramp); - scroller.scrollTop = from + (to - from) * eased; + // Re-measure every frame: evidence can still grow after readiness. + scroller.scrollTop = from + (target() - from) * eased; if (progress < 1) requestAnimationFrame(frame); - else requestAnimationFrame(() => resolve()); + else requestAnimationFrame(() => { scroller.scrollTop = target(); resolve(); }); }; requestAnimationFrame(frame); }); }; - const bottom = Math.max(0, scroller.scrollHeight - scroller.clientHeight); + const target = view === "chat" ? answerEnd : () => Math.max(0, scroller.scrollHeight - scroller.clientHeight); + const bottom = Math.max(0, target()); const minimum = view === "chat" ? 8_000 : 6_000; // Two viewport heights take the baseline duration; longer results get more time. const duration = Math.min(12_000, Math.max(minimum, minimum * bottom / Math.max(1, scroller.clientHeight * 2))); - await animate(bottom, duration); + await animate(target, duration); if (view === "dashboard") { await new Promise(resolve => setTimeout(resolve, 1_500)); - await animate(0, 1_500); + await animate(() => 0, 1_500); + await new Promise(resolve => setTimeout(resolve, 3_000)); + } else { + // Keep the answer end aligned throughout the dwell, including late iframe resizes. + const until = performance.now() + 4_000; + await new Promise(resolve => { + const hold = () => { + scroller.scrollTop = answerEnd(); + if (performance.now() < until) requestAnimationFrame(hold); + else resolve(); + }; + requestAnimationFrame(hold); + }); } }, view); } diff --git a/ui/host/e2e/landing-recordings.spec.ts b/ui/host/e2e/landing-recordings.spec.ts index 31784dc47..65a689141 100644 --- a/ui/host/e2e/landing-recordings.spec.ts +++ b/ui/host/e2e/landing-recordings.spec.ts @@ -2,7 +2,7 @@ import { expect, test } from "@playwright/test"; import { mkdir, writeFile } from "node:fs/promises"; import { fileURLToPath } from "node:url"; import { cardSelector, readyCard, settle } from "./accessibility-support"; -import { recordingTimeline, revealRecording } from "./landing-recording-support"; +import { recordingTimeline, revealRecording, waitForConversation } from "./landing-recording-support"; import type { DashboardSpec, PanelResult } from "../../panels/types"; const baseURL = process.env.FANOUT_LANDING_BASE_URL; @@ -22,7 +22,7 @@ test.use({ const prompts = { chat: "Why is checkout slower in the last hour?", - dashboard: "Build a dashboard for checkout: request rate, error rate, p95 latency by operation, its slowest dependencies, and recent slow traces", + dashboard: "Build a dashboard for checkout: headline stats and trends for request rate, error rate and p95 latency, p95 by operation, slowest dependencies, and recent slow traces", }; for (const view of ["chat", "dashboard"] as const) { @@ -89,7 +89,7 @@ for (const view of ["chat", "dashboard"] as const) { await input.click(); // A steady human typing cadence, including a short pause at punctuation. for (const character of prompts[view]) { - await input.pressSequentially(character, { delay: /[,:?]/.test(character) ? 180 : 55 }); + await input.pressSequentially(character, { delay: /[,:?]/.test(character) ? 120 : 28 }); } const responseReady = page.waitForResponse(response => new URL(response.url()).pathname === "/api/agent/runs" && response.request().method() === "POST", { timeout: 30_000 }); @@ -105,26 +105,33 @@ for (const view of ["chat", "dashboard"] as const) { content?: { tool_result?: { view: { key: string }; results: PanelResult[] } }; }).filter(event => event.type === "ACTIVITY_SNAPSHOT" && event.activityType === "mcp-app") .map(event => event.content!.tool_result!); - expect(new Set(evidence.map(fragment => fragment.view.key)).size, "Chat must show at least four evidence views").toBeGreaterThanOrEqual(4); + if (view === "chat") expect(new Set(evidence.map(fragment => fragment.view.key)).size, "Chat must show at least four evidence views").toBeGreaterThanOrEqual(4); for (const fragment of evidence) { expect(fragment.results.some(result => result.status === "ok" && (result.frame?.rows ?? 0) > 0), "Chat must not contain an empty evidence view").toBe(true); } + const conversationWaitStart = (Date.now() - startedAt) / 1000; // RUN_FINISHED in the real stream plus the UI's idle state gates the reveal. await expect(page.getByRole("button", { name: "Stop", exact: true })).toHaveCount(0); await expect(page.locator(".chat-pane .mantine-Loader-root")).toHaveCount(0); await expect(page.locator(".chat-pane").getByRole("alert")).toHaveCount(0); + await expect(page.locator(".chat-pane").getByText(/^\d+ steps? failed$/), "The run must not show failed steps").toHaveCount(0); await expect(page.locator('[data-answer-position] [data-role="assistant"]')).toBeVisible(); const views = page.locator('iframe[title="Fanout analysis view"]'); for (let index = 0; index < await views.count(); index++) { const frame = views.nth(index).contentFrame(); + await expect.poll(() => frame.locator("html").evaluate(element => element.ownerDocument.readyState)).toBe("complete"); await expect(frame.locator("[data-panel-body]").first()).toBeAttached(); await expect(frame.locator(".mantine-Loader-root, [data-panel-error]")).toHaveCount(0); } + await settle(page); + await waitForConversation(page); + timeline.waits.push({ kind: "conversation_ready", start: conversationWaitStart, end: (Date.now() - startedAt) / 1000 }); if (view === "dashboard") { if (!timeline.dashboardId) throw new Error("No successful create_dashboard result with a committed save receipt; the recording requires a newly saved dashboard"); const receipt = page.locator(`[data-dashboard-result="${timeline.dashboardId}"]`); + const dashboardWaitStart = (Date.now() - startedAt) / 1000; await receipt.getByRole("link", { name: "Open dashboard", exact: true }).click(); await expect(page).toHaveURL(new RegExp(`/dashboards/${timeline.dashboardId}(?:\\?|$)`)); const saved = await context.request.get(`/api/dashboards/${encodeURIComponent(timeline.dashboardId)}`); @@ -148,13 +155,30 @@ for (const view of ["chat", "dashboard"] as const) { const card = page.locator(cardSelector(panel.id)); await expect(card.locator('[aria-label="Loading panel"], [aria-label="Refreshing"], [data-panel-error]')).toHaveCount(0); } + await settle(page); + timeline.waits.push({ kind: "dashboard_ready", start: dashboardWaitStart, end: (Date.now() - startedAt) / 1000 }); } - await settle(page); await page.mouse.move(0, 0); const holdStart = (Date.now() - startedAt) / 1000; await revealRecording(page, view); timeline.waits.push({ kind: "hold", start: holdStart, end: (Date.now() - startedAt) / 1000 }); - await page.waitForTimeout(view === "dashboard" ? 5_000 : 4_000); + if (view === "chat") { + const visible = await page.locator("[data-answer-position]").evaluate(answer => { + const pane = answer.closest(".chat-scroll")!; + const top = pane.getBoundingClientRect().top + pane.clientTop; + const bottom = top + pane.clientHeight; + const markdown = answer.querySelector(".chat-markdown")!; + const walker = document.createTreeWalker(markdown, NodeFilter.SHOW_TEXT); + let last: Node | undefined; + for (let node = walker.nextNode(); node; node = walker.nextNode()) if (node.textContent?.trim()) last = node; + const range = document.createRange(); + if (last) range.selectNodeContents(last); + const line = [...range.getClientRects()].at(-1); + return { aligned: Math.abs(answer.getBoundingClientRect().bottom - bottom) <= 2, + lastLineVisible: !!line && line.top >= top && line.bottom <= bottom + 1 }; + }); + expect(visible, "The final answer end must stay at the pane bottom with its last line in view").toEqual({ aligned: true, lastLineVisible: true }); + } const duration = (Date.now() - startedAt) / 1000; await context.close(); const rawFile = `${view}-${theme}.webm`;