From 9ee02756bdd36ea2a0d04511a6b422b715f82417 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sat, 17 Oct 2026 17:58:11 -0700 Subject: [PATCH] site: scroll landing recordings slowly through what was created Clips cut straight to the end of a result: the answer arrives at once and the encoder compresses waits, so a viewer never saw what the agent made. After a run finishes, the chat recording scrolls from the question through the evidence to the end of the answer, and the dashboard recording scrolls through every panel and back to the top, so its final frame and poster show the headline stats. Scrolls ease with requestAnimationFrame at a pace scaled to the distance, and the timeline marks them as holds that the encoder keeps at real speed even where they overlap a wait. Claude-Session: https://claude.ai/code/session_018HfyEXHyGD1utmKF4CQtyQ --- site/scripts/clip-timeline.mjs | 37 +++++++++++++----- site/scripts/landing-recordings.md | 29 ++++++++++---- site/scripts/landing-recordings.test.mjs | 28 ++++++++++++++ ui/host/e2e/landing-recording-support.ts | 48 +++++++++++++++++++++++- ui/host/e2e/landing-recordings.spec.ts | 7 +++- 5 files changed, 129 insertions(+), 20 deletions(-) diff --git a/site/scripts/clip-timeline.mjs b/site/scripts/clip-timeline.mjs index c56a717f9..16319eca6 100644 --- a/site/scripts/clip-timeline.mjs +++ b/site/scripts/clip-timeline.mjs @@ -1,24 +1,41 @@ -/** Merge concurrent tool/model waits so no source frame is compressed twice. Frames before from are cut. */ +function mergeIntervals(intervals) { + const merged = []; + for (const { start, end } of intervals.sort((a, b) => a.start - b.start)) { + const previous = merged.at(-1); + if (previous && start <= previous.end) previous.end = Math.max(previous.end, end); + else merged.push({ start, end }); + } + return merged; +} + +/** Merge concurrent waits, preserving holds at real speed even inside a wait. Frames before from are cut. */ export function clipSegments(waits, duration, from = 0) { if (!Number.isFinite(duration) || duration <= 0) throw new Error("Video duration must be positive"); if (!Number.isFinite(from) || from < 0 || from >= duration) throw new Error("Clip start must be within the raw video"); if (!Array.isArray(waits)) throw new Error("Timeline waits must be an array"); - const intervals = waits.map(({ start, end }) => { + const intervals = waits.map(({ kind, start, end }) => { if (!Number.isFinite(start) || !Number.isFinite(end) || start < 0 || end <= start) { throw new Error("Wait intervals must have finite seconds with 0 <= start < end"); } if (end > duration + 0.25) throw new Error("Wait interval extends past the raw video"); - return { start: Math.max(from, Math.min(start, duration)), end: Math.min(end, duration) }; - }).filter(({ start, end }) => end > start).sort((a, b) => a.start - b.start); - const merged = []; - for (const interval of intervals) { - const previous = merged.at(-1); - if (previous && interval.start <= previous.end) previous.end = Math.max(previous.end, interval.end); - else merged.push({ ...interval }); + return { kind, start: Math.max(from, Math.min(start, duration)), end: Math.min(end, duration) }; + }).filter(({ start, end }) => end > start); + const holds = mergeIntervals(intervals.filter(interval => interval.kind === "hold")); + const waitsToCompress = []; + for (const { start, end } of mergeIntervals(intervals.filter(interval => interval.kind !== "hold"))) { + let cursor = start; + for (const hold of holds) { + if (hold.end <= cursor) continue; + if (hold.start >= end) break; + if (hold.start > cursor) waitsToCompress.push({ start: cursor, end: hold.start }); + cursor = Math.max(cursor, hold.end); + if (cursor >= end) break; + } + if (cursor < end) waitsToCompress.push({ start: cursor, end }); } const segments = []; let cursor = from; - for (const { start, end } of merged) { + for (const { start, end } of waitsToCompress) { if (start > cursor) segments.push({ start: cursor, end: start, seconds: start - cursor }); if (end > start) segments.push({ start, end, seconds: Math.min(0.6, end - start) }); cursor = end; diff --git a/site/scripts/landing-recordings.md b/site/scripts/landing-recordings.md index 09c4a7320..9abdeb2d8 100644 --- a/site/scripts/landing-recordings.md +++ b/site/scripts/landing-recordings.md @@ -27,11 +27,22 @@ bun run e2e landing-recordings.spec.ts --project=1440-light --project=1440-dark ``` Recordings use a 1280 × 800 viewport and raw video at the same size; Playwright -video captures CSS pixels, so a higher device scale only pads the frame. It types the configured prompts, waits for the real run to finish, and dwells -for four seconds on chat or five seconds on the dashboard. The dashboard clip also -opens the dashboard from its successful `create_dashboard` result and waits -for the saved panels to render. It creates real chats and dashboards; no data -is mocked or intercepted. +video captures CSS pixels, so a higher device scale only pads the frame. This +only uses the external instance. It types the configured prompts and waits for the +real run and its evidence views to finish. The chat clip then returns the +conversation pane to the user's question and scrolls through the evidence to +the end of the answer over about 8 seconds, followed by a 4-second final dwell. +The dashboard clip opens the dashboard from its successful `create_dashboard` +result and waits for every saved panel to render. It scrolls from the top +through all panels to the bottom over about 6 seconds, dwells there for 1.5 +seconds, and scrolls back to the top over 1.5 seconds before its 5-second final +dwell. The dashboard poster therefore shows the top of the dashboard. + +Scrolls use awaited `requestAnimationFrame` animation with brief easing at the +ends and a steady pace through the middle. Downward durations scale with the +scroll distance: two viewport heights take the baseline 8 or 6 seconds, clamped +between that baseline and 12 seconds. It creates real chats and dashboards; +no data is mocked or intercepted. Use a dedicated recording account: before each take, the spec deletes all of that account’s chats and non-default dashboards. @@ -41,7 +52,9 @@ Raw WebM files and matching JSON timelines go to Keep a copy before another Playwright run clears `test-results`. Timelines use seconds from page creation, with server event timestamps anchored to receipt of the run's response headers to avoid server clock skew. They list model-thinking -and tool-call intervals from the unmodified AG-UI stream and contain no session +and tool-call intervals from the unmodified AG-UI stream. The `waits` array also +includes a `kind: "hold"` interval covering the entire result scroll, including +the dashboard's bottom dwell and return to the top. Timelines contain no session credentials or tool result payloads. Raw recordings can contain instance data. From the repository root, with ffmpeg and ffprobe installed: @@ -56,7 +69,9 @@ It encodes every recording present in the input directory. An optional second directory argument writes encoded assets elsewhere for review. The encoder merges overlapping waits, compresses each wait longer than 0.6 s -to 0.6 s, preserves typing and the final dwell, and writes VP9 WebM and H.264 +to 0.6 s, preserves holds at real speed even when they overlap a wait, and +preserves typing and the final dwell. A hold splits an overlapping wait into +separate compressible portions, each capped at 0.6 s. It writes VP9 WebM and H.264 MP4 at 1280 × 800 and 30 fps with a two-pass target of about 4 MB per file. It extracts each PNG poster from the final encoded frame. Review the clips and posters, then upload from the repository root with Wrangler logged in to the diff --git a/site/scripts/landing-recordings.test.mjs b/site/scripts/landing-recordings.test.mjs index 0c75de8e8..eed716995 100644 --- a/site/scripts/landing-recordings.test.mjs +++ b/site/scripts/landing-recordings.test.mjs @@ -71,6 +71,34 @@ test("invalid trim times fail before invoking ffmpeg", () => { assert.throws(() => clipSegments([], 0)); }); +test("encoding preserves overlapping holds at real speed inside merged waits", () => { + const segments = clipSegments([ + { kind: "tool_call", start: 2, end: 10 }, + { kind: "model_thinking", start: 8, end: 18 }, + { kind: "hold", start: 6, end: 12 }, + { kind: "hold", start: 4, end: 9 }, + ], 20); + assert.deepEqual(segments, [ + { start: 0, end: 2, seconds: 2 }, + { start: 2, end: 4, seconds: 0.6 }, + { start: 4, end: 12, seconds: 8 }, + { start: 12, end: 18, seconds: 0.6 }, + { start: 18, end: 20, seconds: 2 }, + ]); + assert.match(clipFilter(segments), /trim=start=4:end=12,setpts=\(PTS-STARTPTS\)\*1\[v2\]/); +}); + +test("holds can cover entire waits and are clipped at the retained video boundaries", () => { + assert.deepEqual(clipSegments([ + { kind: "hold", start: 1, end: 10.1 }, + { kind: "tool_call", start: 2, end: 8 }, + ], 10, 4), [{ start: 4, end: 10, seconds: 6 }]); + assert.deepEqual(clipSegments([{ kind: "hold", start: 2, end: 6 }], 10), [ + { start: 0, end: 10, seconds: 10 }, + ]); + assert.throws(() => clipSegments([{ kind: "hold", start: 2, end: 1 }], 10)); +}); + test("encoding cuts frames before the clip start", () => { const segments = clipSegments([{ start: 1, end: 3 }, { start: 5, end: 8 }], 12, 4); assert.deepEqual(segments, [ diff --git a/ui/host/e2e/landing-recording-support.ts b/ui/host/e2e/landing-recording-support.ts index 43d61081f..e469d1c49 100644 --- a/ui/host/e2e/landing-recording-support.ts +++ b/ui/host/e2e/landing-recording-support.ts @@ -1,3 +1,5 @@ +import type { Page } from "@playwright/test"; + export type AgentEvent = { type: string; timestamp: number; @@ -5,7 +7,51 @@ export type AgentEvent = { toolCallName?: string; content?: string; }; -export type WaitInterval = { kind: "model_thinking" | "tool_call"; start: number; end: number; tool?: string }; +export type WaitInterval = { kind: "model_thinking" | "tool_call" | "hold"; start: number; end: number; tool?: string }; + +/** Show the completed result at reading speed before the recording's final dwell. */ +export async function revealRecording(page: Page, view: "chat" | "dashboard") { + await page.evaluate(async view => { + const scroller = view === "chat" + ? document.querySelector(".chat-pane .chat-scroll") + : document.scrollingElement as HTMLElement | null; + if (!scroller) throw new Error(`No ${view} scroll container for the landing recording`); + + // Tell the chat's stick-to-bottom hook that the reader is scrolling up. + if (view === "chat") scroller.dispatchEvent(new WheelEvent("wheel", { deltaY: -1 })); + scroller.scrollTop = 0; + await new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(() => resolve()))); + + const animate = async (to: number, milliseconds: number) => { + const from = scroller.scrollTop; + if (to === from) return; + const start = performance.now(); + await new Promise(resolve => { + const frame = (now: number) => { + const progress = Math.min(1, (now - start) / milliseconds); + // Brief acceleration/deceleration around a constant-speed middle 80%. + const ramp = 0.1; + const eased = progress < ramp ? progress * progress / (2 * ramp * (1 - ramp)) + : progress > 1 - ramp ? 1 - (1 - progress) ** 2 / (2 * ramp * (1 - ramp)) + : (progress - ramp / 2) / (1 - ramp); + scroller.scrollTop = from + (to - from) * eased; + if (progress < 1) requestAnimationFrame(frame); + else requestAnimationFrame(() => resolve()); + }; + requestAnimationFrame(frame); + }); + }; + const bottom = Math.max(0, scroller.scrollHeight - scroller.clientHeight); + const minimum = view === "chat" ? 8_000 : 6_000; + // Two viewport heights take the baseline duration; longer results get more time. + const duration = Math.min(12_000, Math.max(minimum, minimum * bottom / Math.max(1, scroller.clientHeight * 2))); + await animate(bottom, duration); + if (view === "dashboard") { + await new Promise(resolve => setTimeout(resolve, 1_500)); + await animate(0, 1_500); + } + }, view); +} /** Observe the completed, unmodified SSE response; never route or replace requests. */ export function recordingTimeline(stream: string, responseAt: number) { diff --git a/ui/host/e2e/landing-recordings.spec.ts b/ui/host/e2e/landing-recordings.spec.ts index 170286f5b..a912195e4 100644 --- a/ui/host/e2e/landing-recordings.spec.ts +++ b/ui/host/e2e/landing-recordings.spec.ts @@ -2,7 +2,7 @@ import { expect, test } from "@playwright/test"; import { mkdir, writeFile } from "node:fs/promises"; import { fileURLToPath } from "node:url"; import { cardSelector, readyCard, settle } from "./accessibility-support"; -import { recordingTimeline } from "./landing-recording-support"; +import { recordingTimeline, revealRecording } from "./landing-recording-support"; import type { DashboardSpec } from "../../panels/types"; const baseURL = process.env.FANOUT_LANDING_BASE_URL; @@ -98,7 +98,7 @@ for (const view of ["chat", "dashboard"] as const) { const responseAt = (Date.now() - startedAt) / 1000; expect(response.ok(), "Agent request failed; check chat configuration and the session's agent:run capability").toBe(true); const timeline = recordingTimeline(await response.text(), responseAt); - // RUN_FINISHED in the real stream plus the UI's idle state gates the dwell. + // RUN_FINISHED in the real stream plus the UI's idle state gates the reveal. await expect(page.getByRole("button", { name: "Stop", exact: true })).toHaveCount(0); await expect(page.locator(".chat-pane .mantine-Loader-root")).toHaveCount(0); await expect(page.locator(".chat-pane").getByRole("alert")).toHaveCount(0); @@ -129,6 +129,9 @@ for (const view of ["chat", "dashboard"] as const) { } await settle(page); await page.mouse.move(0, 0); + const holdStart = (Date.now() - startedAt) / 1000; + await revealRecording(page, view); + timeline.waits.push({ kind: "hold", start: holdStart, end: (Date.now() - startedAt) / 1000 }); await page.waitForTimeout(view === "dashboard" ? 5_000 : 4_000); const duration = (Date.now() - startedAt) / 1000; await context.close();