diff --git a/docs/testing.md b/docs/testing.md index d465c62cc..a23f77f82 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -281,4 +281,4 @@ outside the verification environment's scope. Neither command publishes the site Refresh the README's light and dark System overview captures with `just landing-captures` (real seeded server and local Chrome). To capture an existing instance, set `FANOUT_LANDING_BASE_URL` to its URL and `FANOUT_LANDING_SESSION_COOKIE` to a `name=value` session cookie before running the same command; no server starts and no data is seeded. -To record chat and dashboard clips, set `FANOUT_LANDING_RECORDINGS=1` and the external-instance variables above, then run `bun run e2e e2e/specs/app/landing-recordings.spec.ts --project=app-1440-light --project=app-1440-dark` from `ui/host`. Follow [landing-recordings.md](../site/scripts/landing-recordings.md) to encode and upload them. +The landing page's chat and dashboard clips are produced outside this repository; [landing-clips.md](../site/scripts/landing-clips.md) covers the assets they need and how to publish them. diff --git a/site/scripts/clip-timeline.mjs b/site/scripts/clip-timeline.mjs deleted file mode 100644 index 02d54f174..000000000 --- a/site/scripts/clip-timeline.mjs +++ /dev/null @@ -1,91 +0,0 @@ -// Full-frame mean difference: tolerate cursor/caret flicker and small streamed -// text updates, but retain larger visual changes. Require 1.2 s of near-stasis. -export const freezeDetectFilter = "freezedetect=n=0.003:d=1.2"; - -/** Parse ffmpeg stderr; an open freeze at EOF extends to the probed duration. */ -export function parseFreezeDetect(stderr, duration) { - if (!Number.isFinite(duration) || duration <= 0) throw new Error("Video duration must be positive"); - const intervals = []; - let start; - for (const match of stderr.matchAll(/lavfi\.freezedetect\.freeze_(start|end):\s*([^\r\n]*)/g)) { - const value = match[2].trim(); - const at = Number(value); - if (!value || !Number.isFinite(at) || at < 0 || at > duration + 0.25) { - throw new Error("Invalid freezedetect timestamp"); - } - if (match[1] === "start") { - if (start !== undefined || at < (intervals.at(-1)?.end ?? 0)) throw new Error("Unordered freezedetect start"); - start = Math.min(at, duration); - } else { - if (start === undefined || at <= start) throw new Error("Unpaired or unordered freezedetect end"); - const end = Math.min(at, duration); - if (end > start) intervals.push({ start, end }); - start = undefined; - } - } - if (start !== undefined && start < duration) intervals.push({ start, end: duration }); - return intervals; -} - -function mergeIntervals(intervals) { - const merged = []; - for (const { start, end } of intervals.sort((a, b) => a.start - b.start)) { - const previous = merged.at(-1); - if (previous && start <= previous.end) previous.end = Math.max(previous.end, end); - else merged.push({ start, end }); - } - return merged; -} - -/** Merge waits and freezes, then subtract holds/typing before compressing. */ -export function clipSegments(waits, duration, from = 0, freezes = []) { - if (!Number.isFinite(duration) || duration <= 0) throw new Error("Video duration must be positive"); - if (!Number.isFinite(from) || from < 0 || from >= duration) throw new Error("Clip start must be within the raw video"); - if (!Array.isArray(waits)) throw new Error("Timeline waits must be an array"); - if (!Array.isArray(freezes)) throw new Error("Frozen intervals must be an array"); - const intervals = [...waits, ...freezes.map(({ start, end }) => ({ kind: "freeze", start, end }))].map(({ kind, start, end }) => { - if (!Number.isFinite(start) || !Number.isFinite(end) || start < 0 || end <= start) { - throw new Error("Wait intervals must have finite seconds with 0 <= start < end"); - } - if (end > duration + 0.25) throw new Error("Wait interval extends past the raw video"); - return { kind, start: Math.max(from, Math.min(start, duration)), end: Math.min(end, duration) }; - }).filter(({ start, end }) => end > start); - // Version 1 recordings start before typing and first mark a wait after Send. - // Protect that whole opening even if caret flicker/typing looks frozen. With - // no wait boundary, conservatively keep it all; explicit typing overrides this. - if (freezes.length && !waits.some(interval => interval.kind === "typing")) { - const end = waits.filter(interval => interval.kind !== "hold") - .reduce((first, interval) => Math.min(first, interval.start), duration); - if (end > from) intervals.push({ kind: "typing", start: from, end: Math.min(end, duration) }); - } - const protectedInterval = interval => interval.kind === "hold" || interval.kind === "typing"; - const holds = mergeIntervals(intervals.filter(protectedInterval)); - const waitsToCompress = []; - for (const { start, end } of mergeIntervals(intervals.filter(interval => !protectedInterval(interval)))) { - let cursor = start; - for (const hold of holds) { - if (hold.end <= cursor) continue; - if (hold.start >= end) break; - if (hold.start > cursor) waitsToCompress.push({ start: cursor, end: hold.start }); - cursor = Math.max(cursor, hold.end); - if (cursor >= end) break; - } - if (cursor < end) waitsToCompress.push({ start: cursor, end }); - } - const segments = []; - let cursor = from; - for (const { start, end } of waitsToCompress) { - if (start > cursor) segments.push({ start: cursor, end: start, seconds: start - cursor }); - if (end > start) segments.push({ start, end, seconds: Math.min(0.6, end - start) }); - cursor = end; - } - if (cursor < duration) segments.push({ start: cursor, end: duration, seconds: duration - cursor }); - return segments; -} - -export function clipFilter(segments) { - const filters = segments.map(({ start, end, seconds }, index) => - `[0:v]trim=start=${start}:end=${end},setpts=(PTS-STARTPTS)*${seconds / (end - start)}[v${index}]`); - filters.push(`${segments.map((_, index) => `[v${index}]`).join("")}concat=n=${segments.length}:v=1:a=0,scale=1280:800,fps=30,format=yuv420p[out]`); - return filters.join(";\n"); -} diff --git a/site/scripts/clip-timeline.test.mjs b/site/scripts/clip-timeline.test.mjs deleted file mode 100644 index a133b3a74..000000000 --- a/site/scripts/clip-timeline.test.mjs +++ /dev/null @@ -1,101 +0,0 @@ -import { test } from "node:test"; -import assert from "node:assert/strict"; -import { readFileSync } from "node:fs"; -import { clipSegments, parseFreezeDetect } from "./clip-timeline.mjs"; - -const fixture = name => readFileSync(new URL(`./fixtures/freezedetect-${name}.stderr`, import.meta.url), "utf8"); - -test("freezedetect stderr parses paired intervals, including a freeze starting at zero", () => { - const stderr = fixture("complete"); - for (const input of [stderr, stderr.replaceAll("\n", "\r\n")]) { - assert.deepEqual(parseFreezeDetect(input, 12), [{ start: 0, end: 2.4 }, { start: 4.4, end: 9.8 }]); - } -}); - -test("freezedetect closes an EOF freeze at the probed duration and accepts exponent timestamps", () => { - assert.deepEqual(parseFreezeDetect(fixture("eof"), 10), [{ start: 0.12, end: 2.12 }, { start: 3.24, end: 10 }]); - assert.deepEqual(parseFreezeDetect("[freezedetect] lavfi.freezedetect.freeze_start: 0", 10), [{ start: 0, end: 10 }]); - assert.deepEqual(parseFreezeDetect("[freezedetect] lavfi.freezedetect.freeze_start: 10", 10), []); -}); - -test("non-freeze stderr and freeze_duration diagnostics do not create intervals", () => { - assert.deepEqual(parseFreezeDetect("", 10), []); - assert.deepEqual(parseFreezeDetect("frame=250 time=00:00:10.00\n[freezedetect] lavfi.freezedetect.freeze_duration: 2\n", 10), []); -}); - -test("invalid or unordered freezedetect timestamps fail rather than silently over-compress", () => { - const log = events => events.map(([kind, at]) => `[freezedetect] lavfi.freezedetect.freeze_${kind}: ${at}\n`).join(""); - for (const events of [ - [["start", "NaN"]], [["start", "Infinity"]], [["start", ""]], [["start", -1]], [["start", 11]], - [["end", 4]], [["start", 2], ["start", 3]], [["start", 2], ["end", 1]], - [["start", 2], ["end", 2]], [["start", 2], ["end", 4], ["start", 3]], - ]) assert.throws(() => parseFreezeDetect(log(events), 10)); - for (const duration of [0, -1, NaN, Infinity]) assert.throws(() => parseFreezeDetect("", duration)); - // Permit the same small container/frame rounding tolerance as timeline waits. - assert.deepEqual(parseFreezeDetect(log([["start", 2], ["end", 10.1]]), 10), [{ start: 2, end: 10 }]); -}); - -test("freezes merge with waits before overlapping holds are subtracted, without mutating inputs", () => { - const waits = Object.freeze([ - Object.freeze({ kind: "hold", start: 20, end: 28 }), - Object.freeze({ kind: "model_thinking", start: 4, end: 8 }), - Object.freeze({ kind: "hold", start: 12, end: 16 }), - Object.freeze({ kind: "hold", start: 10, end: 14 }), - ]); - const freezes = Object.freeze([Object.freeze({ start: 7, end: 30 }), Object.freeze({ start: 0, end: 5 })]); - assert.deepEqual(clipSegments(waits, 32, 2, freezes), [ - { start: 2, end: 4, seconds: 2 }, // Typing survives a detected freeze. - { start: 4, end: 10, seconds: 0.6 }, // One merged wait/freeze, not two budgets. - { start: 10, end: 16, seconds: 6 }, - { start: 16, end: 20, seconds: 0.6 }, - { start: 20, end: 28, seconds: 8 }, - { start: 28, end: 30, seconds: 0.6 }, - { start: 30, end: 32, seconds: 2 }, - ]); -}); - -test("a freeze spanning typing, waits, scroll and final dwell cannot shorten protected time", () => { - assert.deepEqual(clipSegments([ - { kind: "model_thinking", start: 5, end: 10 }, - { kind: "hold", start: 18, end: 30 }, - ], 30, 2, [{ start: 0, end: 30 }]), [ - { start: 2, end: 5, seconds: 3 }, - { start: 5, end: 18, seconds: 0.6 }, - { start: 18, end: 30, seconds: 12 }, - ]); - // Without a post-typing wait boundary, keep the opening conservatively. - assert.deepEqual(clipSegments([{ kind: "hold", start: 10, end: 20 }], 20, 0, [{ start: 0, end: 20 }]), [ - { start: 0, end: 20, seconds: 20 }, - ]); -}); - -test("explicit typing takes priority over freezes and waits, and short remainders keep real speed", () => { - assert.deepEqual(clipSegments([ - { kind: "typing", start: 2, end: 5 }, - { kind: "tool_call", start: 1, end: 10 }, - { kind: "hold", start: 10.2, end: 12 }, - ], 12, 0, [{ start: 8, end: 10.2 }]), [ - { start: 0, end: 1, seconds: 1 }, - { start: 1, end: 2, seconds: 0.6 }, - { start: 2, end: 5, seconds: 3 }, - { start: 5, end: 10.2, seconds: 0.6 }, - { start: 10.2, end: 12, seconds: 12 - 10.2 }, - ]); - const segments = clipSegments([ - { kind: "typing", start: 0, end: 3 }, { kind: "hold", start: 3.4, end: 10 }, - ], 10, 0, [{ start: 0, end: 10 }]); - assert.equal(segments[1].seconds, 3.4 - 3); -}); - -test("frozen intervals validate and clip at raw video and retained start boundaries", () => { - for (const freezes of [null, [{ start: -1, end: 3 }], [{ start: 2, end: 1 }], [{ start: 0, end: Infinity }], [{ start: 5, end: 11 }]]) { - assert.throws(() => clipSegments([], 10, 0, freezes)); - } - assert.deepEqual(clipSegments([{ kind: "model_thinking", start: 1, end: 2 }], 10, 4, [ - { start: 0, end: 6 }, { start: 8, end: 10.1 }, - ]), [ - { start: 4, end: 6, seconds: 0.6 }, - { start: 6, end: 8, seconds: 2 }, - { start: 8, end: 10, seconds: 0.6 }, - ]); -}); diff --git a/site/scripts/encode-clips.mjs b/site/scripts/encode-clips.mjs deleted file mode 100644 index 060c8407b..000000000 --- a/site/scripts/encode-clips.mjs +++ /dev/null @@ -1,62 +0,0 @@ -import { spawnSync } from "node:child_process"; -import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; -import { mkdir } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { basename, join, resolve } from "node:path"; -import { fileURLToPath } from "node:url"; -import { clipFilter, clipSegments, freezeDetectFilter, parseFreezeDetect } from "./clip-timeline.mjs"; - -const input = resolve(process.argv[2] ?? fileURLToPath(new URL("../../ui/host/test-results/landing-recordings/", import.meta.url))); -const output = resolve(process.argv[3] ?? fileURLToPath(new URL("../src/assets/landing/", import.meta.url))); -const temporary = mkdtempSync(join(tmpdir(), "fanout-clips-")); -function run(command, args, capture = false) { - const result = spawnSync(command, args, { encoding: "utf8", maxBuffer: 16 * 1024 * 1024, - stdio: capture === "stderr" ? ["ignore", "ignore", "pipe"] : capture ? ["ignore", "pipe", "inherit"] : "inherit" }); - if (result.error) throw result.error; - if (result.status !== 0) throw new Error(`${command} exited with status ${result.status}${capture === "stderr" ? `\n${result.stderr}` : ""}`); - return capture === "stderr" ? result.stderr : result.stdout; -} - -try { - await mkdir(output, { recursive: true }); - // Encode every recording present, so a single retake needs no other takes. - const names = ["chat", "dashboard"].flatMap(view => ["light", "dark"].map(theme => `${view}-${theme}`)) - .filter(name => existsSync(join(input, `${name}.json`))); - if (!names.length) throw new Error(`No recordings found in ${input}`); - for (const name of names) { - const timeline = JSON.parse(readFileSync(join(input, `${name}.json`), "utf8")); - if (timeline.version !== 1 || timeline.raw_file !== `${name}.webm` || basename(timeline.raw_file) !== timeline.raw_file) { - throw new Error(`Invalid recording timeline for ${name}`); - } - const raw = join(input, timeline.raw_file); - const probe = JSON.parse(run("ffprobe", ["-v", "error", "-show_entries", "format=duration:stream=width,height", "-of", "json", raw], true)); - if (!probe.streams.some(stream => stream.width === 1280 && stream.height === 800)) throw new Error(`${name} must be recorded at 1280 × 800`); - const duration = Number(probe.format.duration); - // Decode the raw video, before trimming, so detection and timeline use the - // same clock. Info-level stderr carries freeze_start/end (including EOF). - const detection = run("ffmpeg", ["-hide_banner", "-nostdin", "-nostats", "-loglevel", "info", "-i", raw, - "-map", "0:v:0", "-an", "-vf", freezeDetectFilter, "-f", "null", "-"], "stderr"); - const freezes = parseFreezeDetect(detection, duration); - const segments = clipSegments(timeline.waits, duration, timeline.start, freezes); - const seconds = segments.reduce((sum, segment) => sum + segment.seconds, 0); - // Reserve 5% of a decimal 4 MB target for muxing; two passes share this budget. - const bitrate = String(Math.floor(4_000_000 * 8 * 0.95 / seconds)); - const filter = clipFilter(segments); - for (const format of ["webm", "mp4"]) { - const passlog = join(temporary, `${name}-${format}`); - const codec = format === "webm" - ? ["-c:v", "libvpx-vp9", "-deadline", "good", "-cpu-used", "2", "-row-mt", "1"] - : ["-c:v", "libx264", "-preset", "slow"]; - const args = ["-hide_banner", "-loglevel", "warning", "-y", "-i", raw, - "-filter_complex", filter, "-map", "[out]", "-an", ...codec, "-b:v", bitrate, "-passlogfile", passlog]; - run("ffmpeg", [...args, "-pass", "1", "-f", "null", "-"]); - run("ffmpeg", [...args, "-pass", "2", ...(format === "mp4" ? ["-movflags", "+faststart"] : []), join(output, `${name}.${format}`)]); - } - // Reverse only the final second so the PNG is the last encoded frame. - run("ffmpeg", ["-hide_banner", "-loglevel", "warning", "-y", "-sseof", "-1", "-i", join(output, `${name}.mp4`), - "-vf", "reverse", "-frames:v", "1", "-update", "1", join(output, `${name}.png`)]); - console.log(`${name}: ${seconds.toFixed(1)} s, ${freezes.length} frozen intervals, 1280 × 800, 30 fps, approximately 4 MB per format`); - } -} finally { - rmSync(temporary, { recursive: true, force: true }); -} diff --git a/site/scripts/encode-clips.sh b/site/scripts/encode-clips.sh deleted file mode 100644 index a948f0f6d..000000000 --- a/site/scripts/encode-clips.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -node "$(dirname "$0")/encode-clips.mjs" "$@" diff --git a/site/scripts/landing-clips.md b/site/scripts/landing-clips.md new file mode 100644 index 000000000..cb2ae0b21 --- /dev/null +++ b/site/scripts/landing-clips.md @@ -0,0 +1,37 @@ +# Landing clips + +The landing page's chat and dashboard slots play short clips of the real +Fanout UI. They are produced outside this repository from screen captures of a +Fanout instance with chat enabled; answers, panels and numbers are as recorded. + +Expected assets, each at 16:10: + +- `chat-{light,dark}.{webm,mp4,png}` +- `dashboard-{light,dark}.{webm,mp4,png}` + +Keep each video at about 4 MB. The PNG poster is the clip's final frame. + +Videos are never committed. Published videos live in the Cloudflare R2 bucket +`fanout` under `videos/`, served at `https://assets.fanout.run/videos/`. The +bucket holds public assets only, in one folder per file type. PNG posters stay +in `site/src/assets/landing/` so Astro can optimize them. + +Each slot stays poster-only until both themes have both video formats in +`site/src/data/landing-media.json`. An absent manifest also shows posters. The +component follows the site theme, pauses off screen and in hidden tabs, and +shows only the poster when reduced motion is enabled. Its `spedUp` prop marks +clips with shortened waits in the caption. + +Copy the reviewed posters into `site/src/assets/landing/`, then upload the +videos from the repository root with Wrangler logged in to the account that +owns `fanout`: + +```sh +bash site/scripts/upload-clips.sh /path/to/clips +``` + +The uploader writes content-addressed immutable keys, skips unchanged clips +already listed in the manifest, and preserves URLs for clips absent from the +input directory. It never deletes old objects because published pages may still +reference them. After uploading, commit `site/src/data/landing-media.json` and +the PNG posters. diff --git a/site/scripts/landing-recording-scroll.test.mjs b/site/scripts/landing-recording-scroll.test.mjs deleted file mode 100644 index f6bf12d4c..000000000 --- a/site/scripts/landing-recording-scroll.test.mjs +++ /dev/null @@ -1,119 +0,0 @@ -import { test } from "node:test"; -import assert from "node:assert/strict"; -import { runInNewContext } from "node:vm"; -import { revealRecording, waitForConversation } from "../../ui/host/e2e/support/landing-recording-support.ts"; - -// Execute the actual page.evaluate callbacks with a virtual layout and clock. -// No browser is launched; iframe growth is represented by moving the answer. -function recordingPage({ view = "chat", answerReady = true } = {}) { - let now = 0, nextId = 0, scrollTop = 0, answerBottom = 1_300; - const pending = new Map(), observers = new Set(), positions = []; - const schedule = (callback, milliseconds) => { - const id = ++nextId; - pending.set(id, { at: now + milliseconds, callback }); - return id; - }; - class Observer { - constructor(callback) { this.callback = callback; this.targets = new Set(); observers.add(this); } - observe(target) { this.targets.add(target); } - disconnect() { observers.delete(this); } - } - const answer = { getBoundingClientRect: () => ({ bottom: 100 + answerBottom - scrollTop }) }; - const markdown = { textContent: "The final answer." }; - const content = {}; - const scroller = { - clientHeight: 500, clientTop: 0, firstElementChild: content, - get scrollHeight() { return view === "chat" ? answerBottom + 24 : 1_400; }, - get scrollTop() { return scrollTop; }, - set scrollTop(value) { - scrollTop = Math.max(0, Math.min(value, this.scrollHeight - this.clientHeight)); - positions.push({ at: now, top: scrollTop }); - }, - getBoundingClientRect: () => ({ top: 100 }), - querySelector: selector => selector === "[data-answer-position]" ? answer : answerReady ? markdown : null, - querySelectorAll: () => [answer], - dispatchEvent() {}, - }; - const environment = { - document: { querySelector: () => scroller, scrollingElement: scroller, fonts: { ready: Promise.resolve() } }, - performance: { now: () => now }, - requestAnimationFrame: callback => schedule(() => callback(now), 16), - cancelAnimationFrame: id => pending.delete(id), - setTimeout: schedule, clearTimeout: id => pending.delete(id), - ResizeObserver: Observer, MutationObserver: Observer, WheelEvent: class {}, - }; - const page = { - async evaluate(callback, argument) { - let done = false, error; - const result = runInNewContext(`(${callback.toString()})(argument)`, { ...environment, argument }); - result.then(() => { done = true; }, reason => { done = true; error = reason; }); - for (let steps = 0; !done && steps < 10_000; steps++) { - // Drain async continuations before advancing the simulated animation clock. - await new Promise(resolve => setImmediate(resolve)); - if (done) break; - const next = [...pending].sort((a, b) => a[1].at - b[1].at)[0]; - assert.ok(next, "evaluate must resolve or schedule more work"); - pending.delete(next[0]); - now = next[1].at; - next[1].callback(); - } - assert.ok(done, "evaluate must finish within the virtual clock budget"); - if (error) throw error; - return result; - }, - }; - return { - page, positions, observers, pending, - get now() { return now; }, - get expectedEnd() { return answerBottom - scroller.clientHeight; }, - get top() { return scrollTop; }, - resizeAt(at, bottom) { - schedule(() => { - answerBottom = bottom; - for (const observer of observers) observer.callback(); - }, at); - }, - renderAnswerAt(at) { schedule(() => { answerReady = true; }, at); }, - }; -} - -test("chat retargets late evidence growth during the scroll and holds the answer end for 4 seconds", async () => { - const recording = recordingPage(); - recording.resizeAt(3_000, 2_400); - recording.resizeAt(9_500, 3_100); // Growth during the final dwell must also stay aligned. - await revealRecording(recording.page, "chat"); - assert.equal(recording.top, recording.expectedEnd); - assert.ok(recording.positions.some(position => position.at > 8_100 && position.at < 9_500 && position.top === 1_900)); - assert.ok(recording.positions.filter(position => position.at >= 9_520).every(position => position.top === 2_600)); - // Initial distance sets an 8-second scroll; growth must not restart its duration. - assert.ok(recording.now >= 12_000 && recording.now < 12_150); -}); - -test("dashboard keeps the down/bottom/up timings and dwells at the top for only 3 seconds", async () => { - const recording = recordingPage({ view: "dashboard" }); - await revealRecording(recording.page, "dashboard"); - assert.equal(recording.top, 0); - assert.equal(Math.max(...recording.positions.map(position => position.top)), 900); - assert.ok(recording.positions.some(position => position.top > 0 && position.top < 900 && position.at > 7_500)); - assert.ok(recording.now >= 12_000 && recording.now < 12_150); - assert.equal(recording.now - recording.positions.at(-1).at, 3_000); -}); - -test("conversation readiness waits for rendered text and 1.5 seconds after the last resize", async () => { - const recording = recordingPage({ answerReady: false }); - recording.renderAnswerAt(1_500); - recording.resizeAt(1_000, 2_000); - recording.resizeAt(2_000, 2_400); - await waitForConversation(recording.page); - assert.ok(recording.now >= 3_500 && recording.now < 3_550); - assert.equal(recording.observers.size, 0); - assert.equal(recording.pending.size, 0); -}); - -test("conversation readiness fails instead of recording an answer that never renders", async () => { - const recording = recordingPage({ answerReady: false }); - await assert.rejects(waitForConversation(recording.page), /Conversation did not stop resizing/); - assert.equal(recording.now, 30_000); - assert.equal(recording.observers.size, 0); - assert.equal(recording.pending.size, 0); -}); diff --git a/site/scripts/landing-recordings.md b/site/scripts/landing-recordings.md deleted file mode 100644 index 54a2c7ce6..000000000 --- a/site/scripts/landing-recordings.md +++ /dev/null @@ -1,129 +0,0 @@ -# Landing recordings - -Record the chat and dashboard slots from an OpenTelemetry Demo instance with -chat enabled. - -Expected assets (all at 1280 × 800): - -- `site/src/assets/landing/chat-{light,dark}.{webm,mp4,png}` -- `site/src/assets/landing/dashboard-{light,dark}.{webm,mp4,png}` - -Encoded videos stay local and are ignored by Git and Docker; PNG posters stay -in the repository so Astro can optimize them. Published videos live in the -Cloudflare R2 bucket `fanout` under `videos/`, served at -`https://assets.fanout.run/videos/`. The bucket holds public assets only, in one -folder per file type. - -Each slot stays poster-only until both themes have both video formats in -`site/src/data/landing-media.json`. An absent manifest also shows posters. The -component follows the site theme, pauses off screen and in hidden tabs, and -shows only the poster when reduced motion is enabled. Its `spedUp` prop marks -recordings with shortened waits in the caption. - -Set `FANOUT_LANDING_RECORDINGS=1`, `FANOUT_LANDING_BASE_URL` to the existing -instance URL, and `FANOUT_LANDING_SESSION_COOKIE` to a `name=value` session cookie -in the process environment. Run from `ui/host`: - -```sh -bun run e2e e2e/specs/app/landing-recordings.spec.ts --project=app-1440-light --project=app-1440-dark -``` - -Recordings use a 1280 × 800 viewport and raw video at the same size; Playwright -video captures CSS pixels, so a higher device scale only pads the frame. This -only uses the external instance. It types the configured prompts and waits for the -real run and every evidence iframe to load and finish rendering. A take fails -when the run shows a failed step, when a chat run has fewer than four evidence -views or any view without rows, or when the saved dashboard has fewer than six -panels or any non-text panel without rows. Before the -reveal, a ResizeObserver waits for 1.5 seconds without conversation size changes, -including individual evidence and answer blocks, with the final answer rendered. -This settling interval is recorded as a compressible `conversation_ready` wait. -The chat clip then returns the -conversation pane to the user's question and scrolls through the evidence to -the end of the answer over about 8 seconds, followed by a 4-second final dwell. -It measures the `[data-answer-position]` end on every animation frame and keeps -it aligned to the bottom of the conversation viewport throughout the dwell, -retargeting if an evidence view grows. The take fails unless the answer end is -bottom-aligned and its last text line is inside the viewport at the end. -The dashboard clip opens the dashboard from its successful `create_dashboard` -result and records a compressible `dashboard_ready` wait from just before the -Open dashboard click until every saved panel has rendered and the first fold -has settled again. This includes navigation, loading spinners, lazy panel -loading and the setup scroll back to the top. It then scrolls from the top -through all panels to the bottom over about 6 seconds, dwells there for 1.5 -seconds, and scrolls back to the top over 1.5 seconds before its 3-second final -dwell. The dashboard poster therefore shows the top of the dashboard. - -Scrolls use awaited `requestAnimationFrame` animation with brief easing at the -ends and a steady pace through the middle. Downward durations scale with the -scroll distance: two viewport heights take the baseline 8 or 6 seconds, clamped -between that baseline and 12 seconds. Aim for encoded clips of 20–35 seconds; -review the actual duration after encoding, since real answer and panel content -vary. It creates real chats and dashboards; -no data is mocked or intercepted. - -Use a dedicated recording account: before each take, the spec deletes all of -that account’s chats and non-default dashboards. - -Raw WebM files and matching JSON timelines go to -`ui/host/test-results/landing-recordings/`, excluded from Git and Docker inputs. -Keep a copy before another Playwright run clears `test-results`. Timelines use -seconds from page creation, with server event timestamps anchored to receipt of -the run's response headers to avoid server clock skew. They list model-thinking -and tool-call intervals from the unmodified AG-UI stream. The `waits` array also -includes readiness waits and a `kind: "hold"` interval covering the entire result -scroll and final dwell, including the dashboard's bottom dwell and return to the -top. Timelines contain no session -credentials or tool result payloads. Raw recordings can contain instance data. - -From the repository root, with ffmpeg and ffprobe installed: - -```sh -bash site/scripts/encode-clips.sh -# Or use a saved raw output directory: -bash site/scripts/encode-clips.sh /path/to/landing-recordings -``` - -It encodes every recording present in the input directory. An optional second -directory argument writes encoded assets elsewhere for review. - -Before building segments, the encoder decodes the full raw video through -ffmpeg's `freezedetect=n=0.003:d=1.2`. The normalized full-frame noise tolerance -of `0.003` (about −50.5 dB on ffmpeg's scale) ignores small cursor blinks, -typing-caret flicker, and small streamed text or build-stage updates; the -minimum duration of 1.2 s catches near-static stretches without shortening -every brief pause. Detection runs at the raw frame rate and resolution, with -no resampling or downscaling. Its stderr timestamps share the raw timeline's -clock; an unfinished freeze at EOF ends at the ffprobe duration. - -The encoder merges overlapping or touching frozen intervals and timeline waits, -then subtracts protected intervals. Every remaining portion longer than 0.6 s -becomes exactly 0.6 s; shorter portions keep their original duration. Holds -preserve the entire scroll and final dwell at real speed, even if detection -marks them frozen. A hold splits a merged interval into separate compressible -portions, each capped at 0.6 s. Typing is protected from the clip start through -the first timeline wait, which starts after submission in these recordings. -An explicit `kind: "typing"` interval is also protected and replaces that -inference; without either an explicit typing interval or a wait boundary, the -opening is conservatively retained in full. These protections also override -timeline waits. The encoder reports the detected interval count and planned -duration for each clip. - -It writes VP9 WebM and H.264 -MP4 at 1280 × 800 and 30 fps with a two-pass target of about 4 MB per file. -It extracts each PNG poster from the final encoded frame. Review the clips and -posters, then upload from the repository root with Wrangler logged in to the -account that owns `fanout`: - -```sh -bash site/scripts/upload-clips.sh -# Or upload an encoded directory used for review: -bash site/scripts/upload-clips.sh /path/to/encoded-clips -``` - -The uploader writes content-addressed immutable keys, skips unchanged clips -already listed in the manifest, and preserves URLs for clips absent from the -input directory. It never deletes old objects because published pages may still -reference them. After uploading, commit `site/src/data/landing-media.json` and -the PNG posters. If encoding elsewhere, copy the reviewed PNGs into -`site/src/assets/landing/` first. Never commit WebM or MP4 files. diff --git a/site/scripts/landing-recordings.test.mjs b/site/scripts/landing-recordings.test.mjs deleted file mode 100644 index 097352631..000000000 --- a/site/scripts/landing-recordings.test.mjs +++ /dev/null @@ -1,136 +0,0 @@ -import { test } from "node:test"; -import assert from "node:assert/strict"; -import { recordingTimeline } from "../../ui/host/e2e/support/landing-recording-support.ts"; -import { clipSegments, clipFilter } from "./clip-timeline.mjs"; - -const stream = events => events.map(event => `data: ${JSON.stringify(event)}\n\n`).join(""); -const event = (type, at, data = {}) => ({ type, timestamp: 1_000_000 + at * 1000, ...data }); - -test("recording times survive server clock skew and preserve the answer text", () => { - const recorded = recordingTimeline(stream([ - event("RUN_STARTED", 0), event("TOOL_CALL_START", 6, { toolCallId: "q", toolCallName: "query_telemetry" }), - event("TOOL_CALL_END", 6.1, { toolCallId: "q" }), // Arguments ended; the tool is still running. - event("TOOL_CALL_RESULT", 9, { toolCallId: "q", content: "{}" }), - event("TEXT_MESSAGE_START", 12), event("TEXT_MESSAGE_CONTENT", 13), event("RUN_FINISHED", 14), - ]), 10); - assert.deepEqual(recorded.waits, [ - { kind: "model_thinking", start: 10, end: 16 }, - { kind: "tool_call", start: 16, end: 19, tool: "query_telemetry" }, - { kind: "model_thinking", start: 19, end: 22 }, - ]); -}); - -test("concurrent tools remain distinct and only a successful create receipt identifies the new dashboard", () => { - const created = { dashboard: { id: "new-dashboard" }, receipt: { version: 1 } }; - for (const result of [created, { ...created, error: "failed" }, { ...created, isError: true }, { dashboard: created.dashboard }]) { - const timeline = recordingTimeline(stream([ - event("RUN_STARTED", 0), - event("TOOL_CALL_START", 1, { toolCallId: "a", toolCallName: "create_dashboard" }), - event("TOOL_CALL_START", 2, { toolCallId: "b", toolCallName: "search_logs" }), - event("TOOL_CALL_RESULT", 3, { toolCallId: "a", content: JSON.stringify(result) }), - event("TOOL_CALL_RESULT", 4, { toolCallId: "b", content: "{}" }), - event("TEXT_MESSAGE_START", 5), event("RUN_FINISHED", 6), - ]).replaceAll("\n", "\r\n"), 0); - assert.equal(timeline.dashboardId, result === created ? "new-dashboard" : undefined); - assert.deepEqual(timeline.waits.filter(wait => wait.kind === "model_thinking"), [ - { kind: "model_thinking", start: 0, end: 1 }, { kind: "model_thinking", start: 4, end: 5 }, - ]); - assert.equal(timeline.waits.filter(wait => wait.kind === "tool_call").length, 2); - } -}); - -test("failed, truncated, untimed or unresolved runs cannot become successful recordings", () => { - const start = event("RUN_STARTED", 0), end = event("RUN_FINISHED", 10); - for (const events of [[], [start], [start, event("RUN_ERROR", 1), end], - [start, { type: "TEXT_MESSAGE_START" }, end], - [start, event("TOOL_CALL_START", 1, { toolCallId: "a", toolCallName: "query_telemetry" }), end], - [start, event("TOOL_CALL_RESULT", 1, { toolCallId: "a" }), end]]) { - assert.throws(() => recordingTimeline(stream(events), 0)); - } -}); - -test("encoding merges overlapping waits, keeps short waits and preserves typing and dwell", () => { - const segments = clipSegments([{ start: 5, end: 12 }, { start: 4, end: 8 }, { start: 15, end: 15.2 }], 20); - assert.deepEqual(segments, [ - { start: 0, end: 4, seconds: 4 }, { start: 4, end: 12, seconds: 0.6 }, - { start: 12, end: 15, seconds: 3 }, { start: 15, end: 15.2, seconds: 15.2 - 15 }, - { start: 15.2, end: 20, seconds: 20 - 15.2 }, - ]); - const filter = clipFilter(segments); - assert.match(filter, /trim=start=4:end=12,setpts=\(PTS-STARTPTS\)\*0.075\[v1\]/); - assert.match(filter, /concat=n=5:v=1:a=0,scale=1280:800,fps=30,format=yuv420p\[out\]/); - assert.deepEqual(clipSegments([], 3), [{ start: 0, end: 3, seconds: 3 }]); -}); - -test("invalid trim times fail before invoking ffmpeg", () => { - for (const waits of [[{ start: -1, end: 2 }], [{ start: 3, end: 2 }], [{ start: 0, end: Infinity }], [{ start: 1, end: 11 }], null]) { - assert.throws(() => clipSegments(waits, 10)); - } - assert.throws(() => clipSegments([], 0)); -}); - -test("encoding preserves overlapping holds at real speed inside merged waits", () => { - const segments = clipSegments([ - { kind: "tool_call", start: 2, end: 10 }, - { kind: "model_thinking", start: 8, end: 18 }, - { kind: "hold", start: 6, end: 12 }, - { kind: "hold", start: 4, end: 9 }, - ], 20); - assert.deepEqual(segments, [ - { start: 0, end: 2, seconds: 2 }, - { start: 2, end: 4, seconds: 0.6 }, - { start: 4, end: 12, seconds: 8 }, - { start: 12, end: 18, seconds: 0.6 }, - { start: 18, end: 20, seconds: 2 }, - ]); - assert.match(clipFilter(segments), /trim=start=4:end=12,setpts=\(PTS-STARTPTS\)\*1\[v2\]/); -}); - -test("holds can cover entire waits and are clipped at the retained video boundaries", () => { - assert.deepEqual(clipSegments([ - { kind: "hold", start: 1, end: 10.1 }, - { kind: "tool_call", start: 2, end: 8 }, - ], 10, 4), [{ start: 4, end: 10, seconds: 6 }]); - assert.deepEqual(clipSegments([{ kind: "hold", start: 2, end: 6 }], 10), [ - { start: 0, end: 10, seconds: 10 }, - ]); - assert.throws(() => clipSegments([{ kind: "hold", start: 2, end: 1 }], 10)); -}); - -test("encoding cuts frames before the clip start", () => { - const segments = clipSegments([{ start: 1, end: 3 }, { start: 5, end: 8 }], 12, 4); - assert.deepEqual(segments, [ - { start: 4, end: 5, seconds: 1 }, - { start: 5, end: 8, seconds: 0.6 }, - { start: 8, end: 12, seconds: 4 }, - ]); - assert.throws(() => clipSegments([], 12, 12), /Clip start must be within the raw video/); -}); - -test("dashboard navigation and panel loading compress while the scroll and 3-second final dwell stay real time", () => { - const segments = clipSegments([ - { kind: "model_thinking", start: 9, end: 32 }, - { kind: "conversation_ready", start: 32, end: 36 }, - // Includes clicking Open dashboard, navigation, lazy panels, and return to top. - { kind: "dashboard_ready", start: 36, end: 52 }, - { kind: "hold", start: 52, end: 70 }, // 12 down + 1.5 bottom + 1.5 up + 3 final. - ], 70, 2); - assert.deepEqual(segments, [ - { start: 2, end: 9, seconds: 7 }, - { start: 9, end: 52, seconds: 0.6 }, - { start: 52, end: 70, seconds: 18 }, - ]); - const duration = segments.reduce((sum, segment) => sum + segment.seconds, 0); - assert.ok(duration >= 20 && duration <= 35); -}); - -test("conversation layout settling compresses without shortening the answer scroll or 4-second dwell", () => { - assert.deepEqual(clipSegments([ - { kind: "conversation_ready", start: 10, end: 18 }, - { kind: "hold", start: 18, end: 30 }, // 8-second scroll and 4-second answer dwell. - ], 30), [ - { start: 0, end: 10, seconds: 10 }, - { start: 10, end: 18, seconds: 0.6 }, - { start: 18, end: 30, seconds: 12 }, - ]); -}); diff --git a/site/src/assets/landing/chat-dark.png b/site/src/assets/landing/chat-dark.png index 07a5b2f62..1e7175ec9 100644 Binary files a/site/src/assets/landing/chat-dark.png and b/site/src/assets/landing/chat-dark.png differ diff --git a/site/src/assets/landing/chat-light.png b/site/src/assets/landing/chat-light.png index c049d30be..56afab212 100644 Binary files a/site/src/assets/landing/chat-light.png and b/site/src/assets/landing/chat-light.png differ diff --git a/site/src/assets/landing/dashboard-dark.png b/site/src/assets/landing/dashboard-dark.png index d416ca6f7..00e172f5d 100644 Binary files a/site/src/assets/landing/dashboard-dark.png and b/site/src/assets/landing/dashboard-dark.png differ diff --git a/site/src/assets/landing/dashboard-light.png b/site/src/assets/landing/dashboard-light.png index 0f844e44f..0c2bc5e8d 100644 Binary files a/site/src/assets/landing/dashboard-light.png and b/site/src/assets/landing/dashboard-light.png differ diff --git a/site/src/content/docs/index.mdx b/site/src/content/docs/index.mdx index d52ed57e3..e05b730bd 100644 --- a/site/src/content/docs/index.mdx +++ b/site/src/content/docs/index.mdx @@ -74,7 +74,7 @@ Ask for checkout's headline stats and trends, p95 latency by operation, slowest The agent queries traces, logs and metrics, then explains what the evidence shows. - + diff --git a/site/src/data/landing-media.json b/site/src/data/landing-media.json index 8c3b4b02b..cc42f714c 100644 --- a/site/src/data/landing-media.json +++ b/site/src/data/landing-media.json @@ -1,18 +1,18 @@ { "chat-dark": { - "mp4": "https://assets.fanout.run/videos/chat-dark-df7f6aeae8ae.mp4", - "webm": "https://assets.fanout.run/videos/chat-dark-a40b083a1847.webm" + "mp4": "https://assets.fanout.run/videos/chat-dark-e22d7ae9d9cd.mp4", + "webm": "https://assets.fanout.run/videos/chat-dark-38f2d036fdd7.webm" }, "chat-light": { - "mp4": "https://assets.fanout.run/videos/chat-light-aed5fb359884.mp4", - "webm": "https://assets.fanout.run/videos/chat-light-4d3bcc09f5de.webm" + "mp4": "https://assets.fanout.run/videos/chat-light-aa8aea8a2e9a.mp4", + "webm": "https://assets.fanout.run/videos/chat-light-ca1a83d622c9.webm" }, "dashboard-dark": { - "mp4": "https://assets.fanout.run/videos/dashboard-dark-106bbe5a437d.mp4", - "webm": "https://assets.fanout.run/videos/dashboard-dark-5c4580915a9e.webm" + "mp4": "https://assets.fanout.run/videos/dashboard-dark-31a3d43c13d4.mp4", + "webm": "https://assets.fanout.run/videos/dashboard-dark-8b978f47cf1c.webm" }, "dashboard-light": { - "mp4": "https://assets.fanout.run/videos/dashboard-light-096a6eaf1685.mp4", - "webm": "https://assets.fanout.run/videos/dashboard-light-3c5456d88c86.webm" + "mp4": "https://assets.fanout.run/videos/dashboard-light-d1de210204c2.mp4", + "webm": "https://assets.fanout.run/videos/dashboard-light-b736620ad810.webm" } } diff --git a/ui/host/e2e/specs/app/landing-recordings.spec.ts b/ui/host/e2e/specs/app/landing-recordings.spec.ts deleted file mode 100644 index 485907b30..000000000 --- a/ui/host/e2e/specs/app/landing-recordings.spec.ts +++ /dev/null @@ -1,196 +0,0 @@ -import { expect, test } from "../../fixtures/test"; -import { mkdir, writeFile } from "node:fs/promises"; -import { fileURLToPath } from "node:url"; -import { cardSelector, readyCard, settle } from "../../support/accessibility-support"; -import { recordingTimeline, revealRecording, waitForConversation } from "../../support/landing-recording-support"; -import type { DashboardSpec, PanelResult } from "../../../../panels/types"; - -const baseURL = process.env.FANOUT_LANDING_BASE_URL; -const directory = fileURLToPath(new URL("../../../test-results/landing-recordings/", import.meta.url)); -test.skip(process.env.FANOUT_LANDING_RECORDINGS !== "1", "set FANOUT_LANDING_RECORDINGS=1 to record real landing clips"); -test.setTimeout(8 * 60_000); -test.use({ - storageState: async ({}, use) => { - if (!baseURL) throw new Error("Landing recordings require FANOUT_LANDING_BASE_URL for an external instance"); - const match = /^([!#$%&'*+.^_`|~\w-]+)=([^;\s]+)$/.exec(process.env.FANOUT_LANDING_SESSION_COOKIE ?? ""); - if (!match) throw new Error("FANOUT_LANDING_SESSION_COOKIE must be a nonempty name=value session cookie"); - const url = new URL(baseURL); - await use({ cookies: [{ name: match[1], value: match[2], domain: url.hostname, path: "/", expires: -1, - httpOnly: true, secure: url.protocol === "https:", sameSite: "Lax" }], origins: [] }); - }, -}); - -const prompts = { - chat: "Why is checkout slower in the last hour?", - dashboard: "Build a dashboard for checkout: headline stats and trends for request rate, error rate and p95 latency, p95 by operation, slowest dependencies, and recent slow traces", -}; - -for (const view of ["chat", "dashboard"] as const) { - test(`record ${view} on the OpenTelemetry Demo`, async ({ browser, storageState }, info) => { - test.skip(info.project.use.viewport?.width !== 1440, "use the 1440px projects to record each theme once at 1280 × 800 CSS pixels"); - const theme = info.project.use.colorScheme as "light" | "dark"; - await mkdir(directory, { recursive: true }); - const context = await browser.newContext({ baseURL, storageState, colorScheme: theme, - viewport: { width: 1280, height: 800 }, - recordVideo: { dir: directory, size: { width: 1280, height: 800 } } }); - try { - const status = await context.request.get("/api/auth/status"); - expect(status.ok(), "Cannot read runtime status from the external instance").toBe(true); - if (!(await status.json() as { agent_available: boolean }).agent_available) { - throw new Error("Chat is disabled on the external instance; configure an AI provider before recording landing clips"); - } - await context.addInitScript(value => localStorage.setItem("mantine-color-scheme-value", value), theme); - const startedAt = Date.now(); - const page = await context.newPage(); - const video = page.video()!; - await page.goto("/chat"); - // Use this page's session and the owner-scoped APIs to clear earlier takes. - await page.evaluate(async () => { - const request = async (path: string, method = "GET", headers: Record = {}) => { - const response = await fetch(path, { method, credentials: "same-origin", redirect: "error", - headers: { "Fanout-Request": "1", ...headers } }).catch(() => { - throw new Error(`Landing recording cleanup failed: ${method} ${path} (request could not complete)`); - }); - if (!response.ok) throw new Error(`Landing recording cleanup failed: ${method} ${path} (HTTP ${response.status})`); - return response; - }; - // Finish pagination before deleting so removals cannot affect the scan. - const threadIds: string[] = []; - let cursor: string | null = null; - do { - const params = new URLSearchParams({ page_size: "100" }); - if (cursor) params.set("cursor", cursor); - const response = await request(`/api/agent/threads?${params}`); - const page = await response.json() as { items: { id: string }[]; next_cursor: string | null }; - threadIds.push(...page.items.map(thread => thread.id)); - cursor = page.next_cursor; - } while (cursor); - for (const id of threadIds) await request(`/api/agent/threads/${encodeURIComponent(id)}`, "DELETE"); - - const response = await request("/api/dashboards"); - const { dashboards } = await response.json() as { dashboards: { id: string; is_default: boolean }[] }; - for (const dashboard of dashboards) { - // Only an explicit non-default flag permits deletion; never touch the default. - if (dashboard.is_default !== false) continue; - await request(`/api/dashboards/${encodeURIComponent(dashboard.id)}`, "DELETE", - { "Fanout-Confirm-Delete": dashboard.id }); - } - }); - await page.reload(); - const navigation = page.getByRole("navigation", { name: "Navigation", exact: true }); - await expect(navigation.getByText("No chats yet", { exact: true })).toBeVisible(); - await expect(navigation.locator('a[href^="/dashboards/"]')).toHaveCount(1); - await expect(navigation.locator('a[href^="/dashboards/"]').getByText("System overview", { exact: true })).toBeVisible(); - await expect(page.locator("html")).toHaveAttribute("data-mantine-color-scheme", theme); - const input = page.getByRole("textbox", { name: "Message Fanout", exact: true }); - await expect(input, "Chat must be enabled and the session must allow agent runs").toBeEnabled(); - // The clip starts on the settled empty chat, not on page load and cleanup. - const start = (Date.now() - startedAt) / 1000; - await input.click(); - // A steady human typing cadence, including a short pause at punctuation. - for (const character of prompts[view]) { - await input.pressSequentially(character, { delay: /[,:?]/.test(character) ? 120 : 28 }); - } - const responseReady = page.waitForResponse(response => new URL(response.url()).pathname === "/api/agent/runs" - && response.request().method() === "POST", { timeout: 30_000 }); - await page.getByRole("button", { name: "Send message", exact: true }).click(); - const response = await responseReady; - const responseAt = (Date.now() - startedAt) / 1000; - expect(response.ok(), "Agent request failed; check chat configuration and the session's agent:run capability").toBe(true); - const stream = await response.text(); - const timeline = recordingTimeline(stream, responseAt); - // Check every evidence view, including collapsed views without a mounted iframe. - const evidence = stream.split(/\r?\n/).filter(line => line.startsWith("data:")).map(line => JSON.parse(line.slice(5)) as { - type: string; messageId?: string; activityType?: string; - content?: { tool_result?: { view: { key: string }; results: PanelResult[] } }; - }).filter(event => event.type === "ACTIVITY_SNAPSHOT" && event.activityType === "mcp-app") - .map(event => event.content!.tool_result!); - if (view === "chat") expect(new Set(evidence.map(fragment => fragment.view.key)).size, "Chat must show at least four evidence views").toBeGreaterThanOrEqual(4); - for (const fragment of evidence) { - expect(fragment.results.some(result => result.status === "ok" && (result.frame?.rows ?? 0) > 0), - "Chat must not contain an empty evidence view").toBe(true); - } - const conversationWaitStart = (Date.now() - startedAt) / 1000; - // RUN_FINISHED in the real stream plus the UI's idle state gates the reveal. - await expect(page.getByRole("button", { name: "Stop", exact: true })).toHaveCount(0); - await expect(page.locator(".chat-pane .mantine-Loader-root")).toHaveCount(0); - await expect(page.locator(".chat-pane").getByRole("alert")).toHaveCount(0); - await expect(page.locator(".chat-pane").getByText(/^\d+ steps? failed$/), "The run must not show failed steps").toHaveCount(0); - await expect(page.locator('[data-answer-position] [data-role="assistant"]')).toBeVisible(); - const views = page.locator('iframe[title="Fanout analysis view"]'); - for (let index = 0; index < await views.count(); index++) { - const frame = views.nth(index).contentFrame(); - await expect.poll(() => frame.locator("html").evaluate(element => element.ownerDocument.readyState)).toBe("complete"); - await expect(frame.locator("[data-panel-body]").first()).toBeAttached(); - await expect(frame.locator(".mantine-Loader-root, [data-panel-error]")).toHaveCount(0); - } - await settle(page); - await waitForConversation(page); - timeline.waits.push({ kind: "conversation_ready", start: conversationWaitStart, end: (Date.now() - startedAt) / 1000 }); - - if (view === "dashboard") { - if (!timeline.dashboardId) throw new Error("No successful create_dashboard result with a committed save receipt; the recording requires a newly saved dashboard"); - const receipt = page.locator(`[data-dashboard-result="${timeline.dashboardId}"]`); - const dashboardWaitStart = (Date.now() - startedAt) / 1000; - await receipt.getByRole("link", { name: "Open dashboard", exact: true }).click(); - await expect(page).toHaveURL(new RegExp(`/dashboards/${timeline.dashboardId}(?:\\?|$)`)); - const saved = await context.request.get(`/api/dashboards/${encodeURIComponent(timeline.dashboardId)}`); - expect(saved.ok()).toBe(true); - const { spec } = await saved.json() as { spec: DashboardSpec }; - expect(spec.panels.length, "The created dashboard must contain at least six panels").toBeGreaterThanOrEqual(6); - const preview = await context.request.post("/api/panels/query", { - headers: { "Fanout-Request": "1" }, data: { dashboard: spec }, - }); - expect(preview.ok()).toBe(true); - const { results } = await preview.json() as { results: PanelResult[] }; - for (const panel of spec.panels.filter(panel => panel.viz !== "text")) { - const result = results.find(result => result.id === panel.id); - expect(result?.status, `${panel.title} must return data`).toBe("ok"); - expect(result?.frame?.rows ?? 0, `${panel.title} must have rows`).toBeGreaterThan(0); - } - for (const panel of spec.panels) await readyCard(page, panel); - await page.evaluate(() => window.scrollTo(0, 0)); - // Lazy panels returning to the first fold must be settled too. - for (const panel of spec.panels) { - const card = page.locator(cardSelector(panel.id)); - await expect(card.locator('[aria-label="Loading panel"], [aria-label="Refreshing"], [data-panel-error]')).toHaveCount(0); - } - await settle(page); - timeline.waits.push({ kind: "dashboard_ready", start: dashboardWaitStart, end: (Date.now() - startedAt) / 1000 }); - } - await page.mouse.move(0, 0); - const holdStart = (Date.now() - startedAt) / 1000; - await revealRecording(page, view); - timeline.waits.push({ kind: "hold", start: holdStart, end: (Date.now() - startedAt) / 1000 }); - if (view === "chat") { - const visible = await page.locator("[data-answer-position]").evaluate(answer => { - const pane = answer.closest(".chat-scroll")!; - const top = pane.getBoundingClientRect().top + pane.clientTop; - const bottom = top + pane.clientHeight; - const markdown = answer.querySelector(".chat-markdown")!; - const walker = document.createTreeWalker(markdown, NodeFilter.SHOW_TEXT); - let last: Node | undefined; - for (let node = walker.nextNode(); node; node = walker.nextNode()) if (node.textContent?.trim()) last = node; - const range = document.createRange(); - if (last) range.selectNodeContents(last); - const line = [...range.getClientRects()].at(-1); - return { aligned: Math.abs(answer.getBoundingClientRect().bottom - bottom) <= 2, - lastLineVisible: !!line && line.top >= top && line.bottom <= bottom + 1 }; - }); - expect(visible, "The final answer end must stay at the pane bottom with its last line in view").toEqual({ aligned: true, lastLineVisible: true }); - } - const duration = (Date.now() - startedAt) / 1000; - await context.close(); - const rawFile = `${view}-${theme}.webm`; - await video.saveAs(`${directory}${rawFile}`); - await video.delete(); - // No cookies, request bodies, tool results or telemetry are persisted here. - await writeFile(`${directory}${view}-${theme}.json`, JSON.stringify({ version: 1, raw_file: rawFile, - viewport: { width: 1280, height: 800 }, theme, prompt: prompts[view], start, duration, - clock: "seconds from page creation; server event deltas anchored to response headers", - waits: timeline.waits }, null, 2) + "\n"); - } finally { - await context.close(); - } - }); -} diff --git a/ui/host/e2e/support/global-setup.ts b/ui/host/e2e/support/global-setup.ts index 6ed63da8b..06a06e92d 100644 --- a/ui/host/e2e/support/global-setup.ts +++ b/ui/host/e2e/support/global-setup.ts @@ -49,10 +49,6 @@ function seed(binary: string | undefined, base: string, tokenFile: string, log: } export default async function globalSetup({ signalCleanup = true, onSignal = () => {}, directory }: { signalCleanup?: boolean | "startup"; onSignal?: () => void; directory?: string } = {}) { - // Recordings require a real AI-enabled instance; never start the seeded server. - if (process.env.FANOUT_LANDING_RECORDINGS === "1" && !process.env.FANOUT_LANDING_BASE_URL) { - throw new Error("Landing recordings require FANOUT_LANDING_BASE_URL and FANOUT_LANDING_SESSION_COOKIE for an external instance"); - } // External landing media use an existing instance and an in-memory session. if (process.env.FANOUT_LANDING_BASE_URL) return; diff --git a/ui/host/e2e/support/landing-recording-support.ts b/ui/host/e2e/support/landing-recording-support.ts deleted file mode 100644 index ced441e3e..000000000 --- a/ui/host/e2e/support/landing-recording-support.ts +++ /dev/null @@ -1,157 +0,0 @@ -import type { Page } from "@playwright/test"; - -export type AgentEvent = { - type: string; - timestamp: number; - toolCallId?: string; - toolCallName?: string; - content?: string; -}; -export type WaitInterval = { kind: "model_thinking" | "tool_call" | "conversation_ready" | "dashboard_ready" | "hold"; start: number; end: number; tool?: string }; - -/** Iframe load and panel readiness are checked by the caller before measuring layout. */ -export async function waitForConversation(page: Page) { - await page.evaluate(async () => { - await document.fonts.ready; - const scroller = document.querySelector(".chat-pane .chat-scroll"); - if (!scroller?.firstElementChild) throw new Error("No conversation content for the landing recording"); - await new Promise((resolve, reject) => { - let changedAt = performance.now(); - let frame: number; - const resized = new ResizeObserver(() => { changedAt = performance.now(); }); - const observe = () => { - for (const element of [scroller, scroller.firstElementChild!, - ...scroller.querySelectorAll("[data-chat-turn], [data-chat-anchor], [data-answer-position], iframe")]) resized.observe(element); - }; - const mutated = new MutationObserver(() => { changedAt = performance.now(); observe(); }); - const cleanup = () => { resized.disconnect(); mutated.disconnect(); cancelAnimationFrame(frame); clearTimeout(timeout); }; - const timeout = setTimeout(() => { - cleanup(); - reject(new Error("Conversation did not stop resizing for 1.5 seconds")); - }, 30_000); - observe(); - mutated.observe(scroller, { childList: true, subtree: true, characterData: true }); - const check = () => { - const answer = scroller.querySelector('[data-answer-position] .chat-markdown:not([data-provisional])'); - if (answer?.textContent?.trim() && performance.now() - changedAt >= 1_500) { - cleanup(); - resolve(); - } else frame = requestAnimationFrame(check); - }; - frame = requestAnimationFrame(check); - }); - }); -} - -/** Show the completed result at reading speed, then hold its final frame. */ -export async function revealRecording(page: Page, view: "chat" | "dashboard") { - await page.evaluate(async view => { - const scroller = view === "chat" - ? document.querySelector(".chat-pane .chat-scroll") - : document.scrollingElement as HTMLElement | null; - if (!scroller) throw new Error(`No ${view} scroll container for the landing recording`); - - const answer = view === "chat" ? scroller.querySelector("[data-answer-position]") : null; - if (view === "chat" && !answer) throw new Error("No final answer for the landing recording"); - const answerEnd = () => scroller.scrollTop + answer!.getBoundingClientRect().bottom - - scroller.getBoundingClientRect().top - scroller.clientTop - scroller.clientHeight; - - // Release the question anchor before controlling the recording's scroll. - if (view === "chat") scroller.dispatchEvent(new WheelEvent("wheel", { deltaY: -1 })); - scroller.scrollTop = 0; - await new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(() => resolve()))); - - const animate = async (target: () => number, milliseconds: number) => { - const from = scroller.scrollTop; - if (target() === from) return; - const start = performance.now(); - await new Promise(resolve => { - const frame = (now: number) => { - const progress = Math.min(1, (now - start) / milliseconds); - // Brief acceleration/deceleration around a constant-speed middle 80%. - const ramp = 0.1; - const eased = progress < ramp ? progress * progress / (2 * ramp * (1 - ramp)) - : progress > 1 - ramp ? 1 - (1 - progress) ** 2 / (2 * ramp * (1 - ramp)) - : (progress - ramp / 2) / (1 - ramp); - // Re-measure every frame: evidence can still grow after readiness. - scroller.scrollTop = from + (target() - from) * eased; - if (progress < 1) requestAnimationFrame(frame); - else requestAnimationFrame(() => { scroller.scrollTop = target(); resolve(); }); - }; - requestAnimationFrame(frame); - }); - }; - const target = view === "chat" ? answerEnd : () => Math.max(0, scroller.scrollHeight - scroller.clientHeight); - const bottom = Math.max(0, target()); - const minimum = view === "chat" ? 8_000 : 6_000; - // Two viewport heights take the baseline duration; longer results get more time. - const duration = Math.min(12_000, Math.max(minimum, minimum * bottom / Math.max(1, scroller.clientHeight * 2))); - await animate(target, duration); - if (view === "dashboard") { - await new Promise(resolve => setTimeout(resolve, 1_500)); - await animate(() => 0, 1_500); - await new Promise(resolve => setTimeout(resolve, 3_000)); - } else { - // Keep the answer end aligned throughout the dwell, including late iframe resizes. - const until = performance.now() + 4_000; - await new Promise(resolve => { - const hold = () => { - scroller.scrollTop = answerEnd(); - if (performance.now() < until) requestAnimationFrame(hold); - else resolve(); - }; - requestAnimationFrame(hold); - }); - } - }, view); -} - -/** Observe the completed, unmodified SSE response; never route or replace requests. */ -export function recordingTimeline(stream: string, responseAt: number) { - const events = stream.split(/\r?\n\r?\n/).flatMap(frame => { - const data = frame.split(/\r?\n/).filter(line => line.startsWith("data:")).map(line => line.slice(5).trimStart()).join("\n"); - return data ? [JSON.parse(data) as AgentEvent] : []; - }); - const first = events.find(event => event.type === "RUN_STARTED"); - if (!first || !Number.isFinite(first.timestamp)) throw new Error("Agent stream has no timestamped RUN_STARTED event"); - if (events.some(event => event.type === "RUN_ERROR")) throw new Error("Agent run failed; inspect the instance before recording again"); - if (!events.some(event => event.type === "RUN_FINISHED")) throw new Error("Agent stream ended without RUN_FINISHED"); - const waits: WaitInterval[] = []; - const tools = new Map(); - const names = new Map(); - let thinking: number | undefined, dashboardId: string | undefined; - const endThinking = (end: number) => { - if (thinking !== undefined && end > thinking) waits.push({ kind: "model_thinking", start: thinking, end }); - thinking = undefined; - }; - for (const event of events) { - if (!Number.isFinite(event.timestamp)) throw new Error(`Agent event ${event.type} has no timestamp`); - // Anchor server timestamps to receipt of the response headers to avoid clock skew. - const at = responseAt + (event.timestamp - first.timestamp) / 1000; - switch (event.type) { - case "RUN_STARTED": thinking = at; break; - case "TEXT_MESSAGE_START": endThinking(at); break; - case "TOOL_CALL_START": - endThinking(at); - if (!event.toolCallId || !event.toolCallName) throw new Error("Tool call has no id or name"); - tools.set(event.toolCallId, { start: at, tool: event.toolCallName }); - names.set(event.toolCallId, event.toolCallName); - break; - case "TOOL_CALL_RESULT": { - const call = tools.get(event.toolCallId!); - if (!call) throw new Error("Tool result has no matching start event"); - if (at > call.start) waits.push({ kind: "tool_call", ...call, end: at }); - tools.delete(event.toolCallId!); - if (tools.size === 0) thinking = at; - if (names.get(event.toolCallId!) === "create_dashboard" && event.content) { - const result = JSON.parse(event.content) as { isError?: boolean; error?: unknown; dashboard?: { id?: string }; receipt?: unknown }; - if (!result.isError && !result.error && result.receipt && typeof result.dashboard?.id === "string") dashboardId = result.dashboard.id; - } - break; - } - case "RUN_FINISHED": endThinking(at); break; - } - } - if (tools.size) throw new Error("Agent finished with unresolved tool calls"); - return { waits: waits.sort((a, b) => a.start - b.start), dashboardId }; -} diff --git a/ui/host/playwright.config.ts b/ui/host/playwright.config.ts index 73cf28349..f100e598f 100644 --- a/ui/host/playwright.config.ts +++ b/ui/host/playwright.config.ts @@ -16,7 +16,7 @@ export default defineConfig({ forbidOnly: !!process.env.CI, failOnFlakyTests: true, timeout: 120_000, - globalTimeout: (process.env.FANOUT_LANDING_RECORDINGS === "1" ? 40 : 12) * 60_000, + globalTimeout: 12 * 60_000, expect: { timeout: 30_000, toHaveScreenshot: { animations: "disabled" } }, reporter: [["list"], ["./e2e/failure-report.ts", { outputFolder: "playwright-report" }]], outputDir: "test-results",