From 342dcce8212638cb70b97896dc7a06f39ca2c85b Mon Sep 17 00:00:00 2001 From: Ricardo Vega Date: Wed, 30 Sep 2026 06:32:12 -0500 Subject: [PATCH] Slice 4: tyto open, brief, find, and action feedback. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One command now shows everything about a page, for any model: - core (pure): assembleBrief/renderBrief — failed Fetch/XHR requests with a redacted body snippet, uncaught errors, console errors and warnings, API calls, cookie names and lifetimes (never values), storage key counts, outline, up to 40 interactive refs, main text — scoped to the current page by log offsets. Plus findLines (whole-page text search), afterAction (navigation, requests, errors, console since an action), normalizeRefs (e12 → @e12), and the agent-browser JSON mappers. New LogMarks port, fake, and file adapter. - cli: `tyto open`, `tyto brief [--json]`, `tyto find`, and click/fill/ type/press/select/check/uncheck/hover/scroll/dblclick with an AFTER report; `--session` picks the agent's agent-browser session. Found live: passing a different AGENT_BROWSER_DEFAULT_TIMEOUT to one command makes agent-browser restart that session's browser (Edge fell back to its new-tab page). Browse commands now never pass env overrides; a test pins that. Live: open+brief 1.6 s cold / 0.35 s warm (143 tokens, root cause shown), click+AFTER 0.3 s with the TypeError and source line, find on Wikipedia 0.16 s. Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 2 +- docs/IMPLEMENTATION.md | 11 ++ packages/cli/src/browse.ts | 159 +++++++++++++++++++++++ packages/cli/src/compose.ts | 13 +- packages/cli/src/main.ts | 14 +++ packages/cli/test/browse.test.ts | 141 +++++++++++++++++++++ packages/cli/test/cli.test.ts | 4 +- packages/cli/test/live/brief.test.ts | 77 ++++++++++++ packages/core/src/brief/after.ts | 18 +++ packages/core/src/brief/assemble.ts | 70 +++++++++++ packages/core/src/brief/collect.ts | 151 ++++++++++++++++++++++ packages/core/src/brief/find.ts | 17 +++ packages/core/src/brief/net.ts | 18 +++ packages/core/src/brief/refs.ts | 4 + packages/core/src/brief/render.ts | 25 ++++ packages/core/src/brief/types.ts | 70 +++++++++++ packages/core/src/index.ts | 22 +++- packages/core/src/ports/index.ts | 1 + packages/core/src/ports/log-marks.ts | 7 ++ packages/core/src/recipe/render.ts | 5 +- packages/core/src/testing/fakes.ts | 18 ++- packages/core/src/testing/index.ts | 1 + packages/core/test/brief.test.ts | 182 +++++++++++++++++++++++++++ packages/store/src/index.ts | 1 + packages/store/src/log-marks.ts | 39 ++++++ packages/store/test/store.test.ts | 17 ++- 26 files changed, 1078 insertions(+), 9 deletions(-) create mode 100644 packages/cli/src/browse.ts create mode 100644 packages/cli/test/browse.test.ts create mode 100644 packages/cli/test/live/brief.test.ts create mode 100644 packages/core/src/brief/after.ts create mode 100644 packages/core/src/brief/assemble.ts create mode 100644 packages/core/src/brief/collect.ts create mode 100644 packages/core/src/brief/find.ts create mode 100644 packages/core/src/brief/net.ts create mode 100644 packages/core/src/brief/refs.ts create mode 100644 packages/core/src/brief/render.ts create mode 100644 packages/core/src/brief/types.ts create mode 100644 packages/core/src/ports/log-marks.ts create mode 100644 packages/core/test/brief.test.ts create mode 100644 packages/store/src/log-marks.ts diff --git a/README.md b/README.md index af4091e..5b0f82d 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ npm run test:live # opt-in: needs agent-browser installed | `@tyto/llm` | OpenAI-compatible and Anthropic HTTP model adapters | | `@tyto/agent-browser` | Runs the agent-browser CLI (argv, batch JSON) | | `@tyto/store` | Recipes, session locks, replay config under `~/.tyto` | -| `@tyto/cli` | The `tyto` command (`run`, `test`, `recipes`) | +| `@tyto/cli` | The `tyto` command (`open`, `brief`, `find`, actions, `run`, `test`, `recipes`) | | `@tyto/compiler` | Coming in slice 6 | A [YOLOVibeCode](https://github.com/YOLOVibeCode) public repo. Product: Noctusoft, Inc. MIT license. diff --git a/docs/IMPLEMENTATION.md b/docs/IMPLEMENTATION.md index 52bc667..2605a94 100644 --- a/docs/IMPLEMENTATION.md +++ b/docs/IMPLEMENTATION.md @@ -33,6 +33,7 @@ Separate fakes in `@tyto/core/testing`. | `trace-store.ts` | `save`, `get`, `latest` | | `compiler.ts` | `compile(trace)`, `repair(recipe, misses, passes)` → recipe JSON | | `session-lock.ts` | `acquire(session, timeoutMs)` → release function | +| `log-marks.ts` | `get(session)`, `set(session, counts)`: log offsets taken when a page is opened | | `clock.ts`, `model.ts`, `redactor.ts`, `injection-guard.ts` | kept | ## 3. Recipe format (v1) @@ -119,6 +120,9 @@ Each slice: tests first, then code, `npm run check` green, one PR. - live: `a fixture recipe hits, then misses after the page changes`; `a missing locator fails in under 7 s` ### Slice 4 — brief, find, act +Browse commands run in the agent's own session (`--session`, default `$AGENT_BROWSER_SESSION` or `default`) with +the user's agent-browser config. They never pass env overrides: agent-browser restarts a session's browser when +launch settings change mid-session, and the page is lost (observed: Edge fell back to its new-tab page). - `brief lists failed Fetch/XHR requests with a redacted response snippet` - `brief lists uncaught errors and console errors and warnings` - `brief scopes console, errors, and requests to the current navigation by offsets` @@ -129,6 +133,13 @@ Each slice: tests first, then code, `npm run check` green, one PR. - `find with a locator keyword passes through to agent-browser find` - `act reports navigation, new requests, errors, and console since the action` - `bare e12 refs are rewritten to @e12` +- `tyto open marks the logs, opens the page, and prints the brief`; `tyto brief --json prints the brief as JSON` +- `--session selects the agent-browser session` +- `browse commands never change agent-browser settings mid-session (a changed setting restarts the browser)` +- `agent-browser restore status lines are not echoed` +- `FileLogMarks returns zero offsets for an unknown session and round-trips saved offsets with mode 0600` +- live: `the brief shows a planted 500 with its body and the console error`; `a click reports the uncaught TypeError`; + `find locates text anywhere on the page` ### Slice 5 — recorder and traces - `learn opens the session and waits for the sync marker before recording` diff --git a/packages/cli/src/browse.ts b/packages/cli/src/browse.ts new file mode 100644 index 0000000..2cc7441 --- /dev/null +++ b/packages/cli/src/browse.ts @@ -0,0 +1,159 @@ +import { + BRIEF_STEPS, + EXIT, + LOG_STEPS, + PAGE_FACTS_JS, + PAGE_TEXT_JS, + afterAction, + assembleBrief, + briefParts, + findLines, + isApi, + logCounts, + normalizeRefs, + renderBrief, + toBase64Utf8, + toConsole, + toErrors, + toPageFacts, + toRequests, + type BrowserRunner, + type LogMarks, + type Redactor, +} from "@tyto/core"; + +export type BrowseDeps = { + runner: BrowserRunner; + marks: LogMarks; + redactor: Redactor; + /** agent-browser session used when `--session` is not given. */ + session: string; + now: () => number; +}; + +type Out = { out: (line: string) => void; err: (line: string) => void }; + +export const ACTIONS = new Set(["click", "dblclick", "fill", "type", "press", "select", "check", "uncheck", "hover", "scroll"]); +const LOCATORS = new Set(["role", "text", "label", "placeholder", "alt", "title", "testid", "first", "last", "nth"]); +const MAX_FAILED_BODIES = 3; +/** + * Settle before reporting: wait for the load event. Never pass env overrides here — agent-browser restarts the + * session's browser when launch settings change mid-session (the page is lost). + */ +const SETTLE: readonly string[] = ["wait", "--load", "load"]; + +type Flags = { session: string; json: boolean; rest: string[] }; + +/** Drop agent-browser's own restore/save status lines. */ +function quiet(text: string): string { + return text + .split("\n") + .filter((l) => !/^\[agent-browser\] (?:restore|launched browser)/.test(l)) + .join("\n") + .trimEnd(); +} + +/** Pull `--session ` and `--json` out of the arguments. */ +export function browseFlags(args: readonly string[], defaultSession: string): Flags { + const rest: string[] = []; + let session = defaultSession; + let json = false; + for (let i = 0; i < args.length; i += 1) { + const a = args[i] ?? ""; + if (a === "--json") json = true; + else if (a === "--session" && args[i + 1] !== undefined) { + session = args[i + 1] ?? defaultSession; + i += 1; + } else if (a.startsWith("--session=")) session = a.slice("--session=".length); + else rest.push(a); + } + return { session, json, rest }; +} + +async function failedBodies(runner: BrowserRunner, session: string, requests: ReturnType): Promise> { + const bodies: Record = {}; + const failed = requests.filter((r) => isApi(r) && (r.status ?? 0) >= 400).slice(0, MAX_FAILED_BODIES); + for (const r of failed) { + const res = await runner.run(["network", "request", r.requestId, "--json"], { session }); + try { + const data = (JSON.parse(res.stdout) as { data?: { responseBody?: unknown } }).data; + if (typeof data?.responseBody === "string") bodies[r.requestId] = data.responseBody; + } catch { + bodies[r.requestId] = ""; + } + } + return bodies; +} + +export async function brief(args: readonly string[], deps: BrowseDeps, io: Out): Promise { + const f = browseFlags(args, deps.session); + const results = await deps.runner.batch([...BRIEF_STEPS, ["eval", "-b", toBase64Utf8(PAGE_FACTS_JS)]], { session: f.session }); + const parts = briefParts(results); + const mark = await deps.marks.get(f.session); + const bodies = await failedBodies(deps.runner, f.session, parts.requests.slice(mark.requests)); + const b = assembleBrief({ ...parts, page: toPageFacts(results.at(-1)?.result), bodies, mark, now: deps.now() }, deps.redactor); + io.out(f.json ? JSON.stringify(b) : renderBrief(b)); + return EXIT.hit; +} + +export async function open(args: readonly string[], deps: BrowseDeps, io: Out): Promise { + const f = browseFlags(args, deps.session); + const url = f.rest[0]; + if (!url) { + io.err("usage: tyto open [--session name] [--json]"); + return EXIT.usage; + } + const counts = logCounts(await deps.runner.batch(LOG_STEPS, { session: f.session })); + await deps.marks.set(f.session, { console: counts.console, errors: counts.errors, requests: counts.requests }); + const opened = await deps.runner.run(["open", url], { session: f.session }); + if (opened.exitCode !== 0) { + io.err(`open failed: ${(opened.stderr || opened.stdout).trim().slice(0, 300)}`); + return EXIT.miss; + } + await deps.runner.run(SETTLE, { session: f.session }); + return brief([...(f.json ? ["--json"] : []), "--session", f.session], deps, io); +} + +export async function find(args: readonly string[], deps: BrowseDeps, io: Out): Promise { + const f = browseFlags(args, deps.session); + if (f.rest[0] && LOCATORS.has(f.rest[0])) { + const res = await deps.runner.run(["find", ...normalizeRefs(f.rest)], { session: f.session }); + if (quiet(res.stdout)) io.out(quiet(res.stdout)); + if (quiet(res.stderr)) io.err(quiet(res.stderr)); + return res.exitCode === 0 ? EXIT.hit : EXIT.miss; + } + const query = f.rest.join(" "); + if (!query) { + io.err("usage: tyto find [--session name]"); + return EXIT.usage; + } + const results = await deps.runner.batch([["eval", "-b", toBase64Utf8(PAGE_TEXT_JS)]], { session: f.session }); + const value = (results[0]?.result as { result?: unknown } | undefined)?.result; + const found = findLines(typeof value === "string" ? deps.redactor.safe(value) : "", query); + if (!found.count) { + io.out(`no line contains all of: ${query}`); + return EXIT.miss; + } + io.out(`${found.count} match(es) for '${query}' (line ⏎ next lines):`); + for (const h of found.hits) io.out(` • ${h}`); + return EXIT.hit; +} + +export async function act(command: string, args: readonly string[], deps: BrowseDeps, io: Out): Promise { + const f = browseFlags(args, deps.session); + const before = logCounts(await deps.runner.batch(LOG_STEPS, { session: f.session })); + const res = await deps.runner.run(normalizeRefs([command, ...f.rest]), { session: f.session }); + if (quiet(res.stdout)) io.out(quiet(res.stdout)); + if (quiet(res.stderr)) io.err(quiet(res.stderr)); + await deps.runner.run(SETTLE, { session: f.session }); + const after = await deps.runner.batch(LOG_STEPS, { session: f.session }); + io.out( + afterAction(before, { + url: (after[0]?.result as { url?: string } | undefined)?.url ?? "", + console: toConsole(after[1]?.result), + errors: toErrors(after[2]?.result), + requests: toRequests(after[3]?.result), + }), + ); + return res.exitCode === 0 ? EXIT.hit : EXIT.miss; +} diff --git a/packages/cli/src/compose.ts b/packages/cli/src/compose.ts index f63641f..67d54a8 100644 --- a/packages/cli/src/compose.ts +++ b/packages/cli/src/compose.ts @@ -1,20 +1,29 @@ import { homedir } from "node:os"; import { join } from "node:path"; import { AgentBrowserRunner } from "@tyto/agent-browser"; -import { FileSessionLock, FilesystemRecipeStore, ensureReplayFiles } from "@tyto/store"; +import { SecretRedactor } from "@tyto/core"; +import { FileLogMarks, FileSessionLock, FilesystemRecipeStore, ensureReplayFiles } from "@tyto/store"; import type { CliDeps } from "./main.ts"; /** Composition root: the only place real adapters are wired together. */ export async function composeDeps(env: NodeJS.ProcessEnv = process.env): Promise { const home = env.TYTO_HOME && env.TYTO_HOME !== "" ? env.TYTO_HOME : join(homedir(), ".tyto"); const paths = await ensureReplayFiles(home); + const runner = new AgentBrowserRunner({ bin: env.TYTO_AGENT_BROWSER ?? "agent-browser" }); return { store: new FilesystemRecipeStore(join(home, "recipes")), exec: { - runner: new AgentBrowserRunner({ bin: env.TYTO_AGENT_BROWSER ?? "agent-browser" }), + runner, lock: new FileSessionLock(join(home, "locks")), paths, }, + browse: { + runner, + marks: new FileLogMarks(join(home, "marks")), + redactor: new SecretRedactor(), + session: env.AGENT_BROWSER_SESSION && env.AGENT_BROWSER_SESSION !== "" ? env.AGENT_BROWSER_SESSION : "default", + now: () => Date.now(), + }, out: (line) => process.stdout.write(`${line}\n`), err: (line) => process.stderr.write(`${line}\n`), }; diff --git a/packages/cli/src/main.ts b/packages/cli/src/main.ts index 3bcd28b..f35a82b 100644 --- a/packages/cli/src/main.ts +++ b/packages/cli/src/main.ts @@ -1,14 +1,21 @@ import { EXIT, executeRecipe, type ExecDeps, type RecipeStore } from "@tyto/core"; +import { ACTIONS, act, brief, find, open, type BrowseDeps } from "./browse.ts"; export type CliDeps = { store: RecipeStore; exec: ExecDeps; + browse: BrowseDeps; out: (line: string) => void; err: (line: string) => void; }; export const USAGE = `usage: tyto + tyto open [--session s] [--json] open a page and print its brief (errors, network, state, elements) + tyto brief [--session s] [--json] brief of the current page + tyto find [--session s] search the whole page text + tyto click|fill|type|press|select|check|uncheck|hover|scroll|dblclick + act through agent-browser, then print what happened tyto run [--param value ...] replay a recipe with no model (exit 0 hit, 3 miss) tyto test run the recipe's regression cases tyto recipes [--json] list recipes @@ -133,6 +140,12 @@ export async function main(argv: readonly string[], deps: CliDeps): Promise = []; + errors: Array<{ text: string }> = []; + requests: Array<{ requestId: string; url: string; method: string; status: number; resourceType: string }> = []; + bodyText = ""; + + load(url: string): void { + this.url = url; + this.title = "Shop"; + this.requests.push({ requestId: `r${this.requests.length}`, url, method: "GET", status: 200, resourceType: "Document" }); + this.requests.push({ requestId: "api", url: "http://127.0.0.1:8765/api/items", method: "GET", status: 500, resourceType: "Fetch" }); + this.console.push({ type: "error", text: "items API failed: 500" }); + this.bodyText = "Shop\nNo items\nConservation status\nVulnerable (IUCN 3.1)"; + } + + answer(command: readonly string[]): unknown { + const [cmd, sub] = command; + if (cmd === "get" && sub === "url") return { url: this.url }; + if (cmd === "get" && sub === "title") return { title: this.title }; + if (cmd === "snapshot") return { snapshot: '- heading "Shop" [level=1, ref=e1]\n- button "Save" [ref=e2]' }; + if (cmd === "console") return { messages: this.console }; + if (cmd === "errors") return { errors: this.errors }; + if (cmd === "network") return { requests: this.requests }; + if (cmd === "cookies") return { cookies: [{ name: "sid", value: "SECRET", httpOnly: true, session: true, domain: "127.0.0.1" }] }; + if (cmd === "storage") return { data: {} }; + if (cmd === "eval") { + const code = new TextDecoder().decode(Uint8Array.from(atob(command[2] ?? ""), (c) => c.charCodeAt(0))); + if (code.includes("innerText")) return { result: code.includes("performance") ? JSON.stringify({ status: 200, text: "Shop No items", headings: ["h1 Shop"] }) : this.bodyText }; + } + return {}; + } +} + +function harness() { + const page = new FakePage(); + const runner = new FakeBrowserRunner((steps): BatchStepResult[] => + steps.map((command) => ({ command, success: true, result: page.answer(command), error: null })), + ); + runner.onRun = (argv: readonly string[], _opts: RunOptions): RunOutput => { + if (argv[0] === "open") page.load(argv[1] ?? ""); + if (argv[0] === "click" && argv[1] === "@e2") page.errors.push({ text: "TypeError: draft is undefined" }); + if (argv[0] === "network" && argv[1] === "request") { + return { exitCode: 0, stdout: JSON.stringify({ success: true, data: { responseBody: '{"error":"database timeout"}' } }), stderr: "" }; + } + if (argv[0] === "find") return { exitCode: 0, stdout: "✓ Done\n", stderr: "" }; + return { exitCode: 0, stdout: "✓ Done\n", stderr: "" }; + }; + const out: string[] = []; + const marks = new MemoryLogMarks(); + const deps: CliDeps = { + store: new MemoryRecipeStore(), + exec: { runner, lock: new FakeSessionLock(), paths: { config: "/c.json", policy: "/p.json" } }, + browse: { runner, marks, redactor: new SecretRedactor(), session: "default", now: () => 1_790_000_000_000 }, + out: (s) => out.push(s), + err: (s) => out.push(`ERR ${s}`), + }; + return { deps, out, runner, page, marks }; +} + +describe("tyto browse commands", () => { + it("tyto open marks the logs, opens the page, and prints the brief", async () => { + const h = harness(); + h.page.console.push({ type: "error", text: "error from an earlier page" }); + expect(await main(["open", "http://127.0.0.1:8765/shop"], h.deps)).toBe(0); + expect(await h.marks.get("default")).toEqual({ console: 1, errors: 0, requests: 0 }); + expect(h.runner.runs.map((r) => r.argv[0])).toContain("open"); + const text = h.out.join("\n"); + expect(text).toMatch(/network 500 GET \/api\/items \(Fetch\) → \{"error":"database timeout"\}/); + expect(text).toMatch(/console.error 'items API failed: 500'/); + expect(text).not.toMatch(/earlier page/); + expect(text).not.toMatch(/SECRET/); + }); + + it("browse commands never change agent-browser settings mid-session (a changed setting restarts the browser)", async () => { + const h = harness(); + await main(["open", "http://127.0.0.1:8765/shop"], h.deps); + await main(["click", "e2"], h.deps); + expect(h.runner.runs.every((r) => r.opts.env === undefined)).toBe(true); + expect(h.runner.batches.every((b) => b.opts.env === undefined)).toBe(true); + }); + + it("tyto brief --json prints the brief as JSON", async () => { + const h = harness(); + await main(["open", "http://127.0.0.1:8765/shop"], h.deps); + h.out.length = 0; + expect(await main(["brief", "--json"], h.deps)).toBe(0); + const brief = JSON.parse(h.out.join("")) as { url: string; issues: string[]; cookies: Array<{ name: string }> }; + expect(brief.url).toBe("http://127.0.0.1:8765/shop"); + expect(brief.issues.length).toBe(2); + expect(brief.cookies[0]?.name).toBe("sid"); + }); + + it("--session selects the agent-browser session", async () => { + const h = harness(); + await main(["open", "http://127.0.0.1:8765/shop", "--session", "task-7"], h.deps); + expect(h.runner.runs.every((r) => r.opts.session === "task-7")).toBe(true); + expect(await h.marks.get("task-7")).toEqual({ console: 0, errors: 0, requests: 0 }); + }); + + it("agent-browser restore status lines are not echoed", async () => { + const h = harness(); + const base = h.runner.onRun; + h.runner.onRun = (argv, opts) => { + const r = base ? base(argv, opts) : { exitCode: 0, stdout: "", stderr: "" }; + return { ...r, stderr: "[agent-browser] restore: loaded; save: saved\n" }; + }; + await main(["click", "e2"], h.deps); + expect(h.out.join("\n")).not.toMatch(/restore: loaded/); + }); + + it("tyto find searches the whole page text", async () => { + const h = harness(); + await main(["open", "http://127.0.0.1:8765/shop"], h.deps); + h.out.length = 0; + expect(await main(["find", "conservation", "status"], h.deps)).toBe(0); + expect(h.out.join("\n")).toMatch(/Conservation status ⏎ Vulnerable \(IUCN 3\.1\)/); + }); + + it("tyto find with a locator keyword passes through to agent-browser find", async () => { + const h = harness(); + expect(await main(["find", "role", "button", "click", "--name", "Save"], h.deps)).toBe(0); + expect(h.runner.runs.at(-1)?.argv).toEqual(["find", "role", "button", "click", "--name", "Save"]); + }); + + it("tyto click prints what happened after the action, accepting e2 for @e2", async () => { + const h = harness(); + await main(["open", "http://127.0.0.1:8765/shop"], h.deps); + h.out.length = 0; + expect(await main(["click", "e2"], h.deps)).toBe(0); + expect(h.runner.runs.some((r) => r.argv[0] === "click" && r.argv[1] === "@e2")).toBe(true); + expect(h.out.join("\n")).toMatch(/AFTER\n {2}✗ uncaught TypeError: draft is undefined/); + }); +}); diff --git a/packages/cli/test/cli.test.ts b/packages/cli/test/cli.test.ts index 734e60c..685a147 100644 --- a/packages/cli/test/cli.test.ts +++ b/packages/cli/test/cli.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { parseRecipe, type Recipe } from "@tyto/core"; -import { FakeBrowserRunner, FakeSessionLock, MemoryRecipeStore, evalReturns, stepFails } from "@tyto/core/testing"; +import { SecretRedactor } from "@tyto/core"; +import { FakeBrowserRunner, FakeSessionLock, MemoryLogMarks, MemoryRecipeStore, evalReturns, stepFails } from "@tyto/core/testing"; import { main, type CliDeps } from "../src/index.ts"; function recipe(overrides: Record = {}): Recipe { @@ -25,6 +26,7 @@ function harness(runner = new FakeBrowserRunner(evalReturns(JSON.stringify({ tag const deps: CliDeps = { store: new MemoryRecipeStore(...(recipes.length ? recipes : [recipe()])), exec: { runner, lock: new FakeSessionLock(), paths: { config: "/c.json", policy: "/p.json" } }, + browse: { runner, marks: new MemoryLogMarks(), redactor: new SecretRedactor(), session: "default", now: () => 0 }, out: (s) => out.push(s), err: (s) => err.push(s), }; diff --git a/packages/cli/test/live/brief.test.ts b/packages/cli/test/live/brief.test.ts new file mode 100644 index 0000000..7123773 --- /dev/null +++ b/packages/cli/test/live/brief.test.ts @@ -0,0 +1,77 @@ +import { execFile } from "node:child_process"; +import { createServer, type Server } from "node:http"; +import { mkdtemp, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { composeDeps, main } from "../../src/index.ts"; + +// Live: real agent-browser against loopback pages with planted problems. Uses a throwaway agent-browser config +// (no saved logins) so the user's restore state is never touched. +const run = promisify(execFile); +const SESSION = `tyto-live-brief-${process.pid}`; +let server: Server; +let origin = ""; +let abConfig = ""; + +const PAGES: Record = { + "/shop": + 'Shop

Shop

  • Loading…
', + "/click": + 'Editor

Editor

Conservation status

Vulnerable (IUCN 3.1)

', +}; + +beforeAll(async () => { + server = createServer((req, res) => { + if (req.url === "/api/items") { + res.writeHead(500, { "content-type": "application/json" }); + res.end('{"error":"database timeout"}'); + return; + } + res.writeHead(200, { "content-type": "text/html" }); + res.end(PAGES[req.url ?? ""] ?? "404"); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const addr = server.address(); + origin = `http://127.0.0.1:${typeof addr === "object" && addr ? addr.port : 0}`; + const dir = await mkdtemp(join(tmpdir(), "tyto-live-")); + abConfig = join(dir, "ab.json"); + await writeFile(abConfig, "{}"); + process.env.AGENT_BROWSER_CONFIG = abConfig; + process.env.TYTO_HOME = dir; +}); + +afterAll(async () => { + await run("agent-browser", ["--session", SESSION, "close"], { env: { ...process.env, AGENT_BROWSER_CONFIG: abConfig } }).catch(() => undefined); + delete process.env.AGENT_BROWSER_CONFIG; + await new Promise((resolve) => server.close(() => resolve())); +}); + +async function tyto(argv: string[]): Promise<{ code: number; text: string }> { + const deps = await composeDeps(process.env); + const lines: string[] = []; + const code = await main([...argv, "--session", SESSION], { ...deps, out: (s) => lines.push(s), err: (s) => lines.push(s) }); + return { code, text: lines.join("\n") }; +} + +describe("live brief", () => { + it("the brief shows a planted 500 with its body and the console error", async () => { + const { code, text } = await tyto(["open", `${origin}/shop`]); + expect(code).toBe(0); + expect(text).toMatch(/network 500 GET \/api\/items \(Fetch\) → \{"error":"database timeout"\}/); + expect(text).toMatch(/console\.error 'items API failed: 500'/); + }, 60_000); + + it("a click reports the uncaught TypeError", async () => { + await tyto(["open", `${origin}/click`]); + const { text } = await tyto(["click", "#save"]); + expect(text).toMatch(/✗ uncaught TypeError: Cannot read properties of undefined \(reading 'id'\)/); + }, 60_000); + + it("find locates text anywhere on the page", async () => { + const { code, text } = await tyto(["find", "conservation", "status"]); + expect(code).toBe(0); + expect(text).toMatch(/Conservation status ⏎ Vulnerable \(IUCN 3\.1\)/); + }, 60_000); +}); diff --git a/packages/core/src/brief/after.ts b/packages/core/src/brief/after.ts new file mode 100644 index 0000000..dbd66c1 --- /dev/null +++ b/packages/core/src/brief/after.ts @@ -0,0 +1,18 @@ +import { isNoise, originOf, shortUrl } from "./net.ts"; +import type { ConsoleEntry, ErrorEntry, RequestEntry } from "./types.ts"; + +export type Before = { url: string; console: number; errors: number; requests: number }; +export type AfterState = { url: string; console: ConsoleEntry[]; errors: ErrorEntry[]; requests: RequestEntry[] }; + +/** What changed since an action: navigation, new requests, uncaught errors, console output. */ +export function afterAction(before: Before, after: AfterState): string { + const origin = originOf(after.url); + const lines: string[] = []; + if (after.url !== before.url) lines.push(` → now at ${after.url}`); + for (const r of after.requests.slice(before.requests)) { + if (!isNoise(r)) lines.push(` net ${r.method} ${shortUrl(r.url, origin)} → ${r.status ?? "pending"}`); + } + for (const e of after.errors.slice(before.errors)) lines.push(` ✗ uncaught ${e.text.slice(0, 300)}`); + for (const m of after.console.slice(before.console)) lines.push(` console.${m.type} '${m.text.slice(0, 200)}'`); + return `AFTER\n${lines.length ? lines.join("\n") : " no navigation, requests, or console output"}`; +} diff --git a/packages/core/src/brief/assemble.ts b/packages/core/src/brief/assemble.ts new file mode 100644 index 0000000..afb2fac --- /dev/null +++ b/packages/core/src/brief/assemble.ts @@ -0,0 +1,70 @@ +import type { Redactor } from "../ports/redactor.ts"; +import { isApi, isNoise, originOf, shortUrl } from "./net.ts"; +import type { Brief, BriefCookie, BriefInput, CookieEntry } from "./types.ts"; + +const MAX_ELEMENTS = 40; +const MAX_ISSUES_PER_KIND = 5; + +function lifetime(c: CookieEntry, nowMs: number): string { + if (c.session === true || c.expires === undefined || c.expires <= 0) return "session"; + return `${Math.max(0, Math.round((c.expires - nowMs / 1000) / 86_400))}d`; +} + +/** Everything about the current page, scoped by `mark`, secrets masked. */ +export function assembleBrief(input: BriefInput, redactor: Redactor): Brief { + const origin = originOf(input.url); + const requests = input.requests.slice(input.mark.requests).filter((r) => !isNoise(r)); + const errors = input.errors.slice(input.mark.errors); + const consoleEntries = input.console.slice(input.mark.console); + + const failed = requests.filter((r) => (r.status ?? 0) >= 400); + const issues: string[] = []; + for (const r of failed.slice(0, MAX_ISSUES_PER_KIND)) { + let line = `network ${r.status ?? "?"} ${r.method} ${shortUrl(r.url, origin)} (${r.resourceType ?? "?"})`; + const body = input.bodies[r.requestId]; + if (body && isApi(r)) line += ` → ${redactor.safe(body.replace(/\s+/g, " ")).slice(0, 160)}`; + issues.push(line); + } + for (const e of errors.slice(0, MAX_ISSUES_PER_KIND)) issues.push(`uncaught ${redactor.safe(e.text).slice(0, 300)}`); + for (const m of consoleEntries) { + if (m.type === "error" || m.type === "warning" || m.type === "assert") issues.push(`console.${m.type} '${redactor.safe(m.text).slice(0, 200)}'`); + } + + const byType: Record = {}; + for (const r of requests) { + const t = (r.resourceType ?? "other").toLowerCase(); + byType[t] = (byType[t] ?? 0) + 1; + } + + const cookies: BriefCookie[] = input.cookies.map((c) => ({ + name: c.name, + domain: c.domain ?? "", + httpOnly: c.httpOnly === true, + lifetime: lifetime(c, input.now), + })); + + const elements = input.snapshot.split("\n").map((l) => l.trimEnd()).filter((l) => l.trim() !== ""); + + return { + url: input.url, + title: input.title, + status: input.page.status, + framework: input.page.framework, + timing: { dclMs: input.page.dclMs, loadMs: input.page.loadMs }, + forms: input.page.forms, + iframes: input.page.iframes, + issues, + network: { + total: requests.length, + byType, + failed: failed.length, + api: requests.filter(isApi).slice(0, 15).map((r) => `${r.method} ${shortUrl(r.url, origin)} → ${r.status ?? "pending"}`), + }, + cookies, + storage: { local: Object.keys(input.localStorage).length, session: Object.keys(input.sessionStorage).length }, + outline: input.page.headings, + elements: elements.slice(0, MAX_ELEMENTS), + elementsTotal: elements.length, + text: redactor.safe(input.page.text), + }; +} diff --git a/packages/core/src/brief/collect.ts b/packages/core/src/brief/collect.ts new file mode 100644 index 0000000..a941947 --- /dev/null +++ b/packages/core/src/brief/collect.ts @@ -0,0 +1,151 @@ +import type { BatchStepResult } from "../ports/browser-runner.ts"; +import type { ConsoleEntry, CookieEntry, ErrorEntry, LogCounts, PageFacts, RequestEntry } from "./types.ts"; + +/** Facts only the page can tell: framework, navigation timing, outline, main text. Read-only. */ +export const PAGE_FACTS_JS = `JSON.stringify((() => { + const fw = []; + if (window.__NEXT_DATA__ || window.next) fw.push("Next.js"); + if (window.__NUXT__) fw.push("Nuxt"); + if (window.React || document.querySelector("[data-reactroot]") || + [...document.querySelectorAll("body > div")].some((d) => Object.keys(d).some((k) => k.startsWith("__react")))) fw.push("React"); + if (window.Vue || document.querySelector("[data-v-app]")) fw.push("Vue"); + const ng = document.querySelector("[ng-version]"); + if (ng) fw.push("Angular " + ng.getAttribute("ng-version")); + if (window.jQuery) fw.push("jQuery " + ((window.jQuery.fn && window.jQuery.fn.jquery) || "")); + const nav = performance.getEntriesByType("navigation")[0] || {}; + const main = document.querySelector("main, [role=main], article") || document.body; + return { + framework: fw, status: nav.responseStatus || null, + dclMs: Math.round(nav.domContentLoadedEventEnd || 0), loadMs: Math.round(nav.loadEventEnd || 0), + headings: [...document.querySelectorAll("h1,h2,h3")].slice(0, 8).map((h) => h.tagName.toLowerCase() + " " + h.innerText.trim().slice(0, 80)), + forms: document.forms.length, iframes: document.querySelectorAll("iframe").length, + text: ((main && main.innerText) || "").replace(/\\s+/g, " ").trim().slice(0, 300), + }; +})())`; + +export const PAGE_TEXT_JS = "document.body ? document.body.innerText : ''"; + +/** The single batch a brief needs, in the order `briefParts` reads it. */ +export const BRIEF_STEPS: readonly (readonly string[])[] = [ + ["get", "url"], + ["get", "title"], + ["snapshot", "-i"], + ["console"], + ["errors"], + ["network", "requests"], + ["cookies"], + ["storage", "local"], + ["storage", "session"], +]; + +/** Log lengths only: used for offsets before navigation and before/after actions. */ +export const LOG_STEPS: readonly (readonly string[])[] = [["get", "url"], ["console"], ["errors"], ["network", "requests"]]; + +function rec(v: unknown): Record { + return typeof v === "object" && v !== null && !Array.isArray(v) ? (v as Record) : {}; +} + +function arr(v: unknown): unknown[] { + return Array.isArray(v) ? v : []; +} + +function str(v: unknown): string { + return typeof v === "string" ? v : ""; +} + +function num(v: unknown): number | undefined { + return typeof v === "number" && Number.isFinite(v) ? v : undefined; +} + +export function toConsole(v: unknown): ConsoleEntry[] { + return arr(rec(v).messages).map((m) => ({ type: str(rec(m).type) || "log", text: str(rec(m).text) })); +} + +export function toErrors(v: unknown): ErrorEntry[] { + return arr(rec(v).errors).map((e) => ({ text: str(rec(e).text) || str(rec(e).message) })); +} + +export function toRequests(v: unknown): RequestEntry[] { + return arr(rec(v).requests).map((r) => { + const o = rec(r); + const entry: RequestEntry = { requestId: str(o.requestId), url: str(o.url), method: str(o.method) || "GET" }; + const status = num(o.status); + if (status !== undefined) entry.status = status; + if (typeof o.resourceType === "string") entry.resourceType = o.resourceType; + if (typeof o.mimeType === "string") entry.mimeType = o.mimeType; + return entry; + }); +} + +function toCookies(v: unknown): CookieEntry[] { + return arr(rec(v).cookies).map((c) => { + const o = rec(c); + const entry: CookieEntry = { name: str(o.name) }; + if (typeof o.domain === "string") entry.domain = o.domain; + if (typeof o.httpOnly === "boolean") entry.httpOnly = o.httpOnly; + if (typeof o.secure === "boolean") entry.secure = o.secure; + const expires = num(o.expires); + if (expires !== undefined) entry.expires = expires; + if (typeof o.session === "boolean") entry.session = o.session; + return entry; + }); +} + +export function toPageFacts(evalResult: unknown): PageFacts { + let v = rec(evalResult).result; + if (typeof v === "string") { + try { + v = JSON.parse(v) as unknown; + } catch { + v = {}; + } + } + const o = rec(v); + return { + framework: arr(o.framework).map(String), + status: num(o.status) ?? null, + dclMs: num(o.dclMs) ?? 0, + loadMs: num(o.loadMs) ?? 0, + headings: arr(o.headings).map(String), + forms: num(o.forms) ?? 0, + iframes: num(o.iframes) ?? 0, + text: str(o.text), + }; +} + +export type BriefParts = { + url: string; + title: string; + snapshot: string; + console: ConsoleEntry[]; + errors: ErrorEntry[]; + requests: RequestEntry[]; + cookies: CookieEntry[]; + localStorage: Record; + sessionStorage: Record; +}; + +/** Map the results of `BRIEF_STEPS` (agent-browser JSON) into typed parts. */ +export function briefParts(results: readonly BatchStepResult[]): BriefParts { + const r = (i: number): unknown => results[i]?.result; + return { + url: str(rec(r(0)).url), + title: str(rec(r(1)).title), + snapshot: str(rec(r(2)).snapshot), + console: toConsole(r(3)), + errors: toErrors(r(4)), + requests: toRequests(r(5)), + cookies: toCookies(r(6)), + localStorage: rec(rec(r(7)).data), + sessionStorage: rec(rec(r(8)).data), + }; +} + +export function logCounts(results: readonly BatchStepResult[]): LogCounts & { url: string } { + return { + url: str(rec(results[0]?.result).url), + console: toConsole(results[1]?.result).length, + errors: toErrors(results[2]?.result).length, + requests: toRequests(results[3]?.result).length, + }; +} diff --git a/packages/core/src/brief/find.ts b/packages/core/src/brief/find.ts new file mode 100644 index 0000000..37900f3 --- /dev/null +++ b/packages/core/src/brief/find.ts @@ -0,0 +1,17 @@ +export type FindResult = { count: number; hits: string[] }; + +/** Lines of page text containing every word (case-insensitive), each with the next two lines as context. */ +export function findLines(pageText: string, query: string, max = 8): FindResult { + const terms = query.toLowerCase().split(/\s+/).filter(Boolean); + const lines = pageText.split(/\n+/).map((l) => l.trim()).filter(Boolean); + const hits: string[] = []; + let count = 0; + if (!terms.length) return { count, hits }; + lines.forEach((line, i) => { + const low = line.toLowerCase(); + if (!terms.every((t) => low.includes(t))) return; + count += 1; + if (hits.length < max) hits.push(lines.slice(i, i + 3).join(" ⏎ ").slice(0, 300)); + }); + return { count, hits }; +} diff --git a/packages/core/src/brief/net.ts b/packages/core/src/brief/net.ts new file mode 100644 index 0000000..e3197de --- /dev/null +++ b/packages/core/src/brief/net.ts @@ -0,0 +1,18 @@ +import type { RequestEntry } from "./types.ts"; + +export function originOf(url: string): string { + const m = /^[a-z][a-z0-9+.-]*:\/\/[^/]+/i.exec(url); + return m ? m[0] : ""; +} + +export function shortUrl(url: string, origin: string): string { + return origin && url.startsWith(origin) ? url.slice(origin.length) || "/" : url; +} + +export function isNoise(req: RequestEntry): boolean { + return /\/favicon\.ico(?:$|\?)/.test(req.url); +} + +export function isApi(req: RequestEntry): boolean { + return req.resourceType === "Fetch" || req.resourceType === "XHR"; +} diff --git a/packages/core/src/brief/refs.ts b/packages/core/src/brief/refs.ts new file mode 100644 index 0000000..b116ac3 --- /dev/null +++ b/packages/core/src/brief/refs.ts @@ -0,0 +1,4 @@ +/** Accept `e12` where agent-browser expects `@e12` (small models often drop the @). */ +export function normalizeRefs(argv: readonly string[]): string[] { + return argv.map((a) => (/^e\d+$/.test(a) ? `@${a}` : a)); +} diff --git a/packages/core/src/brief/render.ts b/packages/core/src/brief/render.ts new file mode 100644 index 0000000..14f9ef8 --- /dev/null +++ b/packages/core/src/brief/render.ts @@ -0,0 +1,25 @@ +import type { Brief } from "./types.ts"; + +/** Compact text brief for any model. Ends with an approximate token count. */ +export function renderBrief(b: Brief): string { + const out: string[] = []; + out.push(`PAGE ${b.title} — ${b.url} [${b.status ?? "?"}, DOMContentLoaded ${b.timing.dclMs}ms, load ${b.timing.loadMs}ms]`); + out.push(`APP ${b.framework.join(", ") || "no framework detected"} · ${b.forms} form(s) · ${b.iframes} iframe(s)`); + out.push(`ISSUES ${b.issues.length}${b.issues.length ? "" : " — none"}`); + for (const i of b.issues) out.push(` ✗ ${i}`); + const kinds = Object.entries(b.network.byType) + .sort(([a], [z]) => a.localeCompare(z)) + .map(([k, v]) => `${v} ${k}`) + .join(", "); + out.push(`NETWORK ${b.network.total} requests${kinds ? ` (${kinds})` : ""}, ${b.network.failed} failed`); + for (const a of b.network.api.slice(0, 10)) out.push(` api ${a}`); + const cookies = b.cookies.map((c) => `${c.name}(${c.httpOnly ? "httpOnly " : ""}${c.lifetime})`).join(", ") || "none"; + out.push(`STATE cookies: ${cookies} · localStorage: ${b.storage.local} keys · sessionStorage: ${b.storage.session} keys (values masked)`); + if (b.outline.length) out.push(`OUTLINE ${b.outline.join(" | ")}`); + out.push(`ELEMENTS (${b.elementsTotal} interactive; use the @eN refs with tyto click/fill)`); + for (const e of b.elements) out.push(` ${e}`); + if (b.elementsTotal > b.elements.length) out.push(` … ${b.elementsTotal - b.elements.length} more (agent-browser snapshot -i)`); + if (b.text) out.push(`TEXT ${b.text}`); + const text = out.join("\n"); + return `${text}\n(${Math.round(text.length / 4)} tokens)`; +} diff --git a/packages/core/src/brief/types.ts b/packages/core/src/brief/types.ts new file mode 100644 index 0000000..fe85353 --- /dev/null +++ b/packages/core/src/brief/types.ts @@ -0,0 +1,70 @@ +export type ConsoleEntry = { type: string; text: string }; +export type ErrorEntry = { text: string; url?: string | null; line?: number; column?: number }; +export type RequestEntry = { + requestId: string; + url: string; + method: string; + status?: number; + resourceType?: string; + mimeType?: string; +}; +/** Raw cookie as agent-browser reports it. `value` is never copied into a brief. */ +export type CookieEntry = { + name: string; + domain?: string; + httpOnly?: boolean; + secure?: boolean; + expires?: number; + session?: boolean; + value?: string; +}; +export type PageFacts = { + framework: string[]; + status: number | null; + dclMs: number; + loadMs: number; + headings: string[]; + forms: number; + iframes: number; + text: string; +}; +/** Log lengths when the current page was opened; entries before these offsets belong to earlier pages. */ +export type LogCounts = { console: number; errors: number; requests: number }; + +export type BriefInput = { + url: string; + title: string; + snapshot: string; + console: ConsoleEntry[]; + errors: ErrorEntry[]; + requests: RequestEntry[]; + cookies: CookieEntry[]; + localStorage: Record; + sessionStorage: Record; + page: PageFacts; + /** Response bodies of failed Fetch/XHR requests, by requestId. */ + bodies: Record; + mark: LogCounts; + /** Epoch milliseconds, for cookie lifetimes. */ + now: number; +}; + +export type BriefCookie = { name: string; domain: string; httpOnly: boolean; lifetime: string }; + +export type Brief = { + url: string; + title: string; + status: number | null; + framework: string[]; + timing: { dclMs: number; loadMs: number }; + forms: number; + iframes: number; + issues: string[]; + network: { total: number; byType: Record; failed: number; api: string[] }; + cookies: BriefCookie[]; + storage: { local: number; session: number }; + outline: string[]; + elements: string[]; + elementsTotal: number; + text: string; +}; diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 7ca6bba..524f84a 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -6,10 +6,30 @@ export { SystemClock } from "./clock/system.ts"; export type { ParamSpec, ParamType, ParamValue, Recipe, RecipeStatus, RegressionCase, Step, VerifyRule } from "./recipe/types.ts"; export { parseRecipe, type ParseResult } from "./recipe/parse.ts"; export { lintRecipe } from "./recipe/lint.ts"; -export { evalWithParams, renderRecipe, type RenderResult } from "./recipe/render.ts"; +export { evalWithParams, renderRecipe, toBase64Utf8, type RenderResult } from "./recipe/render.ts"; export { verifyResult, type VerifyOutcome } from "./recipe/verify.ts"; export { AgentBrowserMissingError, BatchOutputError } from "./run/errors.ts"; export { EXIT, type ExitCode } from "./run/exit.ts"; export { DEFAULT_ACTION_TIMEOUT_MS, replaySession, type ReplayPaths, type ReplaySession } from "./run/session.ts"; export { isLaunchError } from "./run/launch-error.ts"; export { executeRecipe, type ExecDeps, type ExecOptions, type ExecOutcome } from "./run/execute.ts"; +export type { Brief, BriefCookie, BriefInput, ConsoleEntry, CookieEntry, ErrorEntry, LogCounts, PageFacts, RequestEntry } from "./brief/types.ts"; +export { assembleBrief } from "./brief/assemble.ts"; +export { renderBrief } from "./brief/render.ts"; +export { findLines, type FindResult } from "./brief/find.ts"; +export { afterAction, type AfterState, type Before } from "./brief/after.ts"; +export { normalizeRefs } from "./brief/refs.ts"; +export { isApi, isNoise, originOf, shortUrl } from "./brief/net.ts"; +export { + BRIEF_STEPS, + LOG_STEPS, + PAGE_FACTS_JS, + PAGE_TEXT_JS, + briefParts, + logCounts, + toConsole, + toErrors, + toPageFacts, + toRequests, + type BriefParts, +} from "./brief/collect.ts"; diff --git a/packages/core/src/ports/index.ts b/packages/core/src/ports/index.ts index 453c3fe..b52235f 100644 --- a/packages/core/src/ports/index.ts +++ b/packages/core/src/ports/index.ts @@ -5,3 +5,4 @@ export type { Redactor } from "./redactor.ts"; export type { BatchStepResult, BrowserRunner, RunOptions, RunOutput } from "./browser-runner.ts"; export type { RecipeStore, RecipeSummary } from "./recipe-store.ts"; export type { Release, SessionLock } from "./session-lock.ts"; +export type { LogMarks } from "./log-marks.ts"; diff --git a/packages/core/src/ports/log-marks.ts b/packages/core/src/ports/log-marks.ts new file mode 100644 index 0000000..e16678b --- /dev/null +++ b/packages/core/src/ports/log-marks.ts @@ -0,0 +1,7 @@ +import type { LogCounts } from "../brief/types.ts"; + +/** Per-session log offsets taken when a page is opened, so a brief covers only that page. */ +export interface LogMarks { + get(session: string): Promise; + set(session: string, counts: LogCounts): Promise; +} diff --git a/packages/core/src/recipe/render.ts b/packages/core/src/recipe/render.ts index b9a8b8d..c3af851 100644 --- a/packages/core/src/recipe/render.ts +++ b/packages/core/src/recipe/render.ts @@ -71,7 +71,8 @@ function renderArg(arg: string, params: Record, origins: rea return out; } -function utf8Base64(text: string): string { +/** Base64 of the UTF-8 bytes, for `agent-browser eval -b`. */ +export function toBase64Utf8(text: string): string { let binary = ""; for (const byte of new TextEncoder().encode(text)) binary += String.fromCharCode(byte); return btoa(binary); @@ -86,7 +87,7 @@ export function renderRecipe(recipe: Recipe, input: Readonly { - if (step[0] === "eval") return ["eval", "-b", utf8Base64(evalWithParams(step[1] ?? "", params))]; + if (step[0] === "eval") return ["eval", "-b", toBase64Utf8(evalWithParams(step[1] ?? "", params))]; const rendered = step.map((arg) => renderArg(arg, params, recipe.origins)); if (rendered[0] === "open" && !inOrigins(rendered[1] ?? "", recipe.origins)) { throw new RenderError("rendered URL is outside the recipe's origins"); diff --git a/packages/core/src/testing/fakes.ts b/packages/core/src/testing/fakes.ts index 26bc04c..5b659fb 100644 --- a/packages/core/src/testing/fakes.ts +++ b/packages/core/src/testing/fakes.ts @@ -2,6 +2,8 @@ import type { BatchStepResult, BrowserRunner, RunOptions, RunOutput } from "../p import type { RecipeStore, RecipeSummary } from "../ports/recipe-store.ts"; import type { Release, SessionLock } from "../ports/session-lock.ts"; import type { Recipe } from "../recipe/types.ts"; +import type { LogCounts } from "../brief/types.ts"; +import type { LogMarks } from "../ports/log-marks.ts"; import type { Clock } from "../ports/clock.ts"; import type { ModelPort } from "../ports/model.ts"; import type { CompleteRequest, CompleteResponse } from "../types.ts"; @@ -53,6 +55,8 @@ export class FakeBrowserRunner implements BrowserRunner { readonly runs: Array<{ argv: readonly string[]; opts: RunOptions }> = []; private scripts: BatchScript[]; runOutput: RunOutput = { exitCode: 0, stdout: "", stderr: "" }; + /** Optional per-call answer for `run`; falls back to `runOutput`. */ + onRun?: (argv: readonly string[], opts: RunOptions) => RunOutput; constructor(...scripts: BatchScript[]) { this.scripts = scripts; @@ -60,7 +64,7 @@ export class FakeBrowserRunner implements BrowserRunner { async run(argv: readonly string[], opts: RunOptions): Promise { this.runs.push({ argv, opts }); - return this.runOutput; + return this.onRun ? this.onRun(argv, opts) : this.runOutput; } async batch(steps: readonly (readonly string[])[], opts: RunOptions): Promise { @@ -137,3 +141,15 @@ export class FakeSessionLock implements SessionLock { }; } } + +export class MemoryLogMarks implements LogMarks { + readonly marks = new Map(); + + async get(session: string): Promise { + return this.marks.get(session) ?? { console: 0, errors: 0, requests: 0 }; + } + + async set(session: string, counts: LogCounts): Promise { + this.marks.set(session, counts); + } +} diff --git a/packages/core/src/testing/index.ts b/packages/core/src/testing/index.ts index 20a5a98..4139421 100644 --- a/packages/core/src/testing/index.ts +++ b/packages/core/src/testing/index.ts @@ -3,6 +3,7 @@ export { FakeClock, FakeModel, FakeSessionLock, + MemoryLogMarks, MemoryRecipeStore, evalReturns, stepFails, diff --git a/packages/core/test/brief.test.ts b/packages/core/test/brief.test.ts new file mode 100644 index 0000000..7b76770 --- /dev/null +++ b/packages/core/test/brief.test.ts @@ -0,0 +1,182 @@ +import { describe, expect, it } from "vitest"; +import { SecretRedactor, afterAction, assembleBrief, findLines, normalizeRefs, renderBrief, type BriefInput } from "../src/index.ts"; + +function input(overrides: Partial = {}): BriefInput { + return { + url: "http://127.0.0.1:8765/shop", + title: "Shop", + snapshot: '- heading "Shop" [level=1, ref=e1]', + console: [], + errors: [], + requests: [ + { requestId: "1", url: "http://127.0.0.1:8765/shop", method: "GET", status: 200, resourceType: "Document" }, + ], + cookies: [], + localStorage: {}, + sessionStorage: {}, + page: { framework: [], status: 200, dclMs: 12, loadMs: 20, headings: ["h1 Shop"], forms: 0, iframes: 0, text: "Shop No items" }, + bodies: {}, + mark: { console: 0, errors: 0, requests: 0 }, + now: 1_790_000_000_000, + ...overrides, + }; +} + +const redactor = new SecretRedactor(); + +describe("brief", () => { + it("lists failed Fetch/XHR requests with a redacted response snippet", () => { + const b = assembleBrief( + input({ + requests: [ + { requestId: "1", url: "http://127.0.0.1:8765/shop", method: "GET", status: 200, resourceType: "Document" }, + { requestId: "2", url: "http://127.0.0.1:8765/api/items", method: "GET", status: 500, resourceType: "Fetch" }, + ], + bodies: { "2": '{"error":"database timeout","token":"s3cr3tTOKEN"}' }, + }), + redactor, + ); + const issue = b.issues.find((i) => i.includes("/api/items")); + expect(issue).toMatch(/500 GET \/api\/items \(Fetch\).*database timeout/); + expect(issue).not.toContain("s3cr3tTOKEN"); + expect(b.network.api).toEqual(["GET /api/items → 500"]); + }); + + it("lists uncaught errors and console errors and warnings", () => { + const b = assembleBrief( + input({ + errors: [{ text: "TypeError: Cannot read properties of undefined (reading 'id')" }], + console: [ + { type: "log", text: "saving" }, + { type: "error", text: "items API failed: 500" }, + { type: "warning", text: "deprecated API" }, + ], + }), + redactor, + ); + expect(b.issues).toEqual([ + "uncaught TypeError: Cannot read properties of undefined (reading 'id')", + "console.error 'items API failed: 500'", + "console.warning 'deprecated API'", + ]); + }); + + it("scopes console, errors, and requests to the current navigation by offsets", () => { + const b = assembleBrief( + input({ + console: [{ type: "error", text: "old page error" }, { type: "error", text: "new page error" }], + errors: [{ text: "TypeError: old" }], + requests: [ + { requestId: "1", url: "http://127.0.0.1:8765/old", method: "GET", status: 500, resourceType: "Fetch" }, + { requestId: "2", url: "http://127.0.0.1:8765/shop", method: "GET", status: 200, resourceType: "Document" }, + ], + mark: { console: 1, errors: 1, requests: 1 }, + }), + redactor, + ); + expect(b.issues).toEqual(["console.error 'new page error'"]); + expect(b.network.total).toBe(1); + }); + + it("shows cookie names, httpOnly, and lifetime, never values", () => { + const b = assembleBrief( + input({ + cookies: [ + { name: "sid", domain: "127.0.0.1", httpOnly: true, session: true, value: "SECRETVALUE" }, + { name: "remember", domain: "127.0.0.1", httpOnly: true, expires: 1_790_000_000 + 30 * 86_400, value: "LONGSECRET" }, + ], + }), + redactor, + ); + expect(b.cookies).toEqual([ + { name: "sid", domain: "127.0.0.1", httpOnly: true, lifetime: "session" }, + { name: "remember", domain: "127.0.0.1", httpOnly: true, lifetime: "30d" }, + ]); + const text = renderBrief(b); + expect(text).not.toMatch(/SECRETVALUE|LONGSECRET/); + expect(text).toMatch(/sid\(httpOnly session\)/); + }); + + it("ignores favicon 404s", () => { + const b = assembleBrief( + input({ + requests: [ + { requestId: "1", url: "http://127.0.0.1:8765/shop", method: "GET", status: 200, resourceType: "Document" }, + { requestId: "2", url: "http://127.0.0.1:8765/favicon.ico", method: "GET", status: 404, resourceType: "Other" }, + ], + }), + redactor, + ); + expect(b.issues).toEqual([]); + expect(b.network.total).toBe(1); + }); + + it("caps interactive elements at 40 and reports the remainder", () => { + const snapshot = Array.from({ length: 45 }, (_, i) => `- link "Item ${i}" [ref=e${i + 1}]`).join("\n"); + const b = assembleBrief(input({ snapshot }), redactor); + expect(b.elements).toHaveLength(40); + expect(b.elementsTotal).toBe(45); + expect(renderBrief(b)).toMatch(/… 5 more/); + }); + + it("renders page, issues, network, state, outline, elements, and text sections", () => { + const text = renderBrief(assembleBrief(input(), redactor)); + for (const section of ["PAGE", "APP", "ISSUES", "NETWORK", "STATE", "OUTLINE", "ELEMENTS", "TEXT"]) expect(text).toContain(section); + expect(text).toMatch(/tokens\)$/); + }); +}); + +describe("find", () => { + it("returns lines containing all words with following context, at most 8", () => { + const lines = ["Snowy owl", "Conservation status", "Vulnerable (IUCN 3.1)", "Scientific classification", ...Array.from({ length: 20 }, () => "owl status note")]; + const hits = findLines(lines.join("\n"), "conservation status"); + expect(hits.count).toBe(1); + expect(hits.hits[0]).toBe("Conservation status ⏎ Vulnerable (IUCN 3.1) ⏎ Scientific classification"); + const many = findLines(lines.join("\n"), "owl status"); + expect(many.count).toBe(20); + expect(many.hits).toHaveLength(8); + }); + + it("is case-insensitive and needs every word", () => { + expect(findLines("Tiger\nEndangered species", "ENDANGERED").count).toBe(1); + expect(findLines("Tiger\nEndangered species", "endangered tiger").count).toBe(0); + }); +}); + +describe("afterAction", () => { + it("reports navigation, new requests, errors, and console since the action", () => { + const text = afterAction( + { url: "http://x.test/login", console: 1, errors: 0, requests: 2 }, + { + url: "http://x.test/account", + console: [{ type: "log", text: "old" }, { type: "log", text: "saving" }], + errors: [{ text: "TypeError: x is undefined" }], + requests: [ + { requestId: "1", url: "http://x.test/login", method: "GET", status: 200 }, + { requestId: "2", url: "http://x.test/a.js", method: "GET", status: 200 }, + { requestId: "3", url: "http://x.test/login", method: "POST", status: 303 }, + { requestId: "4", url: "http://x.test/favicon.ico", method: "GET", status: 404 }, + ], + }, + ); + expect(text.split("\n")).toEqual([ + "AFTER", + " → now at http://x.test/account", + " net POST /login → 303", + " ✗ uncaught TypeError: x is undefined", + " console.log 'saving'", + ]); + }); + + it("says so when nothing happened", () => { + const same = { url: "http://x.test/", console: [], errors: [], requests: [] }; + expect(afterAction({ url: "http://x.test/", console: 0, errors: 0, requests: 0 }, same)).toBe("AFTER\n no navigation, requests, or console output"); + }); +}); + +describe("normalizeRefs", () => { + it("bare e12 refs are rewritten to @e12", () => { + expect(normalizeRefs(["click", "e12"])).toEqual(["click", "@e12"]); + expect(normalizeRefs(["fill", "@e3", "e4 is text"])).toEqual(["fill", "@e3", "e4 is text"]); + }); +}); diff --git a/packages/store/src/index.ts b/packages/store/src/index.ts index 76c1e3f..2c3d6e2 100644 --- a/packages/store/src/index.ts +++ b/packages/store/src/index.ts @@ -2,3 +2,4 @@ export { readIfExists, writePrivate } from "./files.ts"; export { FilesystemRecipeStore } from "./recipe-store.ts"; export { FileSessionLock } from "./session-lock.ts"; export { DEFAULT_USER_CONFIG, ensureReplayFiles } from "./replay-files.ts"; +export { FileLogMarks } from "./log-marks.ts"; diff --git a/packages/store/src/log-marks.ts b/packages/store/src/log-marks.ts new file mode 100644 index 0000000..c63e8cf --- /dev/null +++ b/packages/store/src/log-marks.ts @@ -0,0 +1,39 @@ +import { join } from "node:path"; +import type { LogCounts, LogMarks } from "@tyto/core"; +import { readIfExists, writePrivate } from "./files.ts"; + +const SESSION = /^[\w-]+$/; +const ZERO: LogCounts = { console: 0, errors: 0, requests: 0 }; + +function count(v: unknown): number { + return typeof v === "number" && Number.isInteger(v) && v >= 0 ? v : 0; +} + +/** `/.json` holding the log offsets taken when the session last opened a page. */ +export class FileLogMarks implements LogMarks { + readonly #dir: string; + + constructor(dir: string) { + this.#dir = dir; + } + + #path(session: string): string { + if (!SESSION.test(session)) throw new Error(`invalid session name: ${session}`); + return join(this.#dir, `${session}.json`); + } + + async get(session: string): Promise { + const text = await readIfExists(this.#path(session)); + if (text === null) return { ...ZERO }; + try { + const o = JSON.parse(text) as Record; + return { console: count(o.console), errors: count(o.errors), requests: count(o.requests) }; + } catch { + return { ...ZERO }; + } + } + + async set(session: string, counts: LogCounts): Promise { + await writePrivate(this.#path(session), `${JSON.stringify(counts)}\n`); + } +} diff --git a/packages/store/test/store.test.ts b/packages/store/test/store.test.ts index a299d04..fc84456 100644 --- a/packages/store/test/store.test.ts +++ b/packages/store/test/store.test.ts @@ -3,7 +3,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; import { parseRecipe, type Recipe } from "@tyto/core"; -import { FileSessionLock, FilesystemRecipeStore, ensureReplayFiles } from "../src/index.ts"; +import { FileLogMarks, FileSessionLock, FilesystemRecipeStore, ensureReplayFiles } from "../src/index.ts"; function recipe(name = "release", overrides: Record = {}): Recipe { const parsed = parseRecipe({ @@ -101,3 +101,18 @@ describe("ensureReplayFiles", () => { expect(JSON.parse(await readFile(paths.config, "utf8"))).toEqual({}); }); }); + +describe("FileLogMarks", () => { + it("returns zero offsets for an unknown session and round-trips saved offsets with mode 0600", async () => { + const dir = await tmp(); + const marks = new FileLogMarks(dir); + expect(await marks.get("default")).toEqual({ console: 0, errors: 0, requests: 0 }); + await marks.set("task-7", { console: 2, errors: 1, requests: 5 }); + expect(await marks.get("task-7")).toEqual({ console: 2, errors: 1, requests: 5 }); + expect((await stat(join(dir, "task-7.json"))).mode & 0o777).toBe(0o600); + }); + + it("refuses session names with path characters", async () => { + await expect(new FileLogMarks(await tmp()).get("../x")).rejects.toThrow(/session/); + }); +});