From 5f2343a7d54501085417b932cad8ece3edff902f Mon Sep 17 00:00:00 2001 From: Ricardo Vega Date: Wed, 30 Sep 2026 06:40:16 -0500 Subject: [PATCH] Slice 5: record browser tasks with `tyto learn`. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `tyto learn [--session s]` records what an agent does in an agent-browser session; `tyto learn stop --task "…" --param input_N=name` saves a trace the compiler (next slice) turns into a recipe. - core: StreamEvent/Trace types; actionToArgv maps agent-browser's internal stream actions (navigate, getbylabel, waitforurl, …) back to CLI argv; buildTrace pairs command/result by id, flags gaps (orphan result, missing result, socket closed) as lossy, replaces typed values with {{input_N}} (or {{param}} when kept), masks password-like fields, reduces cookie/storage values to names and lengths, and redacts outputs. Recorder waits for a `get url` sync marker before recording. New ports BrowserEventSource and TraceStore, with fakes. - agent-browser: StreamEventSource reads /.stream (mirrors agent-browser's socket-dir rules) and yields events, dropping frames, ending with a closed event. - store: FileTraceStore (0600). - cli: learn start/status/stop and a detached listener that keeps typed values in memory only, controlled over a unix socket in a 0700 dir. Found while probing: the stream's `success` is false even for successful commands, so success comes from the result's error field. Live: recorded a Wikipedia search → 6 steps in CLI form, the search term kept as {{species}} (example "Snowy owl"), eval result captured, no gaps. Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 6 +- docs/IMPLEMENTATION.md | 26 +++- packages/agent-browser/src/index.ts | 1 + packages/agent-browser/src/stream.ts | 116 +++++++++++++++++ packages/agent-browser/test/stream.test.ts | 92 +++++++++++++ packages/cli/src/compose.ts | 36 +++++- packages/cli/src/index.ts | 3 + packages/cli/src/learn/commands.ts | 100 ++++++++++++++ packages/cli/src/learn/control.ts | 88 +++++++++++++ packages/cli/src/learn/listener.ts | 45 +++++++ packages/cli/src/main.ts | 7 + packages/cli/test/browse.test.ts | 1 + packages/cli/test/cli.test.ts | 1 + packages/cli/test/learn.test.ts | 120 +++++++++++++++++ packages/cli/test/live/learn.test.ts | 56 ++++++++ packages/core/src/index.ts | 5 + packages/core/src/ports/browser-events.ts | 6 + packages/core/src/ports/index.ts | 2 + packages/core/src/ports/trace-store.ts | 6 + packages/core/src/recipe/lint.ts | 5 + packages/core/src/testing/fakes.ts | 44 +++++++ packages/core/src/testing/index.ts | 2 + packages/core/src/trace/argv.ts | 83 ++++++++++++ packages/core/src/trace/build.ts | 139 ++++++++++++++++++++ packages/core/src/trace/recorder.ts | 61 +++++++++ packages/core/src/trace/types.ts | 30 +++++ packages/core/test/recorder.test.ts | 41 ++++++ packages/core/test/trace.test.ts | 143 +++++++++++++++++++++ packages/store/src/index.ts | 1 + packages/store/src/trace-store.ts | 25 ++++ packages/store/test/store.test.ts | 14 +- 31 files changed, 1293 insertions(+), 12 deletions(-) create mode 100644 packages/agent-browser/src/stream.ts create mode 100644 packages/agent-browser/test/stream.test.ts create mode 100644 packages/cli/src/learn/commands.ts create mode 100644 packages/cli/src/learn/control.ts create mode 100644 packages/cli/src/learn/listener.ts create mode 100644 packages/cli/test/learn.test.ts create mode 100644 packages/cli/test/live/learn.test.ts create mode 100644 packages/core/src/ports/browser-events.ts create mode 100644 packages/core/src/ports/trace-store.ts create mode 100644 packages/core/src/trace/argv.ts create mode 100644 packages/core/src/trace/build.ts create mode 100644 packages/core/src/trace/recorder.ts create mode 100644 packages/core/src/trace/types.ts create mode 100644 packages/core/test/recorder.test.ts create mode 100644 packages/core/test/trace.test.ts create mode 100644 packages/store/src/trace-store.ts diff --git a/README.md b/README.md index 5b0f82d..99974a5 100644 --- a/README.md +++ b/README.md @@ -51,9 +51,9 @@ npm run test:live # opt-in: needs agent-browser installed |---|---| | `@tyto/core` | Recipes, lint, verify, brief, find, redaction, ports, fakes (pure) | | `@tyto/llm` | OpenAI-compatible and Anthropic HTTP model adapters | -| `@tyto/agent-browser` | Runs the agent-browser CLI (argv, batch JSON) | -| `@tyto/store` | Recipes, session locks, replay config under `~/.tyto` | -| `@tyto/cli` | The `tyto` command (`open`, `brief`, `find`, actions, `run`, `test`, `recipes`) | +| `@tyto/agent-browser` | Runs the agent-browser CLI (argv, batch JSON); reads its event stream | +| `@tyto/store` | Recipes, traces, session locks, log marks, replay config under `~/.tyto` | +| `@tyto/cli` | The `tyto` command (`learn`, `run`, `test`, `recipes`, `open`, `brief`, `find`, actions) | | `@tyto/compiler` | Coming in slice 6 | A [YOLOVibeCode](https://github.com/YOLOVibeCode) public repo. Product: Noctusoft, Inc. MIT license. diff --git a/docs/IMPLEMENTATION.md b/docs/IMPLEMENTATION.md index 2605a94..960a79e 100644 --- a/docs/IMPLEMENTATION.md +++ b/docs/IMPLEMENTATION.md @@ -28,9 +28,9 @@ Separate fakes in `@tyto/core/testing`. | Port | Shape | |---|---| | `browser-runner.ts` | `run(argv, opts)`, `batch(steps, opts)` → typed step results; opts carry session, env, AbortSignal | -| `event-source.ts` | `subscribe(session, signal)` → `AsyncIterable` | +| `browser-events.ts` | `BrowserEventSource.subscribe(session, signal)` → `AsyncIterable` | | `recipe-store.ts` | `get`, `list`, `save`, `remove` | -| `trace-store.ts` | `save`, `get`, `latest` | +| `trace-store.ts` | `save`, `get` | | `compiler.ts` | `compile(trace)`, `repair(recipe, misses, passes)` → recipe JSON | | `session-lock.ts` | `acquire(session, timeoutMs)` → release function | | `log-marks.ts` | `get(session)`, `set(session, counts)`: log offsets taken when a page is opened | @@ -142,14 +142,32 @@ launch settings change mid-session, and the page is lost (observed: Edge fell ba `find locates text anywhere on the page` ### Slice 5 — recorder and traces -- `learn opens the session and waits for the sync marker before recording` +agent-browser's event stream (`/.stream` holds the WebSocket port; socketDir follows +`AGENT_BROWSER_SOCKET_DIR`, then `$XDG_RUNTIME_DIR/agent-browser`, then `~/.agent-browser`) reports **internal** +action names (`navigate`, `getbylabel`, `waitforurl`, …), which `actionToArgv` maps back to CLI argv. Its +`success` field is `false` even for successful commands, so success comes from the result's `error`. Typed values +arrive in plaintext: the listener holds them in memory only; `tyto learn stop --param input_N=name` keeps chosen +ones as params. `tyto learn` talks to its detached listener over a unix socket in `~/.tyto/learn/` (dir 0700). +- `learn waits for the sync marker before recording` - `pairs command and result events by id into trace steps` - `marks the trace lossy on an orphan result, a missing result, or a socket close` - `ignores frame messages` - `replaces fill and type values with {{input_N}} placeholders` - `keeps only inputs named with --param and drops the rest` - `reduces cookie and storage values to names and lengths` -- `runs the Redactor before the trace is saved` +- `runs the Redactor before the trace is returned` +- `actionToArgv maps navigate, click, fill, type, press, select, check, waits, eval, get, snapshot, and find locators` +- `masks values typed into password-like fields and never offers them as params` +- `collects the origins the task opened`; `keeps snapshot text and eval results as step output` +- `StreamEventSource reads the port file and yields parsed events, dropping frames`; `yields a closed event when the socket closes` +- `agentBrowserSocketDir follows agent-browser's socket directory rules` +- `FileTraceStore saves traces with mode 0600 and loads them by name` +- `tyto learn starts the session and a listener, then prints how to finish`; `tyto learn stop requires --task` +- `tyto learn stop sends the task and kept params, then prints the trace summary` +- `tyto learn status lists typed inputs by name and locator, never values` +- `the listener records after the marker and saves the trace with kept params when stopped` +- `control client and server exchange messages over a private socket` +- live: `records command/result pairs from the event stream and keeps only the named input` ### Slice 6 — compiler (Claude Code) - `the prompt fences trace page text with a random nonce and labels it as data` diff --git a/packages/agent-browser/src/index.ts b/packages/agent-browser/src/index.ts index 83f0648..20f5248 100644 --- a/packages/agent-browser/src/index.ts +++ b/packages/agent-browser/src/index.ts @@ -1 +1,2 @@ export { AgentBrowserRunner, type AgentBrowserRunnerOptions } from "./runner.ts"; +export { StreamEventSource, agentBrowserSocketDir, type StreamEventSourceOptions, type WebSocketLike } from "./stream.ts"; diff --git a/packages/agent-browser/src/stream.ts b/packages/agent-browser/src/stream.ts new file mode 100644 index 0000000..f6b54e1 --- /dev/null +++ b/packages/agent-browser/src/stream.ts @@ -0,0 +1,116 @@ +import { readFile } from "node:fs/promises"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import type { BrowserEventSource, StreamEvent } from "@tyto/core"; + +export type WebSocketLike = { + onopen: (() => void) | null; + onmessage: ((e: { data: unknown }) => void) | null; + onclose: (() => void) | null; + onerror: ((e: unknown) => void) | null; + close(): void; +}; + +export type StreamEventSourceOptions = { + socketDir?: string; + WebSocketCtor?: new (url: string) => WebSocketLike; +}; + +/** agent-browser's `sanitize_session_component`: lowercase alphanumerics, single separators. */ +function sanitize(value: string): string { + let out = ""; + let lastSep = false; + for (const c of value) { + if (/[\p{L}\p{N}]/u.test(c)) { + out += c.toLowerCase(); + lastSep = false; + } else if (out !== "" && !lastSep) { + out += c === "_" ? "_" : "-"; + lastSep = true; + } + } + return out.replace(/[-_]+$/, ""); +} + +/** Mirrors agent-browser's `get_socket_dir`, where `.stream` holds the event stream port. */ +export function agentBrowserSocketDir(env: Readonly> = process.env, home = homedir()): string { + const explicit = env.AGENT_BROWSER_SOCKET_DIR; + const runtime = env.XDG_RUNTIME_DIR; + const base = explicit ? explicit : runtime ? join(runtime, "agent-browser") : join(home, ".agent-browser"); + const ns = env.AGENT_BROWSER_NAMESPACE ? sanitize(env.AGENT_BROWSER_NAMESPACE) : ""; + return ns ? join(base, "namespaces", ns, "run") : base; +} + +/** Subscribes to a session's WebSocket event stream (lossy by design; frames are dropped). */ +export class StreamEventSource implements BrowserEventSource { + readonly #dir: string; + readonly #Ws: new (url: string) => WebSocketLike; + + constructor(opts: StreamEventSourceOptions = {}) { + this.#dir = opts.socketDir ?? agentBrowserSocketDir(); + this.#Ws = opts.WebSocketCtor ?? (globalThis.WebSocket as unknown as new (url: string) => WebSocketLike); + } + + async *subscribe(session: string, signal: AbortSignal): AsyncIterable { + if (!/^[\w.-]+$/.test(session)) throw new Error(`invalid session name: ${session}`); + let port: string; + try { + port = (await readFile(join(this.#dir, `${session}.stream`), "utf8")).trim(); + } catch { + throw new Error(`no event stream for session ${session}; is its agent-browser daemon running?`); + } + if (!/^\d+$/.test(port)) throw new Error(`bad event stream port for session ${session}`); + + const queue: StreamEvent[] = []; + let wake: (() => void) | null = null; + let done = false; + const push = (ev: StreamEvent): void => { + queue.push(ev); + wake?.(); + }; + const ws = new this.#Ws(`ws://127.0.0.1:${port}`); + ws.onmessage = (e) => { + if (typeof e.data !== "string") return; + try { + const ev = JSON.parse(e.data) as unknown; + if (typeof ev === "object" && ev !== null && !Array.isArray(ev) && (ev as StreamEvent).type !== "frame") push(ev as StreamEvent); + } catch { + return; // not JSON: the stream is advisory, skip the message + } + }; + ws.onclose = () => { + if (!done) push({ type: "closed" }); + done = true; + }; + ws.onerror = () => { + if (!done) push({ type: "closed" }); + done = true; + }; + const onAbort = (): void => { + done = true; + ws.close(); + wake?.(); + }; + signal.addEventListener("abort", onAbort, { once: true }); + try { + while (!signal.aborted) { + const next = queue.shift(); + if (next) { + yield next; + if (next.type === "closed") return; + continue; + } + await new Promise((resolve) => { + wake = resolve; + }); + wake = null; + } + } finally { + signal.removeEventListener("abort", onAbort); + if (!done) { + done = true; + ws.close(); + } + } + } +} diff --git a/packages/agent-browser/test/stream.test.ts b/packages/agent-browser/test/stream.test.ts new file mode 100644 index 0000000..f74dd20 --- /dev/null +++ b/packages/agent-browser/test/stream.test.ts @@ -0,0 +1,92 @@ +import { mkdtemp, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import type { StreamEvent } from "@tyto/core"; +import { StreamEventSource, agentBrowserSocketDir, type WebSocketLike } from "../src/index.ts"; + +/** Minimal stand-in for the global WebSocket. */ +class FakeSocket implements WebSocketLike { + static last: FakeSocket | null = null; + onopen: (() => void) | null = null; + onmessage: ((e: { data: unknown }) => void) | null = null; + onclose: (() => void) | null = null; + onerror: ((e: unknown) => void) | null = null; + closed = false; + readonly url: string; + constructor(url: string) { + this.url = url; + FakeSocket.last = this; + queueMicrotask(() => this.onopen?.()); + } + send(data: unknown): void { + this.onmessage?.({ data }); + } + close(): void { + this.closed = true; + this.onclose?.(); + } +} + +async function collect(it: AsyncIterable, max: number): Promise { + const out: StreamEvent[] = []; + for await (const ev of it) { + out.push(ev); + if (out.length >= max) break; + } + return out; +} + +describe("StreamEventSource", () => { + it("reads the port file and yields parsed events, dropping frames", async () => { + const dir = await mkdtemp(join(tmpdir(), "tyto-stream-")); + await writeFile(join(dir, "task-1.stream"), "54321\n"); + const source = new StreamEventSource({ socketDir: dir, WebSocketCtor: FakeSocket }); + const ac = new AbortController(); + const pending = collect(source.subscribe("task-1", ac.signal), 2); + await new Promise((r) => setTimeout(r, 5)); + expect(FakeSocket.last?.url).toBe("ws://127.0.0.1:54321"); + FakeSocket.last?.send(JSON.stringify({ type: "frame", data: "…" })); + FakeSocket.last?.send(JSON.stringify({ type: "command", id: "r1", action: "click", params: {} })); + FakeSocket.last?.send("not json"); + FakeSocket.last?.send(JSON.stringify({ type: "result", id: "r1", action: "click" })); + const events = await pending; + expect(events.map((e) => e.type)).toEqual(["command", "result"]); + ac.abort(); + }); + + it("yields a closed event when the socket closes", async () => { + const dir = await mkdtemp(join(tmpdir(), "tyto-stream-")); + await writeFile(join(dir, "s.stream"), "1234"); + const source = new StreamEventSource({ socketDir: dir, WebSocketCtor: FakeSocket }); + const pending = collect(source.subscribe("s", new AbortController().signal), 5); + await new Promise((r) => setTimeout(r, 5)); + FakeSocket.last?.close(); + expect(await pending).toEqual([{ type: "closed" }]); + }); + + it("aborting closes the socket and ends the stream", async () => { + const dir = await mkdtemp(join(tmpdir(), "tyto-stream-")); + await writeFile(join(dir, "s.stream"), "1234"); + const ac = new AbortController(); + const pending = collect(new StreamEventSource({ socketDir: dir, WebSocketCtor: FakeSocket }).subscribe("s", ac.signal), 5); + await new Promise((r) => setTimeout(r, 5)); + ac.abort(); + await pending; + expect(FakeSocket.last?.closed).toBe(true); + }); + + it("a missing port file is a clear error", async () => { + const source = new StreamEventSource({ socketDir: await mkdtemp(join(tmpdir(), "tyto-stream-")), WebSocketCtor: FakeSocket }); + await expect(collect(source.subscribe("nope", new AbortController().signal), 1)).rejects.toThrow(/no event stream for session nope/); + }); +}); + +describe("agentBrowserSocketDir", () => { + it("follows agent-browser: AGENT_BROWSER_SOCKET_DIR, then XDG_RUNTIME_DIR/agent-browser, then ~/.agent-browser", () => { + expect(agentBrowserSocketDir({ AGENT_BROWSER_SOCKET_DIR: "/s" }, "/home/u")).toBe("/s"); + expect(agentBrowserSocketDir({ XDG_RUNTIME_DIR: "/run/user/1" }, "/home/u")).toBe("/run/user/1/agent-browser"); + expect(agentBrowserSocketDir({}, "/home/u")).toBe("/home/u/.agent-browser"); + expect(agentBrowserSocketDir({ AGENT_BROWSER_NAMESPACE: "team a" }, "/home/u")).toBe("/home/u/.agent-browser/namespaces/team-a/run"); + }); +}); diff --git a/packages/cli/src/compose.ts b/packages/cli/src/compose.ts index 67d54a8..1283fac 100644 --- a/packages/cli/src/compose.ts +++ b/packages/cli/src/compose.ts @@ -1,8 +1,15 @@ +import { spawn } from "node:child_process"; import { homedir } from "node:os"; +import { fileURLToPath } from "node:url"; import { join } from "node:path"; -import { AgentBrowserRunner } from "@tyto/agent-browser"; -import { SecretRedactor } from "@tyto/core"; -import { FileLogMarks, FileSessionLock, FilesystemRecipeStore, ensureReplayFiles } from "@tyto/store"; +import { AgentBrowserRunner, StreamEventSource } from "@tyto/agent-browser"; +import { Recorder, SecretRedactor } from "@tyto/core"; +import { FileLogMarks, FileSessionLock, FileTraceStore, FilesystemRecipeStore, ensureReplayFiles } from "@tyto/store"; +import { requestControl, serveControl } from "./learn/control.ts"; +import { runListener } from "./learn/listener.ts"; + +const BIN = fileURLToPath(new URL("../bin/tyto.mjs", import.meta.url)); +const LISTEN_MAX_MS = 2 * 60 * 60 * 1000; import type { CliDeps } from "./main.ts"; /** Composition root: the only place real adapters are wired together. */ @@ -10,6 +17,8 @@ export async function composeDeps(env: NodeJS.ProcessEnv = process.env): Promise const home = env.TYTO_HOME && env.TYTO_HOME !== "" ? env.TYTO_HOME : join(homedir(), ".tyto"); const paths = await ensureReplayFiles(home); const runner = new AgentBrowserRunner({ bin: env.TYTO_AGENT_BROWSER ?? "agent-browser" }); + const redactor = new SecretRedactor(); + const socket = (name: string): string => join(home, "learn", `${name}.sock`); return { store: new FilesystemRecipeStore(join(home, "recipes")), exec: { @@ -20,10 +29,29 @@ export async function composeDeps(env: NodeJS.ProcessEnv = process.env): Promise browse: { runner, marks: new FileLogMarks(join(home, "marks")), - redactor: new SecretRedactor(), + redactor, session: env.AGENT_BROWSER_SESSION && env.AGENT_BROWSER_SESSION !== "" ? env.AGENT_BROWSER_SESSION : "default", now: () => Date.now(), }, + learn: { + spawnListener: async (name, session) => { + const child = spawn(process.execPath, [BIN, "learn", "--listen", name, "--session", session], { detached: true, stdio: "ignore", env }); + child.unref(); + }, + control: (name, req) => requestControl(socket(name), req), + listen: async (name, session) => { + await runListener(name, session, { + recorder: new Recorder({ events: new StreamEventSource(), runner }), + traces: new FileTraceStore(join(home, "traces")), + redactor, + now: () => Date.now(), + serve: (handler) => serveControl(socket(name), handler), + tracePath: (n) => join(home, "traces", `${n}.json`), + maxMs: LISTEN_MAX_MS, + }); + return 0; + }, + }, out: (line) => process.stdout.write(`${line}\n`), err: (line) => process.stderr.write(`${line}\n`), }; diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index e6710a7..b044a0f 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -1,2 +1,5 @@ export { USAGE, main, type CliDeps } from "./main.ts"; export { composeDeps } from "./compose.ts"; +export { requestControl, serveControl, type ControlReply, type ControlRequest } from "./learn/control.ts"; +export { runListener, type ListenerDeps } from "./learn/listener.ts"; +export { learnCommand, type LearnDeps } from "./learn/commands.ts"; diff --git a/packages/cli/src/learn/commands.ts b/packages/cli/src/learn/commands.ts new file mode 100644 index 0000000..2c2ad12 --- /dev/null +++ b/packages/cli/src/learn/commands.ts @@ -0,0 +1,100 @@ +import { EXIT, type BrowserRunner } from "@tyto/core"; +import type { ControlReply, ControlRequest } from "./control.ts"; + +export type LearnDeps = { + spawnListener: (name: string, session: string) => Promise; + control: (name: string, req: ControlRequest) => Promise; + /** Runs the listener in this process (used by the detached `tyto learn --listen`). */ + listen?: (name: string, session: string) => Promise; +}; + +type Io = { out: (line: string) => void; err: (line: string) => void }; + +const NAME = /^[a-z0-9]+(?:-[a-z0-9]+)*$/; +const READY_WAIT_MS = 15_000; + +function flag(args: readonly string[], name: string): string | undefined { + const i = args.indexOf(name); + return i >= 0 ? args[i + 1] : undefined; +} + +function delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function start(name: string, args: readonly string[], learn: LearnDeps, runner: BrowserRunner, io: Io): Promise { + const session = flag(args, "--session") ?? `learn-${name}`; + await runner.run(["get", "url"], { session }); // make sure the session's daemon (and its event stream) is running + await learn.spawnListener(name, session); + const deadline = Date.now() + READY_WAIT_MS; + for (;;) { + try { + const reply = await learn.control(name, { op: "status" }); + if (reply.ok) break; + } catch (err) { + if (Date.now() > deadline) { + io.err(`the recorder did not start: ${err instanceof Error ? err.message : String(err)}`); + return EXIT.internal; + } + await delay(100); + } + } + io.out(`Recording "${name}" in agent-browser session ${session}.`); + io.out(`Do the task with: agent-browser --session ${session} (or tyto open/click/fill … --session ${session})`); + io.out(`See typed inputs: tyto learn status ${name}`); + io.out(`Finish: tyto learn stop ${name} --task "" [--param input_N=name ...]`); + return EXIT.hit; +} + +async function stop(name: string, args: readonly string[], learn: LearnDeps, io: Io): Promise { + const task = flag(args, "--task"); + if (!task) { + io.err(`usage: tyto learn stop ${name} --task "" [--param input_N=name ...]`); + return EXIT.usage; + } + const keep: Record = {}; + args.forEach((a, i) => { + if (a !== "--param") return; + const m = /^(input_\d+)=(\w+)$/.exec(args[i + 1] ?? ""); + if (m?.[1] && m[2]) keep[m[1]] = m[2]; + }); + const reply = await learn.control(name, { op: "stop", task, keep }); + if (!reply.ok) { + io.err(reply.error ?? "the recorder failed"); + return EXIT.internal; + } + io.out(`Saved trace ${reply.trace ?? name}: ${reply.steps ?? 0} steps${reply.lossy ? ` (lossy: ${reply.gaps?.join("; ")})` : ""}`); + const params = Object.entries(reply.params ?? {}); + io.out(params.length ? `Params: ${params.map(([k, v]) => `${k} (example: ${v})`).join(", ")}` : "Params: none kept"); + return EXIT.hit; +} + +async function status(name: string, learn: LearnDeps, io: Io): Promise { + const reply = await learn.control(name, { op: "status" }); + io.out(`Recording "${name}": ${reply.steps ?? 0} steps${reply.lossy ? " (lossy)" : ""}`); + for (const i of reply.inputs ?? []) io.out(` ${i.name.padEnd(9)} ${i.locator}${i.sensitive ? " (secret: never kept)" : ""}`); + if (!reply.inputs?.length) io.out(" no typed inputs yet"); + return EXIT.hit; +} + +export async function learnCommand(args: readonly string[], learn: LearnDeps, runner: BrowserRunner, io: Io): Promise { + const [first, second] = args; + try { + if (first === "--listen" && second && learn.listen) return await learn.listen(second, flag(args, "--session") ?? `learn-${second}`); + if (first === "stop" || first === "status") { + if (!second || !NAME.test(second)) { + io.err(`usage: tyto learn ${first} `); + return EXIT.usage; + } + return first === "stop" ? await stop(second, args.slice(2), learn, io) : await status(second, learn, io); + } + if (!first || !NAME.test(first)) { + io.err("usage: tyto learn [--session s] (name: lowercase words joined by -)"); + return EXIT.usage; + } + return await start(first, args.slice(1), learn, runner, io); + } catch (err) { + io.err(err instanceof Error ? err.message : String(err)); + return EXIT.usage; + } +} diff --git a/packages/cli/src/learn/control.ts b/packages/cli/src/learn/control.ts new file mode 100644 index 0000000..66d00e4 --- /dev/null +++ b/packages/cli/src/learn/control.ts @@ -0,0 +1,88 @@ +import { chmod, mkdir, rm } from "node:fs/promises"; +import { createConnection, createServer } from "node:net"; +import { dirname } from "node:path"; +import type { TraceInput } from "@tyto/core"; + +export type ControlRequest = { op: "status" } | { op: "stop"; task: string; keep: Record }; + +export type ControlReply = { + ok: boolean; + error?: string; + inputs?: TraceInput[]; + steps?: number; + lossy?: boolean; + gaps?: string[]; + params?: Record; + trace?: string; +}; + +const MAX_MESSAGE = 1_000_000; + +/** Serve JSON-line requests on a unix socket inside a 0700 directory. Returns a close function. */ +export async function serveControl(path: string, handler: (req: ControlRequest) => Promise): Promise<() => Promise> { + await mkdir(dirname(path), { recursive: true, mode: 0o700 }); + await chmod(dirname(path), 0o700); + await rm(path, { force: true }); + const server = createServer((socket) => { + let buf = ""; + socket.setEncoding("utf8"); + socket.on("data", (chunk: string) => { + buf += chunk; + if (buf.length > MAX_MESSAGE) socket.destroy(); + const nl = buf.indexOf("\n"); + if (nl < 0) return; + const line = buf.slice(0, nl); + buf = ""; + let req: ControlRequest; + try { + req = JSON.parse(line) as ControlRequest; + } catch { + socket.end(`${JSON.stringify({ ok: false, error: "bad request" })}\n`); + return; + } + handler(req).then( + (reply) => socket.end(`${JSON.stringify(reply)}\n`), + (err: unknown) => socket.end(`${JSON.stringify({ ok: false, error: err instanceof Error ? err.message : String(err) })}\n`), + ); + }); + socket.on("error", () => socket.destroy()); + }); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(path, () => resolve()); + }); + await chmod(path, 0o600); + return async () => { + await new Promise((resolve) => server.close(() => resolve())); + await rm(path, { force: true }); + }; +} + +export function requestControl(path: string, req: ControlRequest, timeoutMs = 15_000): Promise { + return new Promise((resolve, reject) => { + const socket = createConnection(path); + const timer = setTimeout(() => { + socket.destroy(); + reject(new Error("the recorder did not answer")); + }, timeoutMs); + let buf = ""; + socket.setEncoding("utf8"); + socket.on("connect", () => socket.write(`${JSON.stringify(req)}\n`)); + socket.on("data", (chunk: string) => { + buf += chunk; + }); + socket.on("end", () => { + clearTimeout(timer); + try { + resolve(JSON.parse(buf) as ControlReply); + } catch { + reject(new Error("the recorder sent an unreadable reply")); + } + }); + socket.on("error", (err: NodeJS.ErrnoException) => { + clearTimeout(timer); + if (err.code === "ENOENT" || err.code === "ECONNREFUSED") reject(new Error("not recording (no tyto learn listener for that name)")); + else reject(err); + }); + }); +} diff --git a/packages/cli/src/learn/listener.ts b/packages/cli/src/learn/listener.ts new file mode 100644 index 0000000..3fa2072 --- /dev/null +++ b/packages/cli/src/learn/listener.ts @@ -0,0 +1,45 @@ +import { buildTrace, type Recorder, type Redactor, type TraceStore } from "@tyto/core"; +import type { ControlReply, ControlRequest } from "./control.ts"; + +export type ListenerDeps = { + recorder: Recorder; + traces: TraceStore; + redactor: Redactor; + now: () => number; + serve: (handler: (req: ControlRequest) => Promise) => Promise<() => Promise>; + tracePath: (name: string) => string; + /** Give up (without saving) after this long. */ + maxMs?: number; +}; + +/** The background `tyto learn` process: record until `stop`, then save the trace. Typed values stay in memory. */ +export async function runListener(name: string, session: string, deps: ListenerDeps): Promise { + const ac = new AbortController(); + const startedAt = deps.now(); + await deps.recorder.start(session, ac.signal); + const opts = { name, session, redactor: deps.redactor, now: deps.now, startedAt }; + let finish: () => void = () => undefined; + const finished = new Promise((resolve) => { + finish = resolve; + }); + let close: () => Promise = async () => undefined; + const shutdown = (): void => { + setTimeout(() => { + ac.abort(); + void close().finally(finish); + }, 0); + }; + close = await deps.serve(async (req) => { + if (req.op === "status") { + const { trace, inputs } = buildTrace(deps.recorder.events(), opts, {}); + return { ok: true, inputs, steps: trace.steps.length, lossy: trace.lossy, gaps: trace.gaps }; + } + const { trace, inputs } = buildTrace(deps.recorder.events(), { ...opts, task: req.task }, req.keep); + await deps.traces.save(trace); + shutdown(); + return { ok: true, inputs, steps: trace.steps.length, lossy: trace.lossy, gaps: trace.gaps, params: trace.params, trace: deps.tracePath(name) }; + }); + const limit = deps.maxMs ? setTimeout(shutdown, deps.maxMs) : null; + await finished; + if (limit) clearTimeout(limit); +} diff --git a/packages/cli/src/main.ts b/packages/cli/src/main.ts index f35a82b..8c24970 100644 --- a/packages/cli/src/main.ts +++ b/packages/cli/src/main.ts @@ -1,10 +1,12 @@ import { EXIT, executeRecipe, type ExecDeps, type RecipeStore } from "@tyto/core"; import { ACTIONS, act, brief, find, open, type BrowseDeps } from "./browse.ts"; +import { learnCommand, type LearnDeps } from "./learn/commands.ts"; export type CliDeps = { store: RecipeStore; exec: ExecDeps; browse: BrowseDeps; + learn: LearnDeps; out: (line: string) => void; err: (line: string) => void; }; @@ -16,6 +18,9 @@ export const USAGE = `usage: tyto tyto find [--session s] search the whole page text tyto click|fill|type|press|select|check|uncheck|hover|scroll|dblclick act through agent-browser, then print what happened + tyto learn [--session s] record a task done in that agent-browser session + tyto learn status typed inputs recorded so far (names only) + tyto learn stop --task "…" [--param input_N=name ...] save the trace tyto run [--param value ...] replay a recipe with no model (exit 0 hit, 3 miss) tyto test run the recipe's regression cases tyto recipes [--json] list recipes @@ -140,6 +145,8 @@ export async function main(argv: readonly string[], deps: CliDeps): Promise 1_790_000_000_000 }, + learn: { spawnListener: async () => undefined, control: async () => ({ ok: false }) }, out: (s) => out.push(s), err: (s) => out.push(`ERR ${s}`), }; diff --git a/packages/cli/test/cli.test.ts b/packages/cli/test/cli.test.ts index 685a147..ce4ea7c 100644 --- a/packages/cli/test/cli.test.ts +++ b/packages/cli/test/cli.test.ts @@ -27,6 +27,7 @@ function harness(runner = new FakeBrowserRunner(evalReturns(JSON.stringify({ tag store: new MemoryRecipeStore(...(recipes.length ? recipes : [recipe()])), exec: { runner, lock: new FakeSessionLock(), paths: { config: "/c.json", policy: "/p.json" } }, browse: { runner, marks: new MemoryLogMarks(), redactor: new SecretRedactor(), session: "default", now: () => 0 }, + learn: { spawnListener: async () => undefined, control: async () => ({ ok: false }) }, out: (s) => out.push(s), err: (s) => err.push(s), }; diff --git a/packages/cli/test/learn.test.ts b/packages/cli/test/learn.test.ts new file mode 100644 index 0000000..86530ec --- /dev/null +++ b/packages/cli/test/learn.test.ts @@ -0,0 +1,120 @@ +import { mkdtemp, stat } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { Recorder, SecretRedactor, type BatchStepResult } from "@tyto/core"; +import { FakeBrowserRunner, FakeEventSource, FakeSessionLock, MemoryLogMarks, MemoryRecipeStore, MemoryTraceStore } from "@tyto/core/testing"; +import { main, runListener, serveControl, requestControl, type CliDeps, type ControlReply, type ControlRequest } from "../src/index.ts"; + +function harness(reply: (req: ControlRequest) => ControlReply) { + const runner = new FakeBrowserRunner((): BatchStepResult[] => []); + const spawned: Array<{ name: string; session: string }> = []; + const sent: ControlRequest[] = []; + const out: string[] = []; + const deps: CliDeps = { + store: new MemoryRecipeStore(), + exec: { runner, lock: new FakeSessionLock(), paths: { config: "/c", policy: "/p" } }, + browse: { runner, marks: new MemoryLogMarks(), redactor: new SecretRedactor(), session: "default", now: () => 0 }, + learn: { + spawnListener: async (name, session) => { + spawned.push({ name, session }); + }, + control: async (_name, req) => { + sent.push(req); + return reply(req); + }, + }, + out: (s) => out.push(s), + err: (s) => out.push(`ERR ${s}`), + }; + return { deps, runner, spawned, sent, out }; +} + +describe("tyto learn", () => { + it("tyto learn starts the session and a listener, then prints how to finish", async () => { + const h = harness(() => ({ ok: true, inputs: [], steps: 0, lossy: false })); + expect(await main(["learn", "wiki-status", "--session", "task-3"], h.deps)).toBe(0); + expect(h.runner.runs[0]).toMatchObject({ argv: ["get", "url"], opts: { session: "task-3" } }); + expect(h.spawned).toEqual([{ name: "wiki-status", session: "task-3" }]); + const text = h.out.join("\n"); + expect(text).toMatch(/--session task-3/); + expect(text).toMatch(/tyto learn stop wiki-status --task/); + }); + + it("uses learn- as the session when none is given", async () => { + const h = harness(() => ({ ok: true, inputs: [], steps: 0, lossy: false })); + await main(["learn", "wiki-status"], h.deps); + expect(h.spawned[0]?.session).toBe("learn-wiki-status"); + }); + + it("tyto learn stop sends the task and kept params, then prints the trace summary", async () => { + const h = harness(() => ({ ok: true, inputs: [], steps: 5, lossy: false, params: { city: "Paris" }, trace: "/home/.tyto/traces/weather.json" })); + expect(await main(["learn", "stop", "weather", "--task", "weather in Paris", "--param", "input_2=city"], h.deps)).toBe(0); + expect(h.sent).toEqual([{ op: "stop", task: "weather in Paris", keep: { input_2: "city" } }]); + expect(h.out.join("\n")).toMatch(/5 steps/); + expect(h.out.join("\n")).toMatch(/city/); + }); + + it("tyto learn stop requires --task", async () => { + const h = harness(() => ({ ok: true, inputs: [], steps: 0, lossy: false })); + expect(await main(["learn", "stop", "weather"], h.deps)).toBe(64); + }); + + it("tyto learn status lists typed inputs by name and locator, never values", async () => { + const h = harness(() => ({ ok: true, steps: 3, lossy: false, inputs: [{ name: "input_1", locator: "Search Wikipedia", sensitive: false, step: 2 }] })); + expect(await main(["learn", "status", "wiki-status"], h.deps)).toBe(0); + expect(h.out.join("\n")).toMatch(/input_1\s+Search Wikipedia/); + }); +}); + +describe("learn listener", () => { + it("records after the marker and saves the trace with kept params when stopped", async () => { + const events = new FakeEventSource(); + const runner = new FakeBrowserRunner((): BatchStepResult[] => []); + runner.onRun = (argv) => { + if (argv[1] === "url") events.emit({ type: "command", id: "m", action: "url", params: {} }, { type: "result", id: "m", action: "url", data: {} }); + return { exitCode: 0, stdout: "", stderr: "" }; + }; + const traces = new MemoryTraceStore(); + let handler: ((req: ControlRequest) => Promise) | null = null; + const done = runListener("weather", "s", { + recorder: new Recorder({ events, runner }), + traces, + redactor: new SecretRedactor(), + now: () => 5, + serve: async (h) => { + handler = h; + return async () => undefined; + }, + tracePath: (name) => `/traces/${name}.json`, + }); + await new Promise((r) => setTimeout(r, 10)); + events.emit( + { type: "command", id: "a", action: "navigate", params: { url: "https://weather.test/" } }, + { type: "result", id: "a", action: "navigate", data: {} }, + { type: "command", id: "b", action: "getbylabel", params: { label: "City", subaction: "fill", value: "Paris" } }, + { type: "result", id: "b", action: "getbylabel", data: {} }, + ); + await new Promise((r) => setTimeout(r, 10)); + const status = await handler!({ op: "status" }); + expect(status.inputs?.map((i) => i.name)).toEqual(["input_1"]); + expect(JSON.stringify(status)).not.toContain("Paris"); + const reply = await handler!({ op: "stop", task: "weather in Paris", keep: { input_1: "city" } }); + expect(reply).toMatchObject({ ok: true, steps: 2, params: { city: "Paris" } }); + await done; + expect(traces.traces.get("weather")?.steps[1]?.argv).toEqual(["find", "label", "City", "fill", "{{city}}"]); + expect(traces.traces.get("weather")?.task).toBe("weather in Paris"); + }); +}); + +describe("control socket", () => { + it("client and server exchange messages over a private socket", async () => { + const dir = join(await mkdtemp(join(tmpdir(), "tyto-ctl-")), "learn"); + const path = join(dir, "x.sock"); + const close = await serveControl(path, async (req) => ({ ok: true, steps: req.op === "status" ? 7 : 0, lossy: false, inputs: [] })); + expect((await stat(dir)).mode & 0o777).toBe(0o700); + expect(await requestControl(path, { op: "status" })).toMatchObject({ ok: true, steps: 7 }); + await close(); + await expect(requestControl(path, { op: "status" })).rejects.toThrow(/not recording/); + }); +}); diff --git a/packages/cli/test/live/learn.test.ts b/packages/cli/test/live/learn.test.ts new file mode 100644 index 0000000..2992989 --- /dev/null +++ b/packages/cli/test/live/learn.test.ts @@ -0,0 +1,56 @@ +import { execFile } from "node:child_process"; +import { createServer, type Server } from "node:http"; +import { mkdtemp, readFile, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import type { Trace } from "@tyto/core"; +import { composeDeps } from "../../src/index.ts"; + +// Live: record a real agent-browser session through its event stream (throwaway config, loopback page). +const run = promisify(execFile); +const SESSION = `tyto-live-learn-${process.pid}`; +let server: Server; +let origin = ""; +let home = ""; +const env = (): NodeJS.ProcessEnv => ({ ...process.env, AGENT_BROWSER_CONFIG: join(home, "ab.json"), TYTO_HOME: home }); +const ab = (...args: string[]) => run("agent-browser", ["--session", SESSION, ...args], { env: env() }); + +beforeAll(async () => { + server = createServer((_req, res) => { + res.writeHead(200, { "content-type": "text/html" }); + res.end('Search'); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const addr = server.address(); + origin = `http://127.0.0.1:${typeof addr === "object" && addr ? addr.port : 0}`; + home = await mkdtemp(join(tmpdir(), "tyto-live-learn-")); + await writeFile(join(home, "ab.json"), "{}"); +}); + +afterAll(async () => { + await ab("close").catch(() => undefined); + await new Promise((resolve) => server.close(() => resolve())); +}); + +describe("live learn", () => { + it("records command/result pairs from the event stream and keeps only the named input", async () => { + const deps = await composeDeps(env()); + await ab("get", "url"); + const listening = deps.learn.listen?.("city-search", SESSION); + for (let i = 0; i < 100; i += 1) { + const ready = await deps.learn.control("city-search", { op: "status" }).catch(() => null); + if (ready?.ok) break; + await new Promise((r) => setTimeout(r, 100)); + } + await ab("open", `${origin}/`); + await ab("fill", "#city", "Paris"); + await ab("click", "#go"); + const reply = await deps.learn.control("city-search", { op: "stop", task: "search a city", keep: { input_1: "city" } }); + await listening; + expect(reply).toMatchObject({ ok: true, steps: 3, lossy: false, params: { city: "Paris" } }); + const trace = JSON.parse(await readFile(join(home, "traces", "city-search.json"), "utf8")) as Trace; + expect(trace.steps.map((s) => s.argv)).toEqual([["open", `${origin}/`], ["fill", "#city", "{{city}}"], ["click", "#go"]]); + }, 60_000); +}); diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 524f84a..6758254 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -33,3 +33,8 @@ export { toRequests, type BriefParts, } from "./brief/collect.ts"; +export type { StreamEvent, Trace, TraceInput, TraceStep } from "./trace/types.ts"; +export { actionToArgv } from "./trace/argv.ts"; +export { buildTrace, type TraceOptions } from "./trace/build.ts"; +export { isSensitiveLocator } from "./recipe/lint.ts"; +export { Recorder, type RecorderDeps } from "./trace/recorder.ts"; diff --git a/packages/core/src/ports/browser-events.ts b/packages/core/src/ports/browser-events.ts new file mode 100644 index 0000000..58044d0 --- /dev/null +++ b/packages/core/src/ports/browser-events.ts @@ -0,0 +1,6 @@ +import type { StreamEvent } from "../trace/types.ts"; + +/** agent-browser's per-session event stream. Ends with `{ type: "closed" }` if the socket closes. */ +export interface BrowserEventSource { + subscribe(session: string, signal: AbortSignal): AsyncIterable; +} diff --git a/packages/core/src/ports/index.ts b/packages/core/src/ports/index.ts index b52235f..e601aaa 100644 --- a/packages/core/src/ports/index.ts +++ b/packages/core/src/ports/index.ts @@ -6,3 +6,5 @@ export type { BatchStepResult, BrowserRunner, RunOptions, RunOutput } from "./br export type { RecipeStore, RecipeSummary } from "./recipe-store.ts"; export type { Release, SessionLock } from "./session-lock.ts"; export type { LogMarks } from "./log-marks.ts"; +export type { BrowserEventSource } from "./browser-events.ts"; +export type { TraceStore } from "./trace-store.ts"; diff --git a/packages/core/src/ports/trace-store.ts b/packages/core/src/ports/trace-store.ts new file mode 100644 index 0000000..2e58687 --- /dev/null +++ b/packages/core/src/ports/trace-store.ts @@ -0,0 +1,6 @@ +import type { Trace } from "../trace/types.ts"; + +export interface TraceStore { + save(trace: Trace): Promise; + get(name: string): Promise; +} diff --git a/packages/core/src/recipe/lint.ts b/packages/core/src/recipe/lint.ts index 128077d..74b02ce 100644 --- a/packages/core/src/recipe/lint.ts +++ b/packages/core/src/recipe/lint.ts @@ -23,6 +23,11 @@ const EVAL_FORBIDDEN: ReadonlyArray<[RegExp, string]> = [ const SENSITIVE = /(?:^|[^a-z])(?:pass(?:word|wd|code|phrase)?|token|secret|otp|one[-_ ]?time|cvv|cvc|ssn|pin)(?:[^a-z]|$)|cc-(?:number|csc|exp)|card/i; +/** Locators for password, token, OTP, or card fields — never recorded or replayed with typed values. */ +export function isSensitiveLocator(text: string): boolean { + return SENSITIVE.test(text); +} + const URL_PARAM_ONLY = /^\{\{\s*\w+\s*\|\s*url\s*\}\}$/; function opensAllowedOrigin(url: string, origins: readonly string[]): boolean { diff --git a/packages/core/src/testing/fakes.ts b/packages/core/src/testing/fakes.ts index 5b659fb..c10f088 100644 --- a/packages/core/src/testing/fakes.ts +++ b/packages/core/src/testing/fakes.ts @@ -4,6 +4,9 @@ import type { Release, SessionLock } from "../ports/session-lock.ts"; import type { Recipe } from "../recipe/types.ts"; import type { LogCounts } from "../brief/types.ts"; import type { LogMarks } from "../ports/log-marks.ts"; +import type { BrowserEventSource } from "../ports/browser-events.ts"; +import type { TraceStore } from "../ports/trace-store.ts"; +import type { StreamEvent, Trace } from "../trace/types.ts"; import type { Clock } from "../ports/clock.ts"; import type { ModelPort } from "../ports/model.ts"; import type { CompleteRequest, CompleteResponse } from "../types.ts"; @@ -153,3 +156,44 @@ export class MemoryLogMarks implements LogMarks { this.marks.set(session, counts); } } + +/** Push-driven event stream for tests. `emit` delivers to the current subscriber. */ +export class FakeEventSource implements BrowserEventSource { + private queue: StreamEvent[] = []; + private wake: (() => void) | null = null; + subscribed: string[] = []; + + emit(...events: StreamEvent[]): void { + this.queue.push(...events); + this.wake?.(); + } + + async *subscribe(session: string, signal: AbortSignal): AsyncIterable { + this.subscribed.push(session); + while (!signal.aborted) { + const next = this.queue.shift(); + if (next) { + yield next; + if (next.type === "closed") return; + continue; + } + await new Promise((resolve) => { + this.wake = resolve; + signal.addEventListener("abort", () => resolve(), { once: true }); + }); + this.wake = null; + } + } +} + +export class MemoryTraceStore implements TraceStore { + readonly traces = new Map(); + + async save(trace: Trace): Promise { + this.traces.set(trace.name, trace); + } + + async get(name: string): Promise { + return this.traces.get(name) ?? null; + } +} diff --git a/packages/core/src/testing/index.ts b/packages/core/src/testing/index.ts index 4139421..4a0f2df 100644 --- a/packages/core/src/testing/index.ts +++ b/packages/core/src/testing/index.ts @@ -1,10 +1,12 @@ export { FakeBrowserRunner, FakeClock, + FakeEventSource, FakeModel, FakeSessionLock, MemoryLogMarks, MemoryRecipeStore, + MemoryTraceStore, evalReturns, stepFails, type BatchScript, diff --git a/packages/core/src/trace/argv.ts b/packages/core/src/trace/argv.ts new file mode 100644 index 0000000..86cd19b --- /dev/null +++ b/packages/core/src/trace/argv.ts @@ -0,0 +1,83 @@ +type Params = Readonly>; + +function s(v: unknown): string { + return typeof v === "string" ? v : typeof v === "number" || typeof v === "boolean" ? String(v) : ""; +} + +const FIND_BY: Readonly> = { + getbyrole: ["role", "role"], + getbytext: ["text", "text"], + getbylabel: ["label", "label"], + getbyplaceholder: ["placeholder", "placeholder"], + getbyalttext: ["alt", "text"], + getbytitle: ["title", "text"], + getbytestid: ["testid", "testId"], +}; + +function findTail(p: Params): string[] { + const tail = [s(p.subaction) || "click"]; + if (p.value !== undefined) tail.push(s(p.value)); + if (typeof p.name === "string" && p.name !== "") tail.push("--name", p.name); + if (p.exact === true) tail.push("--exact"); + return tail; +} + +/** Map an agent-browser internal action back to its CLI argv (the form recipes use). Unknown → null. */ +export function actionToArgv(action: string, p: Params): string[] | null { + const by = FIND_BY[action]; + if (by) return ["find", by[0], s(p[by[1]]), ...findTail(p)]; + switch (action) { + case "navigate": + return ["open", s(p.url)]; + case "click": + case "dblclick": + case "check": + case "uncheck": + case "hover": + case "scrollintoview": + return [action, s(p.selector)]; + case "fill": + return ["fill", s(p.selector), s(p.value)]; + case "type": + return ["type", s(p.selector), s(p.text)]; + case "press": + return ["press", s(p.key)]; + case "select": + return ["select", s(p.selector), ...(Array.isArray(p.values) ? p.values.map(s) : [s(p.values)])]; + case "scroll": + return ["scroll", ...[s(p.direction), s(p.amount)].filter(Boolean)]; + case "wait": + if (p.text !== undefined) return ["wait", "--text", s(p.text)]; + if (p.selector !== undefined) return ["wait", s(p.selector)]; + return ["wait", s(p.timeout)]; + case "waitforurl": + return ["wait", "--url", s(p.url)]; + case "waitforloadstate": + return ["wait", "--load", s(p.state)]; + case "waitforfunction": + return ["wait", "--fn", s(p.expression)]; + case "evaluate": + return ["eval", s(p.script)]; + case "url": + return ["get", "url"]; + case "title": + return ["get", "title"]; + case "gettext": + return ["get", "text", s(p.selector)]; + case "getattribute": + return ["get", "attr", s(p.selector), s(p.attribute)]; + case "snapshot": + return ["snapshot", ...(p.interactive === true ? ["-i"] : [])]; + case "back": + case "forward": + case "reload": + return [action]; + case "nth": { + const index = typeof p.index === "number" ? p.index : 0; + const where = index === 0 ? ["first"] : index === -1 ? ["last"] : ["nth", String(index)]; + return ["find", ...where, s(p.selector), ...findTail(p)]; + } + default: + return null; + } +} diff --git a/packages/core/src/trace/build.ts b/packages/core/src/trace/build.ts new file mode 100644 index 0000000..a1ebc56 --- /dev/null +++ b/packages/core/src/trace/build.ts @@ -0,0 +1,139 @@ +import type { Redactor } from "../ports/redactor.ts"; +import { isSensitiveLocator } from "../recipe/lint.ts"; +import { actionToArgv } from "./argv.ts"; +import type { StreamEvent, Trace, TraceInput, TraceStep } from "./types.ts"; + +export type TraceOptions = { name: string; session: string; task?: string; redactor: Redactor; now: () => number; startedAt?: number }; + +const OUTPUT_LIMIT = { snapshot: 4000, result: 1500, other: 600 } as const; +/** Actions whose value the user typed, and the param holding it. */ +const TYPED: Readonly> = { fill: "value", type: "text", keyboard: "text" }; +const FIND_ACTIONS = new Set(["getbyrole", "getbytext", "getbylabel", "getbyplaceholder", "getbyalttext", "getbytitle", "getbytestid", "nth"]); + +function rec(v: unknown): Record { + return typeof v === "object" && v !== null && !Array.isArray(v) ? (v as Record) : {}; +} + +function typedKey(action: string, params: Record): string | null { + if (TYPED[action] && typeof params[TYPED[action] ?? ""] === "string") return TYPED[action] ?? null; + if (FIND_ACTIONS.has(action) && (params.subaction === "fill" || params.subaction === "type") && typeof params.value === "string") return "value"; + return null; +} + +function locatorOf(action: string, params: Record): string { + return [params.selector, params.label, params.placeholder, params.text, params.role, params.name, params.testId] + .filter((v) => typeof v === "string" && v !== "" && v !== params.value) + .join(" ") || action; +} + +function summarize(data: unknown, redactor: Redactor): string { + const { lifecycle: _lifecycle, ...d } = rec(data); + if (typeof d.snapshot === "string") return redactor.safe(d.snapshot).slice(0, OUTPUT_LIMIT.snapshot); + if ("result" in d) { + const r = d.result; + return redactor.safe(typeof r === "string" ? r : JSON.stringify(r) ?? "").slice(0, OUTPUT_LIMIT.result); + } + if (Array.isArray(d.cookies)) { + return JSON.stringify({ cookies: d.cookies.map((c) => ({ name: rec(c).name, length: String(rec(c).value ?? "").length })) }); + } + if (d.data && typeof d.data === "object") { + return JSON.stringify({ storage: Object.fromEntries(Object.entries(rec(d.data)).map(([k, v]) => [k, String(v).length])) }); + } + const text = JSON.stringify(d); + return text === "{}" ? "" : redactor.safe(text).slice(0, OUTPUT_LIMIT.other); +} + +function errorOf(ev: StreamEvent): string | null { + const e = ev.error ?? rec(ev.data).error; + if (e === undefined || e === null || e === "") return null; + return typeof e === "string" ? e : JSON.stringify(e); +} + +function origin(url: unknown): string | null { + if (typeof url !== "string") return null; + try { + const u = new URL(url); + return u.protocol === "http:" || u.protocol === "https:" ? u.origin : null; + } catch { + return null; + } +} + +/** + * Turn recorded stream events into a trace the compiler can read. Typed values become `{{input_N}}`, or + * `{{param}}` when the user kept them via `keep` (input_N → param name); every other value is dropped here. + */ +export function buildTrace( + events: readonly StreamEvent[], + opts: TraceOptions, + keep: Readonly>, +): { trace: Trace; inputs: TraceInput[] } { + const steps: TraceStep[] = []; + const pending = new Map(); + const gaps: string[] = []; + const origins: string[] = []; + const inputs: TraceInput[] = []; + const params: Record = {}; + const addOrigin = (u: unknown): void => { + const o = origin(u); + if (o && !origins.includes(o)) origins.push(o); + }; + + for (const ev of events) { + if (ev.type === "command") { + const action = String(ev.action ?? ""); + const p = { ...rec(ev.params) }; + if (action === "navigate") addOrigin(p.url); + const key = typedKey(action, p); + if (key) { + const locator = locatorOf(action, p); + const name = `input_${inputs.length + 1}`; + const sensitive = isSensitiveLocator(locator); + inputs.push({ name, locator, sensitive, step: steps.length + 1 }); + const kept = keep[name]; + if (kept && !sensitive) { + params[kept] = String(p[key]); + p[key] = `{{${kept}}}`; + } else { + p[key] = sensitive ? "‹secret›" : `{{${name}}}`; + } + } + const argv = actionToArgv(action, p); + steps.push({ argv: argv ? argv.map((a) => (action === "evaluate" ? opts.redactor.safe(a) : a)) : null, action, output: "", error: null }); + pending.set(String(ev.id), steps.length - 1); + } else if (ev.type === "result") { + const index = pending.get(String(ev.id)); + if (index === undefined) { + gaps.push(`result ${String(ev.id)} (${String(ev.action)}) without a command`); + continue; + } + pending.delete(String(ev.id)); + const step = steps[index]; + if (step) { + step.output = summarize(ev.data, opts.redactor); + step.error = errorOf(ev); + } + } else if (ev.type === "url") { + addOrigin(ev.url); + } else if (ev.type === "closed") { + gaps.push("event stream closed while recording"); + } + } + for (const [id, index] of pending) gaps.push(`command ${id} (${steps[index]?.action ?? "?"}) without a result`); + + return { + trace: { + name: opts.name, + task: opts.task ?? "", + session: opts.session, + startedAt: opts.startedAt ?? opts.now(), + stoppedAt: opts.now(), + lossy: gaps.length > 0, + gaps, + origins, + params, + steps, + }, + inputs, + }; +} diff --git a/packages/core/src/trace/recorder.ts b/packages/core/src/trace/recorder.ts new file mode 100644 index 0000000..2851cdf --- /dev/null +++ b/packages/core/src/trace/recorder.ts @@ -0,0 +1,61 @@ +import type { BrowserEventSource } from "../ports/browser-events.ts"; +import type { BrowserRunner } from "../ports/browser-runner.ts"; +import type { StreamEvent } from "./types.ts"; + +export type RecorderDeps = { events: BrowserEventSource; runner: BrowserRunner }; + +const MARKER_TIMEOUT_MS = 10_000; + +/** + * Listens to a session's event stream. After subscribing it runs `get url` as a sync marker and records only + * what happens after the marker's result — proof the stream is live and nothing earlier leaks in. + */ +export class Recorder { + readonly #deps: RecorderDeps; + readonly #buffer: StreamEvent[] = []; + #recording = false; + + constructor(deps: RecorderDeps) { + this.#deps = deps; + } + + events(): StreamEvent[] { + return [...this.#buffer]; + } + + start(session: string, signal: AbortSignal, markerTimeoutMs = MARKER_TIMEOUT_MS): Promise { + return new Promise((resolve, reject) => { + let markerId: string | null = null; + let triggered = false; + const timer = setTimeout(() => { + if (!this.#recording) reject(new Error("agent-browser's event stream did not answer the sync marker")); + }, markerTimeoutMs); + const begin = (): void => { + this.#recording = true; + clearTimeout(timer); + resolve(); + }; + const iterate = async (): Promise => { + try { + for await (const ev of this.#deps.events.subscribe(session, signal)) { + if (this.#recording) { + this.#buffer.push(ev); + continue; + } + if (triggered && markerId === null && ev.type === "command" && ev.action === "url") markerId = String(ev.id); + else if (markerId !== null && ev.type === "result" && String(ev.id) === markerId) begin(); + } + } catch (err) { + clearTimeout(timer); + if (!this.#recording) reject(err instanceof Error ? err : new Error(String(err))); + } + }; + void iterate(); + triggered = true; + this.#deps.runner.run(["get", "url"], { session }).catch((err: unknown) => { + clearTimeout(timer); + reject(err instanceof Error ? err : new Error(String(err))); + }); + }); + } +} diff --git a/packages/core/src/trace/types.ts b/packages/core/src/trace/types.ts new file mode 100644 index 0000000..5a3f80f --- /dev/null +++ b/packages/core/src/trace/types.ts @@ -0,0 +1,30 @@ +/** One message from agent-browser's per-session event stream (`~/.agent-browser/.stream`). */ +export type StreamEvent = { type: string; [key: string]: unknown }; + +export type TraceStep = { + /** The agent-browser command, or null when the streamed action has no CLI equivalent. */ + argv: string[] | null; + /** agent-browser's internal action name, e.g. "navigate", "getbylabel". */ + action: string; + /** Redacted, truncated summary of the result (snapshot text, eval result, …). */ + output: string; + error: string | null; +}; + +export type Trace = { + name: string; + task: string; + session: string; + startedAt: number; + stoppedAt: number; + /** Some events were dropped or unmatched; the compiler must confirm details on the live page. */ + lossy: boolean; + gaps: string[]; + origins: string[]; + /** Typed inputs the user kept as params, with their example values. */ + params: Record; + steps: TraceStep[]; +}; + +/** A typed input seen while recording. Values stay with the recorder; only the name and locator leave. */ +export type TraceInput = { name: string; locator: string; sensitive: boolean; step: number }; diff --git a/packages/core/test/recorder.test.ts b/packages/core/test/recorder.test.ts new file mode 100644 index 0000000..a688342 --- /dev/null +++ b/packages/core/test/recorder.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it } from "vitest"; +import { Recorder, type BatchStepResult } from "../src/index.ts"; +import { FakeBrowserRunner, FakeEventSource } from "../src/testing/index.ts"; + +function setup() { + const events = new FakeEventSource(); + const runner = new FakeBrowserRunner((): BatchStepResult[] => []); + // The sync marker: `get url` shows up on the stream as a url command + result. + runner.onRun = (argv) => { + if (argv[0] === "get" && argv[1] === "url") { + events.emit({ type: "command", id: "marker", action: "url", params: {} }, { type: "result", id: "marker", action: "url", data: {} }); + } + return { exitCode: 0, stdout: "about:blank", stderr: "" }; + }; + return { events, runner, recorder: new Recorder({ events, runner }) }; +} + +describe("Recorder", () => { + it("learn waits for the sync marker before recording", async () => { + const { events, runner, recorder } = setup(); + events.emit({ type: "command", id: "old", action: "click", params: { selector: "#before" } }); + const ac = new AbortController(); + await recorder.start("s", ac.signal); + expect(runner.runs.map((r) => r.argv)).toEqual([["get", "url"]]); + events.emit({ type: "command", id: "c1", action: "click", params: { selector: "#go" } }, { type: "result", id: "c1", action: "click", data: {} }); + await new Promise((r) => setTimeout(r, 10)); + ac.abort(); + const recorded = recorder.events(); + expect(recorded.map((e) => e.id)).toEqual(["c1", "c1"]); + }); + + it("records a closed event when the stream ends", async () => { + const { events, recorder } = setup(); + const ac = new AbortController(); + await recorder.start("s", ac.signal); + events.emit({ type: "closed" }); + await new Promise((r) => setTimeout(r, 10)); + expect(recorder.events().at(-1)).toEqual({ type: "closed" }); + ac.abort(); + }); +}); diff --git a/packages/core/test/trace.test.ts b/packages/core/test/trace.test.ts new file mode 100644 index 0000000..2c2a284 --- /dev/null +++ b/packages/core/test/trace.test.ts @@ -0,0 +1,143 @@ +import { describe, expect, it } from "vitest"; +import { SecretRedactor, actionToArgv, buildTrace, type StreamEvent } from "../src/index.ts"; + +let seq = 0; +function cmd(action: string, params: Record = {}): { command: StreamEvent; id: string } { + seq += 1; + const id = `r${seq}`; + return { id, command: { type: "command", id, action, params: { action, id, ...params }, timestamp: seq } }; +} +function res(id: string, action: string, data: Record = {}): StreamEvent { + return { type: "result", id, action, success: false, data: { lifecycle: { launched: false }, ...data }, duration_ms: 1, timestamp: seq }; +} +/** A command followed by its result. */ +function pair(action: string, params: Record = {}, data: Record = {}): StreamEvent[] { + const c = cmd(action, params); + return [c.command, res(c.id, action, data)]; +} + +const opts = { name: "login", session: "s", redactor: new SecretRedactor(), now: () => 1_000 }; + +describe("actionToArgv", () => { + it("maps navigate, click, fill, type, press, select, check, waits, eval, get, snapshot, and find locators", () => { + const cases: Array<[string, Record, string[]]> = [ + ["navigate", { url: "https://x.test/" }, ["open", "https://x.test/"]], + ["click", { selector: "#go" }, ["click", "#go"]], + ["fill", { selector: "#q", value: "owl" }, ["fill", "#q", "owl"]], + ["type", { selector: "#q", text: "owl" }, ["type", "#q", "owl"]], + ["press", { key: "Enter" }, ["press", "Enter"]], + ["select", { selector: "#size", values: ["m", "l"] }, ["select", "#size", "m", "l"]], + ["check", { selector: "#tos" }, ["check", "#tos"]], + ["waitforurl", { url: "**/done" }, ["wait", "--url", "**/done"]], + ["waitforloadstate", { state: "load" }, ["wait", "--load", "load"]], + ["wait", { text: "Saved" }, ["wait", "--text", "Saved"]], + ["evaluate", { script: "document.title" }, ["eval", "document.title"]], + ["url", {}, ["get", "url"]], + ["gettext", { selector: "h1" }, ["get", "text", "h1"]], + ["getattribute", { selector: "a", attribute: "href" }, ["get", "attr", "a", "href"]], + ["snapshot", { interactive: true }, ["snapshot", "-i"]], + ["getbylabel", { label: "Email", subaction: "fill", value: "a@b.test" }, ["find", "label", "Email", "fill", "a@b.test"]], + ["getbyrole", { role: "button", subaction: "click", name: "Save", exact: true }, ["find", "role", "button", "click", "--name", "Save", "--exact"]], + ["getbyplaceholder", { placeholder: "Search", subaction: "fill", value: "owl" }, ["find", "placeholder", "Search", "fill", "owl"]], + ["nth", { selector: ".card", index: 0, subaction: "click" }, ["find", "first", ".card", "click"]], + ["nth", { selector: ".card", index: 2, subaction: "click" }, ["find", "nth", "2", ".card", "click"]], + ]; + for (const [action, params, argv] of cases) expect(actionToArgv(action, params), action).toEqual(argv); + }); + + it("unknown actions map to null", () => { + expect(actionToArgv("webmcp_invoke", { tool: "x" })).toBeNull(); + }); +}); + +describe("buildTrace", () => { + it("pairs command and result events by id into trace steps", () => { + const events = [...pair("navigate", { url: "https://x.test/login" }), ...pair("click", { selector: "#go" }, { clicked: "#go" })]; + const { trace } = buildTrace(events, opts, {}); + expect(trace.steps.map((s) => s.argv)).toEqual([["open", "https://x.test/login"], ["click", "#go"]]); + expect(trace.lossy).toBe(false); + }); + + it("marks the trace lossy on an orphan result, a missing result, or a socket close", () => { + const orphan = buildTrace([res("nope", "click")], opts, {}).trace; + expect(orphan.lossy).toBe(true); + const missing = buildTrace([cmd("click", { selector: "#go" }).command], opts, {}).trace; + expect(missing.gaps.join(" ")).toMatch(/without a result/); + const closed = buildTrace([...pair("url"), { type: "closed" }], opts, {}).trace; + expect(closed.gaps.join(" ")).toMatch(/closed/); + }); + + it("ignores frame, tabs, and status messages", () => { + const events: StreamEvent[] = [ + { type: "frame", data: "…" }, + { type: "tabs", tabs: [] }, + { type: "status", connected: true }, + ...pair("url"), + ]; + const { trace } = buildTrace(events, opts, {}); + expect(trace.steps).toHaveLength(1); + expect(trace.lossy).toBe(false); + }); + + it("replaces fill and type values with {{input_N}} placeholders", () => { + const events = [ + ...pair("fill", { selector: "input[name=user]", value: "ada" }), + ...pair("getbylabel", { label: "City", subaction: "fill", value: "Paris" }), + ...pair("type", { selector: "#note", text: "hello there" }), + ]; + const { trace, inputs } = buildTrace(events, opts, {}); + expect(trace.steps.map((s) => s.argv)).toEqual([ + ["fill", "input[name=user]", "{{input_1}}"], + ["find", "label", "City", "fill", "{{input_2}}"], + ["type", "#note", "{{input_3}}"], + ]); + expect(JSON.stringify(trace)).not.toMatch(/ada|Paris|hello there/); + expect(inputs.map((i) => i.name)).toEqual(["input_1", "input_2", "input_3"]); + }); + + it("keeps only inputs named with --param and drops the rest", () => { + const events = [...pair("fill", { selector: "#user", value: "ada" }), ...pair("getbylabel", { label: "City", subaction: "fill", value: "Paris" })]; + const { trace } = buildTrace(events, opts, { input_2: "city" }); + expect(trace.steps[1]?.argv).toEqual(["find", "label", "City", "fill", "{{city}}"]); + expect(trace.params).toEqual({ city: "Paris" }); + expect(JSON.stringify(trace)).not.toMatch(/ada/); + }); + + it("masks values typed into password-like fields and never offers them as params", () => { + const events = [...pair("fill", { selector: "input[name=password]", value: "hunter2hunter2" })]; + const { trace, inputs } = buildTrace(events, opts, { input_1: "pw" }); + expect(JSON.stringify(trace)).not.toContain("hunter2hunter2"); + expect(trace.params).toEqual({}); + expect(inputs[0]?.sensitive).toBe(true); + }); + + it("reduces cookie and storage values to names and lengths", () => { + const events = [ + ...pair("cookies_get", {}, { cookies: [{ name: "sid", value: "SESSIONVALUE123", httpOnly: true }] }), + ...pair("storage_get", { type: "local" }, { data: { token: "abcdefabcdef", theme: "dark" } }), + ]; + const text = JSON.stringify(buildTrace(events, opts, {}).trace); + expect(text).not.toMatch(/SESSIONVALUE123|abcdefabcdef/); + expect(text).toMatch(/sid/); + }); + + it("runs the Redactor before the trace is returned", () => { + const events = [...pair("evaluate", { script: "document.body.innerText" }, { result: "Authorization: Bearer abcdefghijklmnopqrstuvwxyz" })]; + expect(JSON.stringify(buildTrace(events, opts, {}).trace)).not.toContain("abcdefghijklmnopqrstuvwxyz"); + }); + + it("keeps snapshot text and eval results as step output", () => { + const events = [ + ...pair("snapshot", { interactive: true }, { snapshot: '- button "Save" [ref=e2]' }), + ...pair("evaluate", { script: "document.title" }, { result: "Owl" }), + ]; + const { trace } = buildTrace(events, opts, {}); + expect(trace.steps[0]?.output).toContain('button "Save"'); + expect(trace.steps[1]?.output).toContain("Owl"); + }); + + it("collects the origins the task opened", () => { + const events: StreamEvent[] = [...pair("navigate", { url: "https://a.test/x" }), { type: "url", url: "https://login.b.test/sso", timestamp: 9 }]; + expect(buildTrace(events, opts, {}).trace.origins).toEqual(["https://a.test", "https://login.b.test"]); + }); +}); diff --git a/packages/store/src/index.ts b/packages/store/src/index.ts index 2c3d6e2..ae64243 100644 --- a/packages/store/src/index.ts +++ b/packages/store/src/index.ts @@ -3,3 +3,4 @@ export { FilesystemRecipeStore } from "./recipe-store.ts"; export { FileSessionLock } from "./session-lock.ts"; export { DEFAULT_USER_CONFIG, ensureReplayFiles } from "./replay-files.ts"; export { FileLogMarks } from "./log-marks.ts"; +export { FileTraceStore } from "./trace-store.ts"; diff --git a/packages/store/src/trace-store.ts b/packages/store/src/trace-store.ts new file mode 100644 index 0000000..ac5c87b --- /dev/null +++ b/packages/store/src/trace-store.ts @@ -0,0 +1,25 @@ +import { join } from "node:path"; +import type { Trace, TraceStore } from "@tyto/core"; +import { readIfExists, writePrivate } from "./files.ts"; + +const NAME = /^[a-z0-9]+(?:-[a-z0-9]+)*$/; + +/** Traces as `/.json`, 0600. Traces never contain typed values the user did not keep. */ +export class FileTraceStore implements TraceStore { + readonly #dir: string; + + constructor(dir: string) { + this.#dir = dir; + } + + async save(trace: Trace): Promise { + if (!NAME.test(trace.name)) throw new Error(`invalid trace name: ${trace.name}`); + await writePrivate(join(this.#dir, `${trace.name}.json`), `${JSON.stringify(trace, null, 2)}\n`); + } + + async get(name: string): Promise { + if (!NAME.test(name)) throw new Error(`invalid trace name: ${name}`); + const text = await readIfExists(join(this.#dir, `${name}.json`)); + return text === null ? null : (JSON.parse(text) as Trace); + } +} diff --git a/packages/store/test/store.test.ts b/packages/store/test/store.test.ts index fc84456..2eff62b 100644 --- a/packages/store/test/store.test.ts +++ b/packages/store/test/store.test.ts @@ -3,7 +3,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; import { parseRecipe, type Recipe } from "@tyto/core"; -import { FileLogMarks, FileSessionLock, FilesystemRecipeStore, ensureReplayFiles } from "../src/index.ts"; +import { FileLogMarks, FileSessionLock, FileTraceStore, FilesystemRecipeStore, ensureReplayFiles } from "../src/index.ts"; function recipe(name = "release", overrides: Record = {}): Recipe { const parsed = parseRecipe({ @@ -116,3 +116,15 @@ describe("FileLogMarks", () => { await expect(new FileLogMarks(await tmp()).get("../x")).rejects.toThrow(/session/); }); }); + +describe("FileTraceStore", () => { + it("saves traces with mode 0600 and loads them by name", async () => { + const dir = await tmp(); + const store = new FileTraceStore(dir); + const trace = { name: "login", task: "log in", session: "s", startedAt: 1, stoppedAt: 2, lossy: false, gaps: [], origins: ["https://x.test"], params: {}, steps: [] }; + await store.save(trace); + expect(await store.get("login")).toEqual(trace); + expect(await store.get("other")).toBeNull(); + expect((await stat(join(dir, "login.json"))).mode & 0o777).toBe(0o600); + }); +});