From f59e49286449ac3d9a91164e9c29a338f5fae9c6 Mon Sep 17 00:00:00 2001 From: Sean C Davis Date: Fri, 4 Sep 2026 09:44:19 -0400 Subject: [PATCH 1/6] fix(adapters): add promptVia so a spec can deliver the prompt over stdin Adds an optional promptVia?: "argv" | "stdin" field to AgentAdapterSpec, defaulting to today's argv behavior. When set to "stdin", the base writes input.prompt to the child's stdin instead of leaving buildArgs to place it on the command line, which is what will let claude-code/codex avoid crashing on null bytes in a later slice. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01S6cesfFBiaNe5ttYyXpLce --- src/adapters/base/agent-adapter.ts | 16 ++++++- test/unit/adapters/base/agent-adapter.test.ts | 45 ++++++++++++++++++- 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/src/adapters/base/agent-adapter.ts b/src/adapters/base/agent-adapter.ts index 2b91e01..95fbdff 100644 --- a/src/adapters/base/agent-adapter.ts +++ b/src/adapters/base/agent-adapter.ts @@ -133,6 +133,15 @@ export type AgentAdapterSpec = { /** Build the CLI arguments for the agent process. Prefix args from command resolution are prepended automatically. */ buildArgs: (input: AgentInput) => string[]; + /** + * How the prompt reaches the CLI. + * - "argv" (default): `buildArgs` places `input.prompt` on the command line. + * - "stdin": the base writes `input.prompt` to the child's stdin and closes it; + * `buildArgs` must NOT include the prompt. Use this for CLIs that read the + * prompt from a pipe — argv rejects null bytes and caps argument length. + */ + promptVia?: "argv" | "stdin"; + /** Per-run mutable state. Called once per run to create a fresh state bag for `streamConfig` handlers and `getResult`. */ initialState: () => State; @@ -250,7 +259,12 @@ export function createAgentAdapter(spec: AgentAdapterSpec): AgentA env: input.env ?? { ...process.env }, }); - child.stdin?.end(); + if (spec.promptVia === "stdin") { + child.stdin?.on("error", () => {}); // child exited before reading; exit path reports it + child.stdin?.end(input.prompt); + } else { + child.stdin?.end(); + } // 6. Cleanup handler for Ctrl-C input.registerCleanup?.(() => { diff --git a/test/unit/adapters/base/agent-adapter.test.ts b/test/unit/adapters/base/agent-adapter.test.ts index 2a557d0..2d13f2d 100644 --- a/test/unit/adapters/base/agent-adapter.test.ts +++ b/test/unit/adapters/base/agent-adapter.test.ts @@ -24,7 +24,7 @@ function createMockProcess(opts: { const { stdout: stdoutLines = [], stderr: stderrLines = [], exitCode = 0, delayMs = 0, hang = false } = opts; const stdout = new Readable({ read() {} }); const stderr = new Readable({ read() {} }); - const stdin = { end: vi.fn() }; + const stdin = { end: vi.fn(), on: vi.fn() }; const proc = Object.assign(new EventEmitter(), { stdout, stderr, stdin, kill: vi.fn() }); setTimeout(() => { @@ -424,6 +424,49 @@ describe("createAgentAdapter", () => { expect(out.metadata.error?.length).toBeLessThan(200_000); }); + it("promptVia: stdin writes the prompt to child.stdin and resolves without throwing", async () => { + const fakeChild = createMockProcess({ stdout: ["ok\n"] }); + mockSpawn.mockImplementation((() => fakeChild) as any); + + const adapter = createAgentAdapter<{ r: string | null }>({ + name: "stdin-test", + cliCommand: "test-bin", + buildArgs: () => ["--flag"], + promptVia: "stdin", + initialState: () => ({ r: null }), + streamConfig: { mode: "lines", onLine: (line, ctx) => (ctx.state.r = line) }, + getResult: (ctx) => ({ result: ctx.state.r }), + }); + + const prompt = "hello\0world"; + await expect(adapter.run(makeInput({ prompt }))).resolves.not.toThrow(); + + expect(fakeChild.stdin.end).toHaveBeenCalledWith(prompt); + }); + + it("promptVia omitted: stdin.end is called with no data and argv is untouched", async () => { + const fakeChild = createMockProcess({ stdout: ["ok\n"] }); + let captured: string[] = []; + mockSpawn.mockImplementation(((_cmd: string, args: string[]) => { + captured = args; + return fakeChild; + }) as any); + + const adapter = createAgentAdapter<{ r: string | null }>({ + name: "argv-test", + cliCommand: "test-bin", + buildArgs: () => ["--flag"], + initialState: () => ({ r: null }), + streamConfig: { mode: "lines", onLine: (line, ctx) => (ctx.state.r = line) }, + getResult: (ctx) => ({ result: ctx.state.r }), + }); + + await adapter.run(makeInput({ prompt: "hello\0world" })); + + expect(fakeChild.stdin.end).toHaveBeenCalledWith(); + expect(captured).toEqual(["--flag"]); + }); + it("custom resolveCommand overrides default resolution", async () => { let usedCmd = ""; mockSpawn.mockImplementation(((cmd: string) => { From 27792790a5bf6da43149cad773c351b33ca068c3 Mon Sep 17 00:00:00 2001 From: Sean C Davis Date: Fri, 4 Sep 2026 09:47:39 -0400 Subject: [PATCH 2/6] fix(adapters): claude-code and codex deliver the prompt over stdin Both adapters now set promptVia: "stdin" and no longer place input.prompt on the command line: claude-code drops the -p positional (claude -p reads stdin when no positional is given) and codex drops the trailing args.push(input.prompt) (codex exec reads stdin when no [PROMPT] is given). Keeps a null byte or control character in a transcript-derived prompt from crashing spawn() via argv. Updates claude-code.test.ts and codex.test.ts to assert the prompt is absent from argv and delivered via stdin.end(prompt) instead, and adds the on: vi.fn() stub to the fake stdin in codex-e2e/mcp-e2e/skills-e2e tests so the base adapter's stdin error listener doesn't throw against a mock that previously only had `end`. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01S6cesfFBiaNe5ttYyXpLce --- src/adapters/claude-code.ts | 3 ++- src/adapters/codex.ts | 3 +-- test/unit/adapters/claude-code.test.ts | 21 ++++++++++++++++++++- test/unit/adapters/codex-e2e.test.ts | 2 +- test/unit/adapters/codex.test.ts | 14 +++++++++----- test/unit/adapters/mcp-e2e.test.ts | 2 +- test/unit/adapters/skills-e2e.test.ts | 2 +- 7 files changed, 35 insertions(+), 12 deletions(-) diff --git a/src/adapters/claude-code.ts b/src/adapters/claude-code.ts index f616edf..335d33a 100644 --- a/src/adapters/claude-code.ts +++ b/src/adapters/claude-code.ts @@ -15,6 +15,7 @@ export function createClaudeCodeAdapter(): AgentAdapter { return createAgentAdapter({ name: "claude-code", cliCommand: "claude", + promptVia: "stdin", requiredEnv: () => ["ANTHROPIC_API_KEY"], @@ -77,7 +78,7 @@ export function createClaudeCodeAdapter(): AgentAdapter { // Default dangerously-skip-permissions to true — AXIS runs agents headlessly const skipPermissions = flags["dangerously-skip-permissions"] ?? true; - const args = ["-p", input.prompt, "--output-format", "stream-json", "--verbose"]; + const args = ["-p", "--output-format", "stream-json", "--verbose"]; if (skipPermissions) args.push("--dangerously-skip-permissions"); if (input.config.model) args.push("--model", input.config.model); diff --git a/src/adapters/codex.ts b/src/adapters/codex.ts index fe64042..7c46eba 100644 --- a/src/adapters/codex.ts +++ b/src/adapters/codex.ts @@ -15,6 +15,7 @@ export function createCodexAdapter(): AgentAdapter { return createAgentAdapter({ name: "codex", cliCommand: "codex", + promptVia: "stdin", requiredEnv: () => ["CODEX_API_KEY"], @@ -77,8 +78,6 @@ export function createCodexAdapter(): AgentAdapter { } } - // Prompt is the final positional argument - args.push(input.prompt); return args; }, diff --git a/test/unit/adapters/claude-code.test.ts b/test/unit/adapters/claude-code.test.ts index 12843b0..073115c 100644 --- a/test/unit/adapters/claude-code.test.ts +++ b/test/unit/adapters/claude-code.test.ts @@ -26,7 +26,7 @@ const mockSpawn = vi.mocked(spawn); function createMockProcess(lines: string[], exitCode = 0) { const stdout = new Readable({ read() {} }); const stderr = new Readable({ read() {} }); - const stdin = { end: vi.fn() }; + const stdin = { end: vi.fn(), on: vi.fn() }; const proc = Object.assign(new EventEmitter(), { stdout, stderr, stdin }); // Push lines async so readline can consume them @@ -219,6 +219,25 @@ describe("ClaudeCodeAdapter", () => { expect(output.rawOutput).toBeUndefined(); }); + it("sends the prompt over stdin instead of argv", async () => { + let capturedArgs: string[] = []; + let proc: ReturnType; + + mockSpawn.mockImplementation(((_cmd: string, args: string[]) => { + capturedArgs = args as string[]; + proc = createMockProcess([JSON.stringify({ type: "result", result: "ok" })]); + return proc; + }) as any); + + const prompt = "do the thing\0with a null byte"; + await adapter.run(makeInput(prompt)); + + expect(capturedArgs[0]).toBe("-p"); + expect(capturedArgs[1]).toBe("--output-format"); + expect(capturedArgs).not.toContain(prompt); + expect(proc!.stdin.end).toHaveBeenCalledWith(prompt); + }); + it("passes --strict-mcp-config so only AXIS-declared MCP servers are used", async () => { let capturedArgs: string[] = []; diff --git a/test/unit/adapters/codex-e2e.test.ts b/test/unit/adapters/codex-e2e.test.ts index e825534..853598b 100644 --- a/test/unit/adapters/codex-e2e.test.ts +++ b/test/unit/adapters/codex-e2e.test.ts @@ -24,7 +24,7 @@ const mockSpawn = vi.mocked(spawn); function createMockProcess(lines: string[], exitCode = 0) { const stdout = new Readable({ read() {} }); const stderr = new Readable({ read() {} }); - const stdin = { end: vi.fn() }; + const stdin = { end: vi.fn(), on: vi.fn() }; const proc = Object.assign(new EventEmitter(), { stdout, stderr, stdin, kill: vi.fn() }); setTimeout(() => { diff --git a/test/unit/adapters/codex.test.ts b/test/unit/adapters/codex.test.ts index f71fd26..39c9bd0 100644 --- a/test/unit/adapters/codex.test.ts +++ b/test/unit/adapters/codex.test.ts @@ -23,7 +23,7 @@ const mockSpawn = vi.mocked(spawn); function createMockProcess(lines: string[], exitCode = 0) { const stdout = new Readable({ read() {} }); const stderr = new Readable({ read() {} }); - const stdin = { end: vi.fn() }; + const stdin = { end: vi.fn(), on: vi.fn() }; const proc = Object.assign(new EventEmitter(), { stdout, stderr, stdin, kill: vi.fn() }); setTimeout(() => { @@ -265,21 +265,25 @@ describe("CodexAdapter", () => { expect(capturedArgs).not.toContain("--color"); }); - it("puts prompt as last positional argument after exec --json", async () => { + it("sends the prompt over stdin instead of argv", async () => { let capturedArgs: string[] = []; + let proc: ReturnType; mockSpawn.mockImplementation(((_cmd: string, args: string[]) => { capturedArgs = args as string[]; - return createMockProcess([ + proc = createMockProcess([ JSON.stringify({ type: "item.completed", item: { type: "agent_message", text: "ok" } }), ]); + return proc; }) as any); - await adapter.run(makeInput("do the thing")); + const prompt = "do the thing\0with a null byte"; + await adapter.run(makeInput(prompt)); expect(capturedArgs[0]).toBe("exec"); expect(capturedArgs[1]).toBe("--json"); - expect(capturedArgs[capturedArgs.length - 1]).toBe("do the thing"); + expect(capturedArgs).not.toContain(prompt); + expect(proc!.stdin.end).toHaveBeenCalledWith(prompt); }); it("uses last agent_message when multiple are emitted", async () => { diff --git a/test/unit/adapters/mcp-e2e.test.ts b/test/unit/adapters/mcp-e2e.test.ts index f2eae18..59c866e 100644 --- a/test/unit/adapters/mcp-e2e.test.ts +++ b/test/unit/adapters/mcp-e2e.test.ts @@ -23,7 +23,7 @@ const mockSpawn = vi.mocked(spawn); function createMockProcess(lines: string[], exitCode = 0) { const stdout = new Readable({ read() {} }); const stderr = new Readable({ read() {} }); - const stdin = { end: vi.fn() }; + const stdin = { end: vi.fn(), on: vi.fn() }; const proc = Object.assign(new EventEmitter(), { stdout, stderr, stdin, kill: vi.fn() }); setTimeout(() => { diff --git a/test/unit/adapters/skills-e2e.test.ts b/test/unit/adapters/skills-e2e.test.ts index e7ee469..3647ea9 100644 --- a/test/unit/adapters/skills-e2e.test.ts +++ b/test/unit/adapters/skills-e2e.test.ts @@ -49,7 +49,7 @@ const mockSpawn = vi.mocked(spawn); function createMockProcess(lines: string[], exitCode = 0) { const stdout = new Readable({ read() {} }); const stderr = new Readable({ read() {} }); - const stdin = { end: vi.fn() }; + const stdin = { end: vi.fn(), on: vi.fn() }; const proc = Object.assign(new EventEmitter(), { stdout, stderr, stdin, kill: vi.fn() }); setTimeout(() => { From d99d23174b7610cb6104faddcbcd4a19a2378835 Mon Sep 17 00:00:00 2001 From: Sean C Davis Date: Fri, 4 Sep 2026 09:49:47 -0400 Subject: [PATCH 3/6] fix(scoring): strip control characters from judge prompt values Interpolated {{key}} values in judge prompts are agent output and may contain raw control bytes (e.g. a null byte from binary output), which argv rejects and judge CLIs can't render. interpolate() now strips C0 controls (except tab/newline/CR) and DEL from substituted values only; template text and the missing-variable throw are unchanged. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01S6cesfFBiaNe5ttYyXpLce --- src/scoring/prompt-templates.ts | 10 +++++++++- test/unit/scoring/prompt-templates.test.ts | 8 ++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/scoring/prompt-templates.ts b/src/scoring/prompt-templates.ts index b482b54..830309d 100644 --- a/src/scoring/prompt-templates.ts +++ b/src/scoring/prompt-templates.ts @@ -41,6 +41,14 @@ export interface PromptTemplate { // Interpolation // --------------------------------------------------------------------------- +/** + * C0 control characters (except `\t`, `\n`, `\r`) and DEL. Substituted values + * are agent output — a judge CLI can't render raw control bytes, and argv + * rejects null bytes outright — so they're stripped before interpolation. + */ +// eslint-disable-next-line no-control-regex -- intentionally matching control characters +const CONTROL_CHARS = /[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g; + /** * Replace `{{key}}` placeholders in `template` with values from `vars`. * @@ -52,7 +60,7 @@ export function interpolate(template: string, vars: Record { it("handles empty string variable value", () => { expect(interpolate("before{{gap}}after", { gap: "" })).toBe("beforeafter"); }); + + it("strips control characters from a substituted value", () => { + expect(interpolate("{{v}}", { v: "a\0b\x01c\x7F" })).toBe("abc"); + }); + + it("preserves tab, newline, and carriage return in a substituted value", () => { + expect(interpolate("{{v}}", { v: "x\ny\tz\r" })).toBe("x\ny\tz\r"); + }); }); describe("getPromptTemplates", () => { From bec888f5cc7bb347358ea6e0aabb4f3d3f020db6 Mon Sep 17 00:00:00 2001 From: Sean C Davis Date: Fri, 4 Sep 2026 09:54:03 -0400 Subject: [PATCH 4/6] docs: document promptVia on the agent adapter contract Adds the promptVia?: "argv" | "stdin" spec field to the running.astro custom-adapter example and the AGENTS.md adapter-spec table, and amends the "Spawn + stdin.end" bullet to describe both modes - completing the docs slice of EX-3026 (prompt via stdin). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01S6cesfFBiaNe5ttYyXpLce --- AGENTS.md | 31 ++++++++++++++------------- src/docs-site/src/pages/running.astro | 10 +++++++++ 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index acf9c12..61c2391 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -31,7 +31,7 @@ AXIS (Agent Experience Index Score) is a synthetic testing framework for AI agen Built-in adapters split into two factories. NDJSON-style adapters (`claude-code`, `codex`) are created via `createAgentAdapter(spec)` from `src/adapters/base/agent-adapter.ts`. ACP-based adapters (`claude-sdk`, `codex-sdk`, `gemini`, `goose`, `opencode`, `qwen-code`, `stakpak`, `blackbox`, `fast-agent`, `mistral-vibe`, `factory-droid`, `poolside`, `vtcode`, `cursor-agent`, `auggie`, `kimi`, `openhands`, `cline`, `kiro-cli`, `kilo`, `qoder`) are created via `createAcpBasedAdapter(spec)` from `src/adapters/base/acp-adapter.ts`. Each adapter is a plain factory function (e.g. `createGeminiAdapter()`) that returns an `AgentAdapter` -no classes, no inheritance. The factory owns the shared plumbing: -- Spawn + stdin.end + cleanup registration (SIGTERM on Ctrl-C) +- Spawn + cleanup registration (SIGTERM on Ctrl-C); stdin is closed immediately by default, or written with the prompt then closed when `promptVia: "stdin"` - 10-minute timeout → SIGTERM → SIGKILL after 5s grace (timer cleared on clean exit) - stderr capped at 100 KB - `close` event listener registered BEFORE stdout stream to avoid missing it @@ -46,20 +46,21 @@ The NDJSON-style adapters (`claude-code`, `codex`) use `lines` mode for NDJSON p Call `createAgentAdapter(spec)` with an `AgentAdapterSpec`. The spec is a single typed object -no class inheritance, no protected hooks: -| Spec field | Purpose | -| ------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `name` | Adapter name (registered in `src/adapters/registry.ts`) | -| `cliCommand?` | CLI binary for `resolveCommand`; omit if user-supplied | -| `timeoutMs?` | Execution timeout (default 10 min) | -| `requiredEnv?` | Env vars validated by the runner pre-flight (e.g. `ANTHROPIC_API_KEY`) | -| `hasLocalSession?` | Detect a usable local CLI login (e.g. `claude login`, `codex login`). Runner calls this only when `requiredEnv` is missing — explicit API keys always win | -| `isolationEnv?` | Isolation vars (e.g. `CLAUDE_CONFIG_DIR`, `CODEX_HOME`). Signature: `({ workspace, home }) => Record`. Point `*_HOME`-style paths under `home`, never `workspace` | -| `prepare?` | Side effects (mkdir, MCP / skills writers) before spawn | -| `resolveCommand?` | Override how the CLI command is resolved | -| `buildArgs` | Build CLI arguments (prefix args from command resolution prepended automatically) | -| `initialState` | Per-run mutable state used by `streamConfig` handlers and `getResult` | -| `streamConfig` | How to process agent stdout. Discriminated union: `{ mode: "lines", onLine, onEnd? }` or `{ mode: "aggregate", onChunk, onEnd? }` | -| `getResult` | Build final `{ result, metadata? }` from accumulated state after exit | +| Spec field | Purpose | +| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `name` | Adapter name (registered in `src/adapters/registry.ts`) | +| `cliCommand?` | CLI binary for `resolveCommand`; omit if user-supplied | +| `timeoutMs?` | Execution timeout (default 10 min) | +| `requiredEnv?` | Env vars validated by the runner pre-flight (e.g. `ANTHROPIC_API_KEY`) | +| `hasLocalSession?` | Detect a usable local CLI login (e.g. `claude login`, `codex login`). Runner calls this only when `requiredEnv` is missing — explicit API keys always win | +| `isolationEnv?` | Isolation vars (e.g. `CLAUDE_CONFIG_DIR`, `CODEX_HOME`). Signature: `({ workspace, home }) => Record`. Point `*_HOME`-style paths under `home`, never `workspace` | +| `prepare?` | Side effects (mkdir, MCP / skills writers) before spawn | +| `resolveCommand?` | Override how the CLI command is resolved | +| `buildArgs` | Build CLI arguments (prefix args from command resolution prepended automatically) | +| `promptVia?` | How the prompt reaches the CLI: `"argv"` (default) -`buildArgs` places `input.prompt` on the command line; `"stdin"` -the base writes `input.prompt` to the child's stdin and closes it, and `buildArgs` must omit the prompt. `claude-code` and `codex` use `"stdin"`: argv rejects null bytes and caps argument length, and agent transcripts can contain either | +| `initialState` | Per-run mutable state used by `streamConfig` handlers and `getResult` | +| `streamConfig` | How to process agent stdout. Discriminated union: `{ mode: "lines", onLine, onEnd? }` or `{ mode: "aggregate", onChunk, onEnd? }` | +| `getResult` | Build final `{ result, metadata? }` from accumulated state after exit | The `streamConfig` field uses a discriminated union so the mode and its handler can never get out of sync -no runtime assertions needed. `getResult` returns `null` for "no result" (never `""`). Metadata overrides (e.g. upstream `durationMs`) are spread on top of base-computed fields. diff --git a/src/docs-site/src/pages/running.astro b/src/docs-site/src/pages/running.astro index 474772d..9f77404 100644 --- a/src/docs-site/src/pages/running.astro +++ b/src/docs-site/src/pages/running.astro @@ -63,6 +63,16 @@ export default createAgentAdapter<{ stdout: string }>({ result: ctx.state.stdout.trim() || null, }), });`} +

+ The example above uses the default promptVia: "argv" behavior: buildArgs + places input.prompt on the command line. Set promptVia: "stdin" and AXIS + writes the prompt to the process's stdin and closes it instead -in that mode + buildArgs must omit the prompt. The built-in claude-code and + codex adapters use "stdin": argv rejects null bytes and caps argument + length, and agent transcripts can contain either. +

+ 
{`promptVia: "stdin",
+buildArgs: (input) => [/* prompt omitted -delivered via stdin */],`}

Register it in axis.config.json:

From 992ec3d4bfa526ff31d5ab66a68115cf5928d00d Mon Sep 17 00:00:00 2001 From: Sean C Davis Date: Fri, 4 Sep 2026 10:03:00 -0400 Subject: [PATCH 5/6] refactor: tighten comments and reuse the adapter test fixture Fix three low findings from an audit of the prompt-via-stdin change: - CONTROL_CHARS doc comment overclaimed that substituted values are always agent output; interpolate() also substitutes scenario metadata, prompts, counts, criteria, and paths. - The two new promptVia tests in agent-adapter.test.ts duplicated the createLinesTestAdapter() fixture inline; createLinesTestAdapter now accepts an optional spec override so both tests reuse it. - running.astro's custom-adapter example left an unused `input` parameter and a comment narrating an empty array. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01S6cesfFBiaNe5ttYyXpLce --- src/docs-site/src/pages/running.astro | 2 +- src/scoring/prompt-templates.ts | 7 +++-- test/unit/adapters/base/agent-adapter.test.ts | 31 +++++++------------ 3 files changed, 17 insertions(+), 23 deletions(-) diff --git a/src/docs-site/src/pages/running.astro b/src/docs-site/src/pages/running.astro index 9f77404..9bdb046 100644 --- a/src/docs-site/src/pages/running.astro +++ b/src/docs-site/src/pages/running.astro @@ -72,7 +72,7 @@ export default createAgentAdapter<{ stdout: string }>({ length, and agent transcripts can contain either. 

{`promptVia: "stdin",
-buildArgs: (input) => [/* prompt omitted -delivered via stdin */],`}
+buildArgs: () => [],`}

Register it in axis.config.json:

diff --git a/src/scoring/prompt-templates.ts b/src/scoring/prompt-templates.ts index 830309d..d7025cd 100644 --- a/src/scoring/prompt-templates.ts +++ b/src/scoring/prompt-templates.ts @@ -42,9 +42,10 @@ export interface PromptTemplate { // --------------------------------------------------------------------------- /** - * C0 control characters (except `\t`, `\n`, `\r`) and DEL. Substituted values - * are agent output — a judge CLI can't render raw control bytes, and argv - * rejects null bytes outright — so they're stripped before interpolation. + * C0 control characters (except `\t`, `\n`, `\r`) and DEL. Some substituted + * values carry agent output, which can contain raw control bytes; a judge + * CLI can't render them and argv rejects null bytes outright, so they're + * stripped from every value at interpolation. */ // eslint-disable-next-line no-control-regex -- intentionally matching control characters const CONTROL_CHARS = /[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g; diff --git a/test/unit/adapters/base/agent-adapter.test.ts b/test/unit/adapters/base/agent-adapter.test.ts index 2d13f2d..00383a9 100644 --- a/test/unit/adapters/base/agent-adapter.test.ts +++ b/test/unit/adapters/base/agent-adapter.test.ts @@ -1,7 +1,11 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { EventEmitter, Readable } from "node:stream"; import type { AgentAdapter, AgentInput, AgentMetadata } from "../../../../src/types/agent.js"; -import { createAgentAdapter, type SetupContext } from "../../../../src/adapters/base/agent-adapter.js"; +import { + createAgentAdapter, + type AgentAdapterSpec, + type SetupContext, +} from "../../../../src/adapters/base/agent-adapter.js"; vi.mock("node:child_process", () => ({ spawn: vi.fn(), @@ -62,7 +66,9 @@ let setupCalls: SetupContext[] = []; let getResultCalls = 0; let resultOverride: Partial | null = null; -function createLinesTestAdapter(): AgentAdapter { +function createLinesTestAdapter( + overrides: Partial> = {}, +): AgentAdapter { setupCalls = []; getResultCalls = 0; resultOverride = null; @@ -96,6 +102,8 @@ function createLinesTestAdapter(): AgentAdapter { metadata: resultOverride ?? {}, }; }, + + ...overrides, }); } @@ -428,15 +436,7 @@ describe("createAgentAdapter", () => { const fakeChild = createMockProcess({ stdout: ["ok\n"] }); mockSpawn.mockImplementation((() => fakeChild) as any); - const adapter = createAgentAdapter<{ r: string | null }>({ - name: "stdin-test", - cliCommand: "test-bin", - buildArgs: () => ["--flag"], - promptVia: "stdin", - initialState: () => ({ r: null }), - streamConfig: { mode: "lines", onLine: (line, ctx) => (ctx.state.r = line) }, - getResult: (ctx) => ({ result: ctx.state.r }), - }); + const adapter = createLinesTestAdapter({ promptVia: "stdin" }); const prompt = "hello\0world"; await expect(adapter.run(makeInput({ prompt }))).resolves.not.toThrow(); @@ -452,14 +452,7 @@ describe("createAgentAdapter", () => { return fakeChild; }) as any); - const adapter = createAgentAdapter<{ r: string | null }>({ - name: "argv-test", - cliCommand: "test-bin", - buildArgs: () => ["--flag"], - initialState: () => ({ r: null }), - streamConfig: { mode: "lines", onLine: (line, ctx) => (ctx.state.r = line) }, - getResult: (ctx) => ({ result: ctx.state.r }), - }); + const adapter = createLinesTestAdapter(); await adapter.run(makeInput({ prompt: "hello\0world" })); From 52b55527479fb4e501d1677e7ee7d44c74fa1b35 Mon Sep 17 00:00:00 2001 From: Sean C Davis Date: Fri, 4 Sep 2026 10:10:04 -0400 Subject: [PATCH 6/6] docs(adapters): correct the stdin error-listener comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The old comment implied the exit path reports the stdin error itself; it doesn't — it only reports the child's exit code/stderr. Clarify that the listener's job is just to prevent an unhandled stream error. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01S6cesfFBiaNe5ttYyXpLce --- src/adapters/base/agent-adapter.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/adapters/base/agent-adapter.ts b/src/adapters/base/agent-adapter.ts index 95fbdff..2b41ae5 100644 --- a/src/adapters/base/agent-adapter.ts +++ b/src/adapters/base/agent-adapter.ts @@ -260,7 +260,7 @@ export function createAgentAdapter(spec: AgentAdapterSpec): AgentA }); if (spec.promptVia === "stdin") { - child.stdin?.on("error", () => {}); // child exited before reading; exit path reports it + child.stdin?.on("error", () => {}); // Prevent an unhandled stream error if the child closes stdin early. child.stdin?.end(input.prompt); } else { child.stdin?.end();