From 87266bfbb71f6a0868bacf286f10520639196e80 Mon Sep 17 00:00:00 2001 From: Francis Eytan Dortort Date: Fri, 31 Jul 2026 07:59:10 -0400 Subject: [PATCH 1/3] feat(claude): support --plugin-dir and extra CLI args in the driver The Claude driver built a fixed argv with no way to load an additional plugin directory, so agent CLIs under test could not exercise Claude Code plugins. Add `pluginDir` (maps to `--plugin-dir`) and a generic `extraArgs` passthrough to RunOptions/AgentRunExtra, thread them through buildArgs, and export buildArgs to unit-test the argv assembly. Constraint: RunOptions is the driver-agnostic seam; extraArgs keeps future flag needs from reopening it. Rejected: Hardcode --plugin-dir only | loses the generic escape hatch Confidence: high Scope-risk: narrow Not-tested: live end-to-end plugin load (covered by the consumer repo) Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/claude/src/driver.ts | 4 ++- packages/claude/test/build-args.test.ts | 35 +++++++++++++++++++++++++ packages/core/src/driver.ts | 4 +++ packages/core/src/runner.ts | 2 ++ 4 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 packages/claude/test/build-args.test.ts diff --git a/packages/claude/src/driver.ts b/packages/claude/src/driver.ts index ad6793c..2b863b7 100644 --- a/packages/claude/src/driver.ts +++ b/packages/claude/src/driver.ts @@ -140,8 +140,9 @@ function aggregateUsage(events: AgentEvent[]): Usage { return totals; } -function buildArgs(opts: RunOptions): string[] { +export function buildArgs(opts: RunOptions): string[] { const args = ['-p', opts.prompt, '--output-format', 'stream-json', '--verbose', '--model', opts.model]; + if (opts.pluginDir !== undefined) args.push('--plugin-dir', opts.pluginDir); if (opts.mcpConfig !== undefined) { args.push('--strict-mcp-config', '--mcp-config', JSON.stringify(opts.mcpConfig)); } @@ -150,6 +151,7 @@ function buildArgs(opts: RunOptions): string[] { if (opts.allowedTools?.length) args.push('--allowedTools', ...opts.allowedTools); if (opts.disallowedTools?.length) args.push('--disallowedTools', ...opts.disallowedTools); if (opts.appendSystemPrompt) args.push('--append-system-prompt', opts.appendSystemPrompt); + if (opts.extraArgs?.length) args.push(...opts.extraArgs); return args; } diff --git a/packages/claude/test/build-args.test.ts b/packages/claude/test/build-args.test.ts new file mode 100644 index 0000000..16dc7d9 --- /dev/null +++ b/packages/claude/test/build-args.test.ts @@ -0,0 +1,35 @@ +import { describe, it, expect } from 'vitest'; +import type { RunOptions } from '@agentry/core'; +import { buildArgs } from '../src/driver'; + +const base: RunOptions = { prompt: '/scheduler:status', model: 'claude-haiku-4-5', cwd: '/tmp/x' }; + +describe('buildArgs', () => { + it('emits the headless stream-json base invocation', () => { + expect(buildArgs(base)).toEqual([ + '-p', + '/scheduler:status', + '--output-format', + 'stream-json', + '--verbose', + '--model', + 'claude-haiku-4-5', + ]); + }); + + it('passes --plugin-dir when pluginDir is set', () => { + const args = buildArgs({ ...base, pluginDir: '/repo/root' }); + const i = args.indexOf('--plugin-dir'); + expect(i).toBeGreaterThanOrEqual(0); + expect(args[i + 1]).toBe('/repo/root'); + }); + + it('omits --plugin-dir when pluginDir is absent', () => { + expect(buildArgs(base)).not.toContain('--plugin-dir'); + }); + + it('appends extraArgs verbatim at the end', () => { + const args = buildArgs({ ...base, extraArgs: ['--foo', 'bar'] }); + expect(args.slice(-2)).toEqual(['--foo', 'bar']); + }); +}); diff --git a/packages/core/src/driver.ts b/packages/core/src/driver.ts index f07aaad..1a9b4a4 100644 --- a/packages/core/src/driver.ts +++ b/packages/core/src/driver.ts @@ -31,6 +31,10 @@ export interface RunOptions { appendSystemPrompt?: string; /** Permission mode passed through to the agent (e.g. 'bypassPermissions'). */ permissionMode?: string; + /** Load an additional agent plugin directory (Claude Code `--plugin-dir`). */ + pluginDir?: string; + /** Arbitrary extra CLI args appended verbatim to the agent invocation. */ + extraArgs?: string[]; /** Per-run wall-clock timeout (ms). */ timeoutMs?: number; signal?: AbortSignal; diff --git a/packages/core/src/runner.ts b/packages/core/src/runner.ts index 1fa8116..a8fb00a 100644 --- a/packages/core/src/runner.ts +++ b/packages/core/src/runner.ts @@ -75,6 +75,8 @@ export interface AgentRunExtra { disallowedTools?: string[]; permissionMode?: string; appendSystemPrompt?: string; + pluginDir?: string; + extraArgs?: string[]; } export interface TestFixtures { From ca8b95826ea29fdabeecb9759c838c0120317973 Mon Sep 17 00:00:00 2001 From: Francis Eytan Dortort Date: Fri, 31 Jul 2026 09:08:35 -0400 Subject: [PATCH 2/3] feat(core): add per-run env passthrough to agent.run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `agent.run(prompt, { env })` now merges the given vars OVER the base env (so it augments, never clobbers, the LLM-proxy's ANTHROPIC_BASE_URL). Lets a test point the agent's subprocess at scenario-scoped locations — e.g. a plugin's state directory — without touching the shared process env or $HOME. Constraint: base env carries ANTHROPIC_BASE_URL in live/record modes — must be preserved, hence merge (per-run wins) rather than replace. Confidence: high Scope-risk: narrow Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/core/src/runner.ts | 7 ++++-- packages/core/test/runner.test.ts | 39 +++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/packages/core/src/runner.ts b/packages/core/src/runner.ts index a8fb00a..0d2541a 100644 --- a/packages/core/src/runner.ts +++ b/packages/core/src/runner.ts @@ -77,6 +77,8 @@ export interface AgentRunExtra { appendSystemPrompt?: string; pluginDir?: string; extraArgs?: string[]; + /** Extra environment for this run; merged over (not replacing) the base env. */ + env?: Record; } export interface TestFixtures { @@ -114,13 +116,14 @@ export class AgentHandle implements RunViewProvider { async run(prompt: string, extra: AgentRunExtra = {}): Promise { this.lastPrompt = prompt; const before = await this.sandbox.snapshot(); + const { env: extraEnv, ...restExtra } = extra; const opts: RunOptions = { prompt, model: this.base.model, cwd: this.sandbox.dir, - env: this.base.env, maxBudgetUSD: this.base.maxBudgetUSD, - ...extra, + ...restExtra, + env: { ...this.base.env, ...extraEnv }, }; const rec = await this.driver.run(opts); const after = await this.sandbox.snapshot(); diff --git a/packages/core/test/runner.test.ts b/packages/core/test/runner.test.ts index d3c0327..6826df5 100644 --- a/packages/core/test/runner.test.ts +++ b/packages/core/test/runner.test.ts @@ -15,7 +15,11 @@ import { EventFactory, RunRecord, summarize, + AgentHandle, + Sandbox, type AgentEvent, + type AgentDriver, + type RunOptions, type RegisteredTest, } from '@agentry/core'; @@ -104,3 +108,38 @@ describe('runner (replay mode)', () => { v(ran).toBe(false); }); }); + +describe('AgentHandle env passthrough', () => { + it('merges base env with per-run env (per-run wins) and forwards run options', async () => { + const seen: RunOptions[] = []; + const driver: AgentDriver = { + id: 'fake', + capabilities: () => ({ + structuredStream: true, + llmInterception: 'none', + mcpTransports: [], + toolPermissionControl: false, + nativeBudgetControl: false, + }), + async run(opts) { + seen.push(opts); + return new RunRecord([], { exitCode: 0, reason: 'completed', usage: { inputTokens: 0, outputTokens: 0, costUSD: 0 } }); + }, + }; + const sandbox = await Sandbox.create({ prefix: 'agentry-envtest-' }); + try { + const handle = new AgentHandle( + driver, + { model: 'claude-haiku-4-5', env: { ANTHROPIC_BASE_URL: 'http://proxy', KEEP: 'base' } }, + sandbox, + ); + await handle.run('p', { env: { CLAUDE_SCHEDULER_STATE_DIR: '/x/.claude', KEEP: 'override' }, pluginDir: '/repo' }); + const opts = seen[0]!; + v(opts.env).toEqual({ ANTHROPIC_BASE_URL: 'http://proxy', KEEP: 'override', CLAUDE_SCHEDULER_STATE_DIR: '/x/.claude' }); + v(opts.pluginDir).toBe('/repo'); + v(opts.cwd).toBe(sandbox.dir); + } finally { + await sandbox.cleanup(); + } + }); +}); From f47ff1ab1935cc1b9cdc0e738a40c64c14dee74a Mon Sep 17 00:00:00 2001 From: Francis Eytan Dortort Date: Fri, 31 Jul 2026 11:45:59 -0400 Subject: [PATCH 3/3] feat(core): retry failed scenarios up to config.retries The `retries` config field existed but the runner never honored it. Live agent runs vary in phrasing and path, so a bounded retry is the standard flake tolerance (Playwright parity): a scenario passes if any attempt passes, and the result records `attempts` when a retry occurred. Confidence: high Scope-risk: narrow Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/core/src/runner.ts | 20 +++++++++++++++++++- packages/core/test/runner.test.ts | 19 +++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/packages/core/src/runner.ts b/packages/core/src/runner.ts index 0d2541a..8fe9ea8 100644 --- a/packages/core/src/runner.ts +++ b/packages/core/src/runner.ts @@ -163,6 +163,8 @@ export interface TestResult { mode: RunMode; error?: string; costUSD?: number; + /** Number of attempts taken (present only when > 1, i.e. a retry occurred). */ + attempts?: number; } export interface RunnerDeps { @@ -203,10 +205,26 @@ function makeRedactor(redact: ResolvedConfig['redact']): (text: string) => strin export async function runTests(tests: RegisteredTest[], deps: RunnerDeps): Promise { const results: TestResult[] = []; - for (const t of tests) results.push(await runOne(t, deps)); + for (const t of tests) results.push(await runWithRetries(t, deps)); return results; } +/** + * Run a scenario, retrying on failure up to `config.retries` times. Live agent + * runs are inherently non-deterministic (model phrasing/paths vary), so a bounded + * retry is the standard flake tolerance — a scenario passes if any attempt does. + */ +async function runWithRetries(t: RegisteredTest, deps: RunnerDeps): Promise { + const maxAttempts = 1 + Math.max(0, deps.config.retries ?? 0); + let result = await runOne(t, deps); + let attempt = 1; + while (result.status === 'failed' && attempt < maxAttempts) { + attempt += 1; + result = await runOne(t, deps); + } + return attempt > 1 ? { ...result, attempts: attempt } : result; +} + async function runOne(t: RegisteredTest, deps: RunnerDeps): Promise { const now = deps.now ?? (() => Date.now()); const start = now(); diff --git a/packages/core/test/runner.test.ts b/packages/core/test/runner.test.ts index 6826df5..eb6470e 100644 --- a/packages/core/test/runner.test.ts +++ b/packages/core/test/runner.test.ts @@ -65,6 +65,25 @@ describe('runner (replay mode)', () => { v(summarize(results)).toMatchObject({ passed: 1, failed: 0 }); }); + it('retries a failing scenario up to config.retries and passes if a later attempt succeeds', async () => { + dir = await mkdtemp(join(tmpdir(), 'agentry-runner-')); + setCurrentFile(join(dir, 'demo.agentry.ts')); + let calls = 0; + aTest('flaky', async ({ agent, expect }) => { + calls += 1; + await agent.run('p'); + if (calls < 2) throw new Error('flaky fail'); + await expect(agent).toHaveToolCall('read_file'); + }); + const config = resolveConfig({ use: { model: 'claude-haiku-4-5' }, testDir: dir, retries: 2 }); + await writeTranscriptFor(getRegistry()[0]!, dir, transcriptJson()); + + const results = await runTests(getRegistry(), { mode: 'replay', config }); + v(results[0]!.status).toBe('passed'); + v(results[0]!.attempts).toBe(2); + v(calls).toBe(2); + }); + it('fails when an assertion does not hold', async () => { dir = await mkdtemp(join(tmpdir(), 'agentry-runner-')); setCurrentFile(join(dir, 'demo.agentry.ts'));