feat(driver): --plugin-dir, extraArgs, and per-run env passthrough

packages/claude/src/driver.ts

@@ -140,8 +140,9 @@ function aggregateUsage(events: AgentEvent[]): Usage {
   return totals;
 }
 
-function buildArgs(opts: RunOptions): string[] {
+export function buildArgs(opts: RunOptions): string[] {
   const args = ['-p', opts.prompt, '--output-format', 'stream-json', '--verbose', '--model', opts.model];
+  if (opts.pluginDir !== undefined) args.push('--plugin-dir', opts.pluginDir);
   if (opts.mcpConfig !== undefined) {
     args.push('--strict-mcp-config', '--mcp-config', JSON.stringify(opts.mcpConfig));
   }
@@ -150,6 +151,7 @@ function buildArgs(opts: RunOptions): string[] {
   if (opts.allowedTools?.length) args.push('--allowedTools', ...opts.allowedTools);
   if (opts.disallowedTools?.length) args.push('--disallowedTools', ...opts.disallowedTools);
   if (opts.appendSystemPrompt) args.push('--append-system-prompt', opts.appendSystemPrompt);
+  if (opts.extraArgs?.length) args.push(...opts.extraArgs);
   return args;
 }
 

packages/claude/test/build-args.test.ts

@@ -0,0 +1,35 @@
+import { describe, it, expect } from 'vitest';
+import type { RunOptions } from '@agentry/core';
+import { buildArgs } from '../src/driver';
+
+const base: RunOptions = { prompt: '/scheduler:status', model: 'claude-haiku-4-5', cwd: '/tmp/x' };
+
+describe('buildArgs', () => {
+  it('emits the headless stream-json base invocation', () => {
+    expect(buildArgs(base)).toEqual([
+      '-p',
+      '/scheduler:status',
+      '--output-format',
+      'stream-json',
+      '--verbose',
+      '--model',
+      'claude-haiku-4-5',
+    ]);
+  });
+
+  it('passes --plugin-dir <dir> when pluginDir is set', () => {
+    const args = buildArgs({ ...base, pluginDir: '/repo/root' });
+    const i = args.indexOf('--plugin-dir');
+    expect(i).toBeGreaterThanOrEqual(0);
+    expect(args[i + 1]).toBe('/repo/root');
+  });
+
+  it('omits --plugin-dir when pluginDir is absent', () => {
+    expect(buildArgs(base)).not.toContain('--plugin-dir');
+  });
+
+  it('appends extraArgs verbatim at the end', () => {
+    const args = buildArgs({ ...base, extraArgs: ['--foo', 'bar'] });
+    expect(args.slice(-2)).toEqual(['--foo', 'bar']);
+  });
+});

packages/core/src/driver.ts

@@ -31,6 +31,10 @@ export interface RunOptions {
   appendSystemPrompt?: string;
   /** Permission mode passed through to the agent (e.g. 'bypassPermissions'). */
   permissionMode?: string;
+  /** Load an additional agent plugin directory (Claude Code `--plugin-dir`). */
+  pluginDir?: string;
+  /** Arbitrary extra CLI args appended verbatim to the agent invocation. */
+  extraArgs?: string[];
   /** Per-run wall-clock timeout (ms). */
   timeoutMs?: number;
   signal?: AbortSignal;

packages/core/src/runner.ts

@@ -75,6 +75,10 @@ export interface AgentRunExtra {
   disallowedTools?: string[];
   permissionMode?: string;
   appendSystemPrompt?: string;
+  pluginDir?: string;
+  extraArgs?: string[];
+  /** Extra environment for this run; merged over (not replacing) the base env. */
+  env?: Record<string, string>;
 }
 
 export interface TestFixtures {
@@ -112,13 +116,14 @@ export class AgentHandle implements RunViewProvider {
   async run(prompt: string, extra: AgentRunExtra = {}): Promise<RunRecord> {
     this.lastPrompt = prompt;
     const before = await this.sandbox.snapshot();
+    const { env: extraEnv, ...restExtra } = extra;
     const opts: RunOptions = {
       prompt,
       model: this.base.model,
       cwd: this.sandbox.dir,
-      env: this.base.env,
       maxBudgetUSD: this.base.maxBudgetUSD,
-      ...extra,
+      ...restExtra,
+      env: { ...this.base.env, ...extraEnv },
     };
     const rec = await this.driver.run(opts);
     const after = await this.sandbox.snapshot();
@@ -158,6 +163,8 @@ export interface TestResult {
   mode: RunMode;
   error?: string;
   costUSD?: number;
+  /** Number of attempts taken (present only when > 1, i.e. a retry occurred). */
+  attempts?: number;
 }
 
 export interface RunnerDeps {
@@ -198,10 +205,26 @@ function makeRedactor(redact: ResolvedConfig['redact']): (text: string) => strin
 
 export async function runTests(tests: RegisteredTest[], deps: RunnerDeps): Promise<TestResult[]> {
   const results: TestResult[] = [];
-  for (const t of tests) results.push(await runOne(t, deps));
+  for (const t of tests) results.push(await runWithRetries(t, deps));
   return results;
 }
 
+/**
+ * Run a scenario, retrying on failure up to `config.retries` times. Live agent
+ * runs are inherently non-deterministic (model phrasing/paths vary), so a bounded
+ * retry is the standard flake tolerance — a scenario passes if any attempt does.
+ */
+async function runWithRetries(t: RegisteredTest, deps: RunnerDeps): Promise<TestResult> {
+  const maxAttempts = 1 + Math.max(0, deps.config.retries ?? 0);
+  let result = await runOne(t, deps);
+  let attempt = 1;
+  while (result.status === 'failed' && attempt < maxAttempts) {
+    attempt += 1;
+    result = await runOne(t, deps);
+  }
+  return attempt > 1 ? { ...result, attempts: attempt } : result;
+}
+
 async function runOne(t: RegisteredTest, deps: RunnerDeps): Promise<TestResult> {
   const now = deps.now ?? (() => Date.now());
   const start = now();

packages/core/test/runner.test.ts

@@ -15,7 +15,11 @@ import {
   EventFactory,
   RunRecord,
   summarize,
+  AgentHandle,
+  Sandbox,
   type AgentEvent,
+  type AgentDriver,
+  type RunOptions,
   type RegisteredTest,
 } from '@agentry/core';
 
@@ -61,6 +65,25 @@ describe('runner (replay mode)', () => {
     v(summarize(results)).toMatchObject({ passed: 1, failed: 0 });
   });
 
+  it('retries a failing scenario up to config.retries and passes if a later attempt succeeds', async () => {
+    dir = await mkdtemp(join(tmpdir(), 'agentry-runner-'));
+    setCurrentFile(join(dir, 'demo.agentry.ts'));
+    let calls = 0;
+    aTest('flaky', async ({ agent, expect }) => {
+      calls += 1;
+      await agent.run('p');
+      if (calls < 2) throw new Error('flaky fail');
+      await expect(agent).toHaveToolCall('read_file');
+    });
+    const config = resolveConfig({ use: { model: 'claude-haiku-4-5' }, testDir: dir, retries: 2 });
+    await writeTranscriptFor(getRegistry()[0]!, dir, transcriptJson());
+
+    const results = await runTests(getRegistry(), { mode: 'replay', config });
+    v(results[0]!.status).toBe('passed');
+    v(results[0]!.attempts).toBe(2);
+    v(calls).toBe(2);
+  });
+
   it('fails when an assertion does not hold', async () => {
     dir = await mkdtemp(join(tmpdir(), 'agentry-runner-'));
     setCurrentFile(join(dir, 'demo.agentry.ts'));
@@ -104,3 +127,38 @@ describe('runner (replay mode)', () => {
     v(ran).toBe(false);
   });
 });
+
+describe('AgentHandle env passthrough', () => {
+  it('merges base env with per-run env (per-run wins) and forwards run options', async () => {
+    const seen: RunOptions[] = [];
+    const driver: AgentDriver = {
+      id: 'fake',
+      capabilities: () => ({
+        structuredStream: true,
+        llmInterception: 'none',
+        mcpTransports: [],
+        toolPermissionControl: false,
+        nativeBudgetControl: false,
+      }),
+      async run(opts) {
+        seen.push(opts);
+        return new RunRecord([], { exitCode: 0, reason: 'completed', usage: { inputTokens: 0, outputTokens: 0, costUSD: 0 } });
+      },
+    };
+    const sandbox = await Sandbox.create({ prefix: 'agentry-envtest-' });
+    try {
+      const handle = new AgentHandle(
+        driver,
+        { model: 'claude-haiku-4-5', env: { ANTHROPIC_BASE_URL: 'http://proxy', KEEP: 'base' } },
+        sandbox,
+      );
+      await handle.run('p', { env: { CLAUDE_SCHEDULER_STATE_DIR: '/x/.claude', KEEP: 'override' }, pluginDir: '/repo' });
+      const opts = seen[0]!;
+      v(opts.env).toEqual({ ANTHROPIC_BASE_URL: 'http://proxy', KEEP: 'override', CLAUDE_SCHEDULER_STATE_DIR: '/x/.claude' });
+      v(opts.pluginDir).toBe('/repo');
+      v(opts.cwd).toBe(sandbox.dir);
+    } finally {
+      await sandbox.cleanup();
+    }
+  });
+});