diff --git a/packages/claude/src/driver.ts b/packages/claude/src/driver.ts
index ad6793c..2b863b7 100644
--- a/packages/claude/src/driver.ts
+++ b/packages/claude/src/driver.ts
@@ -140,8 +140,9 @@ function aggregateUsage(events: AgentEvent[]): Usage {
return totals;
}
-function buildArgs(opts: RunOptions): string[] {
+export function buildArgs(opts: RunOptions): string[] {
const args = ['-p', opts.prompt, '--output-format', 'stream-json', '--verbose', '--model', opts.model];
+ if (opts.pluginDir !== undefined) args.push('--plugin-dir', opts.pluginDir);
if (opts.mcpConfig !== undefined) {
args.push('--strict-mcp-config', '--mcp-config', JSON.stringify(opts.mcpConfig));
}
@@ -150,6 +151,7 @@ function buildArgs(opts: RunOptions): string[] {
if (opts.allowedTools?.length) args.push('--allowedTools', ...opts.allowedTools);
if (opts.disallowedTools?.length) args.push('--disallowedTools', ...opts.disallowedTools);
if (opts.appendSystemPrompt) args.push('--append-system-prompt', opts.appendSystemPrompt);
+ if (opts.extraArgs?.length) args.push(...opts.extraArgs);
return args;
}
diff --git a/packages/claude/test/build-args.test.ts b/packages/claude/test/build-args.test.ts
new file mode 100644
index 0000000..16dc7d9
--- /dev/null
+++ b/packages/claude/test/build-args.test.ts
@@ -0,0 +1,35 @@
+import { describe, it, expect } from 'vitest';
+import type { RunOptions } from '@agentry/core';
+import { buildArgs } from '../src/driver';
+
+const base: RunOptions = { prompt: '/scheduler:status', model: 'claude-haiku-4-5', cwd: '/tmp/x' };
+
+describe('buildArgs', () => {
+ it('emits the headless stream-json base invocation', () => {
+ expect(buildArgs(base)).toEqual([
+ '-p',
+ '/scheduler:status',
+ '--output-format',
+ 'stream-json',
+ '--verbose',
+ '--model',
+ 'claude-haiku-4-5',
+ ]);
+ });
+
+ it('passes --plugin-dir
when pluginDir is set', () => {
+ const args = buildArgs({ ...base, pluginDir: '/repo/root' });
+ const i = args.indexOf('--plugin-dir');
+ expect(i).toBeGreaterThanOrEqual(0);
+ expect(args[i + 1]).toBe('/repo/root');
+ });
+
+ it('omits --plugin-dir when pluginDir is absent', () => {
+ expect(buildArgs(base)).not.toContain('--plugin-dir');
+ });
+
+ it('appends extraArgs verbatim at the end', () => {
+ const args = buildArgs({ ...base, extraArgs: ['--foo', 'bar'] });
+ expect(args.slice(-2)).toEqual(['--foo', 'bar']);
+ });
+});
diff --git a/packages/core/src/driver.ts b/packages/core/src/driver.ts
index f07aaad..1a9b4a4 100644
--- a/packages/core/src/driver.ts
+++ b/packages/core/src/driver.ts
@@ -31,6 +31,10 @@ export interface RunOptions {
appendSystemPrompt?: string;
/** Permission mode passed through to the agent (e.g. 'bypassPermissions'). */
permissionMode?: string;
+ /** Load an additional agent plugin directory (Claude Code `--plugin-dir`). */
+ pluginDir?: string;
+ /** Arbitrary extra CLI args appended verbatim to the agent invocation. */
+ extraArgs?: string[];
/** Per-run wall-clock timeout (ms). */
timeoutMs?: number;
signal?: AbortSignal;
diff --git a/packages/core/src/runner.ts b/packages/core/src/runner.ts
index 1fa8116..8fe9ea8 100644
--- a/packages/core/src/runner.ts
+++ b/packages/core/src/runner.ts
@@ -75,6 +75,10 @@ export interface AgentRunExtra {
disallowedTools?: string[];
permissionMode?: string;
appendSystemPrompt?: string;
+ pluginDir?: string;
+ extraArgs?: string[];
+ /** Extra environment for this run; merged over (not replacing) the base env. */
+ env?: Record;
}
export interface TestFixtures {
@@ -112,13 +116,14 @@ export class AgentHandle implements RunViewProvider {
async run(prompt: string, extra: AgentRunExtra = {}): Promise {
this.lastPrompt = prompt;
const before = await this.sandbox.snapshot();
+ const { env: extraEnv, ...restExtra } = extra;
const opts: RunOptions = {
prompt,
model: this.base.model,
cwd: this.sandbox.dir,
- env: this.base.env,
maxBudgetUSD: this.base.maxBudgetUSD,
- ...extra,
+ ...restExtra,
+ env: { ...this.base.env, ...extraEnv },
};
const rec = await this.driver.run(opts);
const after = await this.sandbox.snapshot();
@@ -158,6 +163,8 @@ export interface TestResult {
mode: RunMode;
error?: string;
costUSD?: number;
+ /** Number of attempts taken (present only when > 1, i.e. a retry occurred). */
+ attempts?: number;
}
export interface RunnerDeps {
@@ -198,10 +205,26 @@ function makeRedactor(redact: ResolvedConfig['redact']): (text: string) => strin
export async function runTests(tests: RegisteredTest[], deps: RunnerDeps): Promise {
const results: TestResult[] = [];
- for (const t of tests) results.push(await runOne(t, deps));
+ for (const t of tests) results.push(await runWithRetries(t, deps));
return results;
}
+/**
+ * Run a scenario, retrying on failure up to `config.retries` times. Live agent
+ * runs are inherently non-deterministic (model phrasing/paths vary), so a bounded
+ * retry is the standard flake tolerance — a scenario passes if any attempt does.
+ */
+async function runWithRetries(t: RegisteredTest, deps: RunnerDeps): Promise {
+ const maxAttempts = 1 + Math.max(0, deps.config.retries ?? 0);
+ let result = await runOne(t, deps);
+ let attempt = 1;
+ while (result.status === 'failed' && attempt < maxAttempts) {
+ attempt += 1;
+ result = await runOne(t, deps);
+ }
+ return attempt > 1 ? { ...result, attempts: attempt } : result;
+}
+
async function runOne(t: RegisteredTest, deps: RunnerDeps): Promise {
const now = deps.now ?? (() => Date.now());
const start = now();
diff --git a/packages/core/test/runner.test.ts b/packages/core/test/runner.test.ts
index d3c0327..eb6470e 100644
--- a/packages/core/test/runner.test.ts
+++ b/packages/core/test/runner.test.ts
@@ -15,7 +15,11 @@ import {
EventFactory,
RunRecord,
summarize,
+ AgentHandle,
+ Sandbox,
type AgentEvent,
+ type AgentDriver,
+ type RunOptions,
type RegisteredTest,
} from '@agentry/core';
@@ -61,6 +65,25 @@ describe('runner (replay mode)', () => {
v(summarize(results)).toMatchObject({ passed: 1, failed: 0 });
});
+ it('retries a failing scenario up to config.retries and passes if a later attempt succeeds', async () => {
+ dir = await mkdtemp(join(tmpdir(), 'agentry-runner-'));
+ setCurrentFile(join(dir, 'demo.agentry.ts'));
+ let calls = 0;
+ aTest('flaky', async ({ agent, expect }) => {
+ calls += 1;
+ await agent.run('p');
+ if (calls < 2) throw new Error('flaky fail');
+ await expect(agent).toHaveToolCall('read_file');
+ });
+ const config = resolveConfig({ use: { model: 'claude-haiku-4-5' }, testDir: dir, retries: 2 });
+ await writeTranscriptFor(getRegistry()[0]!, dir, transcriptJson());
+
+ const results = await runTests(getRegistry(), { mode: 'replay', config });
+ v(results[0]!.status).toBe('passed');
+ v(results[0]!.attempts).toBe(2);
+ v(calls).toBe(2);
+ });
+
it('fails when an assertion does not hold', async () => {
dir = await mkdtemp(join(tmpdir(), 'agentry-runner-'));
setCurrentFile(join(dir, 'demo.agentry.ts'));
@@ -104,3 +127,38 @@ describe('runner (replay mode)', () => {
v(ran).toBe(false);
});
});
+
+describe('AgentHandle env passthrough', () => {
+ it('merges base env with per-run env (per-run wins) and forwards run options', async () => {
+ const seen: RunOptions[] = [];
+ const driver: AgentDriver = {
+ id: 'fake',
+ capabilities: () => ({
+ structuredStream: true,
+ llmInterception: 'none',
+ mcpTransports: [],
+ toolPermissionControl: false,
+ nativeBudgetControl: false,
+ }),
+ async run(opts) {
+ seen.push(opts);
+ return new RunRecord([], { exitCode: 0, reason: 'completed', usage: { inputTokens: 0, outputTokens: 0, costUSD: 0 } });
+ },
+ };
+ const sandbox = await Sandbox.create({ prefix: 'agentry-envtest-' });
+ try {
+ const handle = new AgentHandle(
+ driver,
+ { model: 'claude-haiku-4-5', env: { ANTHROPIC_BASE_URL: 'http://proxy', KEEP: 'base' } },
+ sandbox,
+ );
+ await handle.run('p', { env: { CLAUDE_SCHEDULER_STATE_DIR: '/x/.claude', KEEP: 'override' }, pluginDir: '/repo' });
+ const opts = seen[0]!;
+ v(opts.env).toEqual({ ANTHROPIC_BASE_URL: 'http://proxy', KEEP: 'override', CLAUDE_SCHEDULER_STATE_DIR: '/x/.claude' });
+ v(opts.pluginDir).toBe('/repo');
+ v(opts.cwd).toBe(sandbox.dir);
+ } finally {
+ await sandbox.cleanup();
+ }
+ });
+});