Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ All TypeScript is executed directly via `tsx` — no build step required for dev
|---|---|
| `@agentry/core` | Event model, config, sandbox, runner, assertions |
| `@agentry/claude` | Claude Code driver (`claude -p --output-format stream-json`) |
| `@agentry/codex` | Codex CLI driver (`codex exec --json`) |
| `@agentry/mcp` | `MockMcpServer` (JSON-RPC + stdio shim) + MCP matchers |
| `agentry` | CLI (`agentry test`, `record`, `init`, `doctor`) |

Expand Down Expand Up @@ -245,6 +246,7 @@ The SPEC describes the full vision. What is implemented vs. planned:

**Implemented**
- Claude driver (`claude -p --output-format stream-json`)
- Codex driver (`codex exec --json`)
- Normalized causal event model (`tool_use`, `tool_result`, `mcp_request`, `llm_request`/`llm_response`, `message`, `usage`, `run.end`, `plugin`, `skill`, `fs`)
- Assertions, Tiers 1–3: `toHaveToolCall`, `toHaveCalledToolTimes`, `toUseToolsFrom`, `toHaveCalledAll`, `toHaveMcpRequest`, `toFinishWithin`, `toHaveFile`, `toMatchSchema`
- Skill/plugin effect matchers: `toHaveLoadedPlugin`, `toFireHook` (CH2) and `toInjectContext`, `toRegisterTools` (CH1, via the proxy)
Expand Down
1 change: 1 addition & 0 deletions packages/cli/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
"dependencies": {
"@agentry/core": "workspace:*",
"@agentry/claude": "workspace:*",
"@agentry/codex": "workspace:*",
"@agentry/mcp": "workspace:*"
}
}
15 changes: 15 additions & 0 deletions packages/cli/src/commands/doctor.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { spawnSync } from 'node:child_process';
import { ClaudeDriver } from '@agentry/claude';
import { CodexDriver } from '@agentry/codex';

/** \`agentry doctor\` — probe installed agent CLIs and print driver capabilities. */
export async function cmdDoctor(): Promise<number> {
Expand All @@ -18,5 +19,19 @@ export async function cmdDoctor(): Promise<number> {
for (const [k, val] of Object.entries(caps)) {
console.log(` ${k}: ${JSON.stringify(val)}`);
}

const codexBin = process.env.AGENTRY_CODEX_BIN ?? 'codex';
const cv = spawnSync(codexBin, ['--version'], { encoding: 'utf8' });
if (cv.status === 0) {
console.log(`\n codex ✓ ${cv.stdout.trim()}`);
} else {
console.log(`\n codex ✗ not found (looked for '${codexBin}')`);
}

console.log('\n codex driver capabilities:');
const codexCaps = new CodexDriver().capabilities();
for (const [k, val] of Object.entries(codexCaps)) {
console.log(` ${k}: ${JSON.stringify(val)}`);
}
return 0;
}
14 changes: 13 additions & 1 deletion packages/cli/src/commands/run.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,22 @@ import {
reportConsole,
type RunMode,
type AgentryConfig,
type AgentDriver,
} from '@agentry/core';
import { ClaudeDriver } from '@agentry/claude';
import { CodexDriver } from '@agentry/codex';
import { discoverTests } from '../discover';

function selectDriver(agent: string | undefined): AgentDriver {
switch (agent) {
case 'codex':
return new CodexDriver();
case 'claude':
default:
return new ClaudeDriver();
}
}

/** `agentry test` / `agentry record`. Returns a process exit code. */
export async function cmdRun(args: string[], forceMode?: RunMode): Promise<number> {
const { values } = parseArgs({
Expand Down Expand Up @@ -59,7 +71,7 @@ export async function cmdRun(args: string[], forceMode?: RunMode): Promise<numbe
}

console.log(`\nagentry — ${tests.length} scenario(s) · mode=${mode}\n`);
const liveDriver = new ClaudeDriver();
const liveDriver = selectDriver(config.use.agent);
const results = await runTests(tests, { mode, config, liveDriver });
const summary = reportConsole(results);
return summary.failed > 0 ? 1 : 0;
Expand Down
12 changes: 12 additions & 0 deletions packages/codex/package.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
{
"name": "@agentry/codex",
"version": "0.0.0",
"type": "module",
"description": "Agentry driver for the Codex CLI (headless exec --json).",
"exports": { ".": "./src/index.ts" },
"publishConfig": {
"exports": { ".": { "types": "./dist/index.d.ts", "default": "./dist/index.js" } }
},
"scripts": { "build": "tsup src/index.ts --format esm --dts --clean" },
"dependencies": { "@agentry/core": "workspace:*" }
}
256 changes: 256 additions & 0 deletions packages/codex/src/driver.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,256 @@
/**
* Codex CLI driver (SPEC §4.2). Drives `codex exec --json` headlessly and
* normalizes the native thread/item stream into AgentEvents.
*
* `parseCodexEvent` is a pure mapping (one native event → zero+ AgentEvents)
* so it can be unit-tested without spawning the CLI. Codex normalizes shell as
* tool `shell` and patches as `apply_patch`; LLM interception is provider-config
* (not wired to Agentry's Anthropic proxy); `reasoning` items and non-live
* `mcp_tool_call` mapping are best-effort/dropped in the MVP.
*/
import { spawn } from 'node:child_process';
import { createInterface } from 'node:readline';
import {
EventFactory,
RunRecord,
type AgentDriver,
type AgentEvent,
type DriverCapabilities,
type RunOptions,
type RunResult,
type RunEndReason,
type Usage,
isEvent,
} from '@agentry/core';

/* eslint-disable @typescript-eslint/no-explicit-any */
type Raw = any;

/** Map a single Codex `exec --json` event into zero or more AgentEvents. */
export function parseCodexEvent(raw: Raw, f: EventFactory, runId: string): AgentEvent[] {
const out: AgentEvent[] = [];
const type = raw?.type;

if (type === 'thread.started') {
out.push(
f.make(
{ type: 'run.start', runId, agent: 'codex', model: raw.model ?? '' },
{ turnId: 'init', source: 'agent', agentNativeType: 'thread.started', raw },
),
);
return out;
}

if (type === 'turn.started') {
return out;
}

if (type === 'item.started' || type === 'item.completed') {
const item = raw.item ?? {};
const turn = item.id ?? 'item';

if (item.type === 'agent_message' && type === 'item.completed') {
out.push(
f.make(
{ type: 'message', role: 'assistant', text: item.text ?? '' },
{ turnId: turn, source: 'agent', agentNativeType: 'item.agent_message', raw: item },
),
);
return out;
}

if (item.type === 'command_execution') {
if (type === 'item.started') {
out.push(
f.make(
{ type: 'tool_use', id: item.id, name: 'shell', args: { command: item.command } },
{ turnId: turn, source: 'agent', capability: 'tool', agentNativeType: 'item.command_execution', raw: item },
),
);
} else {
out.push(
f.make(
{
type: 'tool_result',
id: item.id,
name: 'shell',
result: item.aggregated_output,
isError: typeof item.exit_code === 'number' && item.exit_code !== 0,
},
{ turnId: turn, source: 'agent', capability: 'tool', agentNativeType: 'item.command_execution', raw: item },
),
);
}
return out;
}

if (item.type === 'file_change') {
if (type === 'item.started') {
out.push(
f.make(
{ type: 'tool_use', id: item.id, name: 'apply_patch', args: { changes: item.changes } },
{ turnId: turn, source: 'agent', capability: 'tool', agentNativeType: 'item.file_change', raw: item },
),
);
} else {
out.push(
f.make(
{ type: 'tool_result', id: item.id, name: 'apply_patch', result: item.changes, isError: false },
{ turnId: turn, source: 'agent', capability: 'tool', agentNativeType: 'item.file_change', raw: item },
),
);
}
return out;
}

if (item.type === 'mcp_tool_call' && type === 'item.started') {
out.push(
f.make(
{
type: 'mcp_request',
server: item.server ?? '',
method: 'tools/call',
params: { name: item.tool ?? item.name, arguments: item.arguments ?? item.args },
},
{ turnId: turn, source: 'agent', capability: 'mcp', agentNativeType: 'item.mcp_tool_call', raw: item },
),
);
return out;
}

return out;
}

if (type === 'turn.completed') {
const u = raw.usage ?? {};
out.push(
f.make(
{
type: 'usage',
inputTokens: u.input_tokens ?? 0,
outputTokens: u.output_tokens ?? 0,
cacheReadTokens: u.cached_input_tokens,
cacheCreationTokens: u.cache_write_input_tokens,
},
{ turnId: 'turn', source: 'agent', agentNativeType: 'turn.completed', raw: u },
),
);
return out;
}

if (type === 'error') {
out.push(
f.make(
{ type: 'error', kind: 'api', detail: { message: raw.message } },
{ turnId: 'error', source: 'agent', agentNativeType: 'error', raw },
),
);
return out;
}

return out;
}

/** Aggregate usage across the stream's `usage` events (Codex reports no cost). */
function aggregateUsage(events: AgentEvent[]): Usage {
const totals: Usage = { inputTokens: 0, outputTokens: 0, costUSD: 0 };
for (const e of events) {
if (isEvent(e, 'usage')) {
totals.inputTokens += e.payload.inputTokens;
totals.outputTokens += e.payload.outputTokens;
totals.costUSD = (totals.costUSD ?? 0) + (e.payload.costUSD ?? 0);
}
}
return totals;
}

export function buildArgs(opts: RunOptions): string[] {
const args = [
'exec',
'--json',
'--color',
'never',
'--skip-git-repo-check',
'--ephemeral',
'-C',
opts.cwd,
'-m',
opts.model,
];
if (opts.permissionMode === 'bypassPermissions') {
args.push('--dangerously-bypass-approvals-and-sandbox');
} else {
args.push('--sandbox', 'workspace-write');
}
if (opts.extraArgs?.length) args.push(...opts.extraArgs);
args.push(opts.prompt);
return args;
}

let runCounter = 0;

export class CodexDriver implements AgentDriver {
readonly id = 'codex';
constructor(private readonly bin = process.env.AGENTRY_CODEX_BIN ?? 'codex') {}

capabilities(): DriverCapabilities {
return {
structuredStream: true,
llmInterception: 'provider-config',
mcpTransports: ['stdio'],
toolPermissionControl: true,
nativeBudgetControl: false,
};
}

async run(opts: RunOptions): Promise<RunRecord> {
const runId = `codex-${runCounter++}`;
const factory = new EventFactory(runId);
const events: AgentEvent[] = [];

const child = spawn(this.bin, buildArgs(opts), {
cwd: opts.cwd,
env: { ...process.env, ...opts.env },
stdio: ['ignore', 'pipe', 'pipe'],
});

const rl = createInterface({ input: child.stdout });
rl.on('line', (line) => {
const s = line.trim();
if (!s) return;
let obj: unknown;
try {
obj = JSON.parse(s);
} catch {
return; // tolerate non-JSON noise
}
for (const e of parseCodexEvent(obj, factory, runId)) events.push(e);
});

let stderr = '';
child.stderr.on('data', (d) => (stderr += String(d)));

let timer: NodeJS.Timeout | undefined;
const exitCode = await new Promise<number | null>((resolve, reject) => {
child.on('error', reject);
child.on('close', (code) => resolve(code));
if (opts.timeoutMs) timer = setTimeout(() => child.kill('SIGTERM'), opts.timeoutMs);
opts.signal?.addEventListener('abort', () => child.kill('SIGTERM'), { once: true });
}).finally(() => timer && clearTimeout(timer));

const ended = events.find((e) => isEvent(e, 'run.end'));
const reason: RunEndReason = ended && isEvent(ended, 'run.end') ? ended.payload.reason : exitCode === 0 ? 'completed' : 'crash';
if (!ended) {
// Codex emits no terminal event; synthesize run.end so the stream is well-formed (SPEC §6)
events.push(
factory.make(
{ type: 'run.end', runId, exitCode, reason },
{ turnId: 'result', source: 'runner', agentNativeType: 'synthetic' },
),
);
}

const result: RunResult = { exitCode, reason, usage: aggregateUsage(events) };
return new RunRecord(events, result);
}
}
2 changes: 2 additions & 0 deletions packages/codex/src/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
// @agentry/codex — codex driver.
export * from './driver';
37 changes: 37 additions & 0 deletions packages/codex/test/build-args.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
import { describe, it, expect } from 'vitest';
import type { RunOptions } from '@agentry/core';
import { buildArgs } from '../src/driver';

const base: RunOptions = { prompt: 'create hello.txt', model: 'gpt-5-codex', cwd: '/tmp/x' };

describe('buildArgs', () => {
it('emits the headless exec --json base invocation with the prompt last', () => {
expect(buildArgs(base)).toEqual([
'exec',
'--json',
'--color',
'never',
'--skip-git-repo-check',
'--ephemeral',
'-C',
'/tmp/x',
'-m',
'gpt-5-codex',
'--sandbox',
'workspace-write',
'create hello.txt',
]);
});

it('bypasses approvals/sandbox when permissionMode is bypassPermissions', () => {
const args = buildArgs({ ...base, permissionMode: 'bypassPermissions' });
expect(args).toContain('--dangerously-bypass-approvals-and-sandbox');
expect(args).not.toContain('--sandbox');
expect(args[args.length - 1]).toBe('create hello.txt');
});

it('appends extraArgs before the trailing prompt positional', () => {
const args = buildArgs({ ...base, extraArgs: ['--foo', 'bar'] });
expect(args.slice(-3)).toEqual(['--foo', 'bar', 'create hello.txt']);
});
});
Loading
Loading