diff --git a/src/benchmarks/agent-cli/candidate-models.test.ts b/src/benchmarks/agent-cli/candidate-models.test.ts index 6a4cab04..796121e3 100644 --- a/src/benchmarks/agent-cli/candidate-models.test.ts +++ b/src/benchmarks/agent-cli/candidate-models.test.ts @@ -16,12 +16,12 @@ describe("sandboxAgentPluginError", () => { it("names the benchmark and agent when a candidate list would be dropped", () => { const error = sandboxAgentPluginError({ benchmarkId: "terminal_bench", - agent: "pi", + agent: "claude", models: ["openai/gpt-4.1-nano", "anthropic/claude-sonnet-4.5"], switchyardAlgorithm: undefined, }); expect(error?.message).toBe( - "terminal_bench cannot use candidate models: the pi agent calls the model from inside the sandbox, which does not forward the models list" + "terminal_bench cannot use candidate models: the claude agent calls the model from inside the sandbox and does not forward the models list (supported agents: pi)" ); }); it("names the benchmark and agent when a switchyard algorithm would be dropped", () => { @@ -35,6 +35,16 @@ describe("sandboxAgentPluginError", () => { "terminal_bench cannot use switchyardAlgorithm: the claude agent calls the model from inside the sandbox and does not forward the switchyard-router plugin (supported agents: pi)" ); }); + it("accepts candidate models and an algorithm for an agent that forwards them", () => { + expect( + sandboxAgentPluginError({ + benchmarkId: "terminal_bench", + agent: "pi", + models: ["z-ai/glm-5.3-flash", "anthropic/claude-opus-5.5"], + switchyardAlgorithm: "stage", + }) + ).toBeUndefined(); + }); it("accepts a switchyard algorithm for an agent that forwards the plugin", () => { expect( sandboxAgentPluginError({ diff --git a/src/benchmarks/agent-cli/candidate-models.ts b/src/benchmarks/agent-cli/candidate-models.ts index 313c38ee..3af03ed4 100644 --- a/src/benchmarks/agent-cli/candidate-models.ts +++ b/src/benchmarks/agent-cli/candidate-models.ts @@ -10,9 +10,12 @@ export function sandboxAgentPluginError(opts: { readonly models: readonly string[] | undefined; readonly switchyardAlgorithm: SwitchyardAlgorithm | undefined; }): Error | undefined { - if (opts.models !== undefined) { + if ( + opts.models !== undefined && + !forwardsSwitchyardRouterPlugin(opts.agent) + ) { return new Error( - `${opts.benchmarkId} cannot use candidate models: the ${opts.agent} agent calls the model from inside the sandbox, which does not forward the models list` + `${opts.benchmarkId} cannot use candidate models: the ${opts.agent} agent calls the model from inside the sandbox and does not forward the models list (supported agents: ${SWITCHYARD_ROUTER_PLUGIN_AGENTS.join(", ")})` ); } if ( diff --git a/src/benchmarks/agent-cli/harness.test.ts b/src/benchmarks/agent-cli/harness.test.ts index b775da38..7332323f 100644 --- a/src/benchmarks/agent-cli/harness.test.ts +++ b/src/benchmarks/agent-cli/harness.test.ts @@ -4,6 +4,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { + AGENT_CANDIDATE_MODELS_ENV, AGENT_REQUEST_PLUGINS_ENV, getOriHarness, ORI_HARNESSES, @@ -28,13 +29,20 @@ const RUN_OPTIONS: OriRunScriptOptions = { type ProviderRequestHandler = (event: { readonly payload: unknown }) => unknown; async function loadRequestPluginsHandler( - plugins: readonly unknown[] + plugins: readonly unknown[], + models?: readonly string[] ): Promise { const dir = mkdtempSync(join(tmpdir(), "pi-request-plugins-")); const path = join(dir, "openrouter-request-plugins.ts"); writeFileSync(path, PI_REQUEST_PLUGINS_EXTENSION_SOURCE); const previous = process.env[AGENT_REQUEST_PLUGINS_ENV]; + const previousModels = process.env[AGENT_CANDIDATE_MODELS_ENV]; process.env[AGENT_REQUEST_PLUGINS_ENV] = JSON.stringify(plugins); + if (models === undefined) { + Reflect.deleteProperty(process.env, AGENT_CANDIDATE_MODELS_ENV); + } else { + process.env[AGENT_CANDIDATE_MODELS_ENV] = JSON.stringify(models); + } try { const extension: { readonly default: (pi: { @@ -58,6 +66,11 @@ async function loadRequestPluginsHandler( } else { process.env[AGENT_REQUEST_PLUGINS_ENV] = previous; } + if (previousModels === undefined) { + Reflect.deleteProperty(process.env, AGENT_CANDIDATE_MODELS_ENV); + } else { + process.env[AGENT_CANDIDATE_MODELS_ENV] = previousModels; + } } } @@ -95,6 +108,57 @@ describe("pi request plugins extension", () => { }); }); + it("adds the candidate models and the switchyard-router plugin together", async () => { + const handler = await loadRequestPluginsHandler( + [{ id: "switchyard-router", algorithm: "stage" }], + ["z-ai/glm-5.3-flash", "anthropic/claude-opus-5.5"] + ); + expect( + handler({ + payload: { + model: "nvidia/switchyard", + input: [], + max_output_tokens: 235_929, + }, + }) + ).toEqual({ + model: "nvidia/switchyard", + input: [], + models: ["z-ai/glm-5.3-flash", "anthropic/claude-opus-5.5"], + plugins: [{ id: "switchyard-router", algorithm: "stage" }], + }); + }); + + it("adds only the candidate models when no plugins are configured", async () => { + const handler = await loadRequestPluginsHandler( + [], + ["deepseek/deepseek-v4.1-flash", "openai/gpt-6-sol"] + ); + expect( + handler({ + payload: { model: "nvidia/switchyard", models: ["stale/model"] }, + }) + ).toEqual({ + model: "nvidia/switchyard", + models: ["deepseek/deepseek-v4.1-flash", "openai/gpt-6-sol"], + }); + }); + + it("keeps max_output_tokens when no candidate models are configured", async () => { + const handler = await loadRequestPluginsHandler([ + { id: "auto-router", cost_tier: "low" }, + ]); + expect( + handler({ + payload: { model: "openrouter/auto", max_output_tokens: 235_929 }, + }) + ).toEqual({ + model: "openrouter/auto", + max_output_tokens: 235_929, + plugins: [{ id: "auto-router", cost_tier: "low" }], + }); + }); + it("leaves non-object payloads unchanged", async () => { const handler = await loadRequestPluginsHandler([ { id: "auto-router", cost_tier: "low" }, @@ -109,7 +173,7 @@ describe("pi request plugins run script", () => { const pi = getOriHarness("pi"); const script = pi.buildRunScript({ ...RUN_OPTIONS, - hasRequestPlugins: true, + loadsRequestExtension: true, }); expect(script).toContain( `cat > ${PI_REQUEST_PLUGINS_EXTENSION_PATH} <<'TB_PI_REQUEST_PLUGINS_EXTENSION'\n${PI_REQUEST_PLUGINS_EXTENSION_SOURCE}\nTB_PI_REQUEST_PLUGINS_EXTENSION\n` @@ -128,7 +192,7 @@ describe("pi request plugins run script", () => { it("loads the extension explicitly even when agent config is isolated", () => { const script = getOriHarness("pi").buildRunScript({ ...RUN_OPTIONS, - hasRequestPlugins: true, + loadsRequestExtension: true, isolateAgentConfig: true, }); expect(script).toContain("--no-extensions"); diff --git a/src/benchmarks/agent-cli/harness.ts b/src/benchmarks/agent-cli/harness.ts index c1489611..af2d07f8 100644 --- a/src/benchmarks/agent-cli/harness.ts +++ b/src/benchmarks/agent-cli/harness.ts @@ -47,6 +47,8 @@ export const DEFAULT_AGENT_RUNTIME_SHA256 = export const AGENT_REQUEST_PLUGINS_ENV = "TB_OPENROUTER_PLUGINS" as const; +export const AGENT_CANDIDATE_MODELS_ENV = "TB_OPENROUTER_MODELS" as const; + const PI_REQUEST_PLUGINS_EXTENSION_DIR = "/root/.bench-harness"; export const PI_REQUEST_PLUGINS_EXTENSION_PATH = @@ -56,6 +58,7 @@ const PI_REQUEST_PLUGINS_EXTENSION_EOF = "TB_PI_REQUEST_PLUGINS_EXTENSION"; export const PI_REQUEST_PLUGINS_EXTENSION_SOURCE = [ `const plugins = JSON.parse(process.env.${AGENT_REQUEST_PLUGINS_ENV} ?? "[]");`, + `const models = JSON.parse(process.env.${AGENT_CANDIDATE_MODELS_ENV} ?? "[]");`, "const pluginIds = new Set(plugins.map((plugin) => plugin.id));", "export default function registerRequestPlugins(pi) {", ' pi.on("before_provider_request", (event) => {', @@ -63,10 +66,15 @@ export const PI_REQUEST_PLUGINS_EXTENSION_SOURCE = [ ' if (typeof payload !== "object" || payload === null || Array.isArray(payload)) {', " return undefined;", " }", + " const { max_output_tokens: _maxOutputTokens, ...uncapped } = payload;", + " const withModels = models.length > 0 ? { ...uncapped, models } : payload;", + " if (plugins.length === 0) {", + " return withModels;", + " }", " const callerPlugins = Array.isArray(payload.plugins)", " ? payload.plugins.filter((plugin) => !pluginIds.has(plugin?.id))", " : [];", - " return { ...payload, plugins: [...callerPlugins, ...plugins] };", + " return { ...withModels, plugins: [...callerPlugins, ...plugins] };", " });", "}", ].join("\n"); @@ -162,7 +170,7 @@ export interface OriRunScriptOptions { readonly hasAllowedTools: boolean; readonly hasDisallowedTools: boolean; readonly isolateAgentConfig: boolean; - readonly hasRequestPlugins?: boolean; + readonly loadsRequestExtension?: boolean; } export interface OriImageStepsOptions { @@ -512,13 +520,13 @@ const ORI_PI_HARNESS: OriHarnessDef = { "export HOME=/root", "mkdir -p /logs/agent", ...writePiUsageExtension(), - ...(options.hasRequestPlugins === true + ...(options.loadsRequestExtension === true ? writePiRequestPluginsExtension() : []), 'ori pi --model "$TB_MODEL" \\', ` --reasoning-effort ${options.reasoningEffort} -- \\`, " --print --mode json --no-session \\", - ...(options.hasRequestPlugins === true + ...(options.loadsRequestExtension === true ? [` --extension ${PI_REQUEST_PLUGINS_EXTENSION_PATH} \\`] : []), ` --extension ${PI_USAGE_EXTENSION_PATH} \\`, diff --git a/src/benchmarks/agent-cli/runner.ts b/src/benchmarks/agent-cli/runner.ts index 1192513a..e97a63ea 100644 --- a/src/benchmarks/agent-cli/runner.ts +++ b/src/benchmarks/agent-cli/runner.ts @@ -24,7 +24,10 @@ import { import { getCurrentEpoch } from "../../runtime/response-cache"; import type { SandboxSessionInstance } from "../../sandbox/session"; import type { OriAgentRun, OriHarnessDef } from "./harness"; -import { AGENT_REQUEST_PLUGINS_ENV } from "./harness"; +import { + AGENT_CANDIDATE_MODELS_ENV, + AGENT_REQUEST_PLUGINS_ENV, +} from "./harness"; import type { OriChannel, OriReasoningEffort } from "./schema"; import { DEFAULT_ORI_CHANNEL, DEFAULT_ORI_INSTALL_URL } from "./schema"; @@ -157,6 +160,7 @@ export interface AgentCliOpts { readonly disallowedTools?: readonly string[]; readonly isolateAgentConfig?: boolean; readonly requestPlugins?: readonly AgentRequestPlugin[]; + readonly models?: readonly string[]; } export interface AgentCliRunResult extends OriAgentRun { @@ -203,6 +207,10 @@ export function buildAgentCliEnv(opts: AgentCliOpts): Record { if (requestPlugins.length > 0) { env[AGENT_REQUEST_PLUGINS_ENV] = JSON.stringify(requestPlugins); } + const models = opts.models ?? []; + if (models.length > 0) { + env[AGENT_CANDIDATE_MODELS_ENV] = JSON.stringify(models); + } return env; } @@ -248,6 +256,7 @@ export function runAgentCli(input: { }): Effect { const { session, harness, opts, instructionPath, timeoutMs } = input; const requestPlugins = opts.requestPlugins ?? []; + const models = opts.models ?? []; const script = harness.buildRunScript({ instructionPath, logPath: harness.remoteLogPath, @@ -257,7 +266,7 @@ export function runAgentCli(input: { hasAllowedTools: (opts.allowedTools ?? []).length > 0, hasDisallowedTools: (opts.disallowedTools ?? []).length > 0, isolateAgentConfig: opts.isolateAgentConfig === true, - hasRequestPlugins: requestPlugins.length > 0, + loadsRequestExtension: requestPlugins.length > 0 || models.length > 0, }); return gen(function* () { if (requestPlugins.length > 0 && !harness.forwardsRequestPlugins) { @@ -265,6 +274,11 @@ export function runAgentCli(input: { message: `the ${harness.id} agent does not forward OpenRouter request plugins (${requestPlugins.map((plugin) => plugin.id).join(", ")}) from inside the sandbox`, }); } + if (models.length > 0 && !harness.forwardsRequestPlugins) { + return yield* new SolverError({ + message: `the ${harness.id} agent does not forward the OpenRouter models list from inside the sandbox`, + }); + } if ( opts.sessionId !== undefined && opts.sessionId.length > 0 && diff --git a/src/benchmarks/deep-swe/benchmark.ts b/src/benchmarks/deep-swe/benchmark.ts index d447f9d5..2d029bc6 100644 --- a/src/benchmarks/deep-swe/benchmark.ts +++ b/src/benchmarks/deep-swe/benchmark.ts @@ -122,6 +122,7 @@ function makeDeepSweLayer( isolateAgentConfig: benchmarkConfig.isolateAgentConfig, requestPlugins: requestPlugins.length > 0 ? requestPlugins : undefined, + models: benchmarkConfig.models, }), endpointId: benchmarkConfig.endpointId, sessionId: input.sessionId, diff --git a/src/benchmarks/swe-atlas/benchmark.ts b/src/benchmarks/swe-atlas/benchmark.ts index 9cfc5245..d036a9b6 100644 --- a/src/benchmarks/swe-atlas/benchmark.ts +++ b/src/benchmarks/swe-atlas/benchmark.ts @@ -132,6 +132,7 @@ function makeSweAtlasLayer( isolateAgentConfig: benchmarkConfig.isolateAgentConfig, requestPlugins: requestPlugins.length > 0 ? requestPlugins : undefined, + models: benchmarkConfig.models, }), endpointId: benchmarkConfig.endpointId, inference: { diff --git a/src/benchmarks/terminal-bench/benchmark.test.ts b/src/benchmarks/terminal-bench/benchmark.test.ts index 44397058..c1205285 100644 --- a/src/benchmarks/terminal-bench/benchmark.test.ts +++ b/src/benchmarks/terminal-bench/benchmark.test.ts @@ -50,4 +50,26 @@ describe("terminal_bench layer", () => { "terminal_bench cannot use switchyardAlgorithm: the claude agent calls the model from inside the sandbox and does not forward the switchyard-router plugin (supported agents: pi)" ); }); + + it("rejects candidate models for an agent that cannot forward them", async () => { + const exit = await buildLayer({ + agent: "claude", + model: "nvidia/switchyard", + models: ["z-ai/glm-5.3-flash", "anthropic/claude-opus-5.5"], + }); + assertFailure(exit); + expect(getOrThrow(failureOption(exit.cause)).message).toBe( + "terminal_bench cannot use candidate models: the claude agent calls the model from inside the sandbox and does not forward the models list (supported agents: pi)" + ); + }); + + it("accepts candidate models and a switchyard algorithm for pi", async () => { + const exit = await buildLayer({ + agent: "pi", + model: "nvidia/switchyard", + models: ["z-ai/glm-5.3-flash", "anthropic/claude-opus-5.5"], + switchyardAlgorithm: "stage", + }); + expect(exit._tag).toBe("Success"); + }); }); diff --git a/src/benchmarks/terminal-bench/benchmark.ts b/src/benchmarks/terminal-bench/benchmark.ts index 577c7571..352081c3 100644 --- a/src/benchmarks/terminal-bench/benchmark.ts +++ b/src/benchmarks/terminal-bench/benchmark.ts @@ -77,6 +77,7 @@ function makeTerminalBenchLayer( disallowedTools: benchmarkConfig.disallowedTools, isolateAgentConfig: benchmarkConfig.isolateAgentConfig, requestPlugins: requestPlugins.length > 0 ? requestPlugins : undefined, + models: benchmarkConfig.models, }); const datasetLayer = makeTerminalBenchDatasetLayer( definedValues({ diff --git a/src/benchmarks/terminal-bench/ori-solver.test.ts b/src/benchmarks/terminal-bench/ori-solver.test.ts index b6a0622e..672a23ea 100644 --- a/src/benchmarks/terminal-bench/ori-solver.test.ts +++ b/src/benchmarks/terminal-bench/ori-solver.test.ts @@ -49,6 +49,7 @@ import { GenerationResolver } from "../../runtime/generation-resolver"; import { setCurrentSampleId } from "../../runtime/request-session-id"; import { setCurrentEpoch } from "../../runtime/response-cache"; import { + AGENT_CANDIDATE_MODELS_ENV, AGENT_REQUEST_PLUGINS_ENV, BUN_RELEASE_SHA256, BUN_RELEASE_URL, @@ -988,6 +989,60 @@ describe("terminal-bench pi via ori", () => { ); }); + it("forwards candidate models and the switchyard-router plugin to pi", async () => { + const execCalls: ExecCalls = []; + const layer = makeTerminalBenchFakeSandboxLayer({ + reward: 1, + execCalls, + agentExitCode: 0, + }); + const solverLayer = layerEffect(Solver)( + gen(function* () { + const sessionFactory = yield* SandboxSession; + return Solver.of( + oriSolver( + sessionFactory, + { + ...SOLVER_OPTS, + model: "nvidia/switchyard", + models: ["z-ai/glm-5.3-flash", "anthropic/claude-opus-5.5"], + requestPlugins: [{ id: "switchyard-router", algorithm: "stage" }], + }, + getOriHarness("pi") + ) + ); + }) + ); + await runPromise( + gen(function* () { + const solver = yield* Solver; + return yield* solver(sampleState()); + }).pipe( + provide( + layerMergeAll( + solverLayer.pipe(layerProvide(layer)), + noopProgressLayer, + noopCheckpointLayer + ) + ) + ) + ); + const agentCall = execCalls[0]; + if (agentCall === undefined) { + throw new Error("fake sandbox did not capture the agent invocation"); + } + expect(agentCall.env["TB_MODEL"]).toBe("nvidia/switchyard"); + expect(agentCall.env[AGENT_CANDIDATE_MODELS_ENV]).toBe( + '["z-ai/glm-5.3-flash","anthropic/claude-opus-5.5"]' + ); + expect(agentCall.env[AGENT_REQUEST_PLUGINS_ENV]).toBe( + '[{"id":"switchyard-router","algorithm":"stage"}]' + ); + expect(agentCall.argv[2]).toContain( + `--extension ${PI_REQUEST_PLUGINS_EXTENSION_PATH}` + ); + }); + it("does not configure request plugins for pi when none are requested", async () => { const execCalls: ExecCalls = []; const layer = makeTerminalBenchFakeSandboxLayer({ @@ -1019,6 +1074,7 @@ describe("terminal-bench pi via ori", () => { ); const agentCall = execCalls[0]; expect(agentCall?.env[AGENT_REQUEST_PLUGINS_ENV]).toBeUndefined(); + expect(agentCall?.env[AGENT_CANDIDATE_MODELS_ENV]).toBeUndefined(); expect(agentCall?.argv[2]).not.toContain(PI_REQUEST_PLUGINS_EXTENSION_PATH); });