From 4f6f85a4d696bd7ac0505a735d2677ccbfa90858 Mon Sep 17 00:00:00 2001 From: Ayush Patel Date: Wed, 30 Sep 2026 02:31:31 +0000 Subject: [PATCH 1/3] feat(agent-cli): forward the switchyard-router plugin through sandbox request plugins --- .../agent-cli/candidate-models.test.ts | 14 ++++- src/benchmarks/agent-cli/candidate-models.ts | 11 +++- .../agent-cli/request-plugins.test.ts | 50 ++++++++++++++++- src/benchmarks/agent-cli/request-plugins.ts | 56 ++++++++++++------- src/benchmarks/agent-cli/schema.ts | 10 ++++ src/benchmarks/deep-swe/benchmark.ts | 18 ++++++ src/benchmarks/swe-atlas/benchmark.ts | 18 ++++++ .../terminal-bench/benchmark.test.ts | 12 ++++ src/benchmarks/terminal-bench/benchmark.ts | 1 + 9 files changed, 166 insertions(+), 24 deletions(-) diff --git a/src/benchmarks/agent-cli/candidate-models.test.ts b/src/benchmarks/agent-cli/candidate-models.test.ts index 53d3fcbf..6a4cab04 100644 --- a/src/benchmarks/agent-cli/candidate-models.test.ts +++ b/src/benchmarks/agent-cli/candidate-models.test.ts @@ -27,12 +27,22 @@ describe("sandboxAgentPluginError", () => { it("names the benchmark and agent when a switchyard algorithm would be dropped", () => { const error = sandboxAgentPluginError({ benchmarkId: "terminal_bench", - agent: "pi", + agent: "claude", models: undefined, switchyardAlgorithm: "stage", }); expect(error?.message).toBe( - "terminal_bench cannot use switchyardAlgorithm: the pi agent calls the model from inside the sandbox, which does not forward the switchyard-router plugin" + "terminal_bench cannot use switchyardAlgorithm: the claude agent calls the model from inside the sandbox and does not forward the switchyard-router plugin (supported agents: pi)" ); }); + it("accepts a switchyard algorithm for an agent that forwards the plugin", () => { + expect( + sandboxAgentPluginError({ + benchmarkId: "terminal_bench", + agent: "pi", + models: undefined, + switchyardAlgorithm: "stage", + }) + ).toBeUndefined(); + }); }); diff --git a/src/benchmarks/agent-cli/candidate-models.ts b/src/benchmarks/agent-cli/candidate-models.ts index 930ab62c..313c38ee 100644 --- a/src/benchmarks/agent-cli/candidate-models.ts +++ b/src/benchmarks/agent-cli/candidate-models.ts @@ -1,4 +1,8 @@ import type { SwitchyardAlgorithm } from "../../harness/constants"; +import { + forwardsSwitchyardRouterPlugin, + SWITCHYARD_ROUTER_PLUGIN_AGENTS, +} from "./schema"; export function sandboxAgentPluginError(opts: { readonly benchmarkId: string; @@ -11,9 +15,12 @@ export function sandboxAgentPluginError(opts: { `${opts.benchmarkId} cannot use candidate models: the ${opts.agent} agent calls the model from inside the sandbox, which does not forward the models list` ); } - if (opts.switchyardAlgorithm !== undefined) { + if ( + opts.switchyardAlgorithm !== undefined && + !forwardsSwitchyardRouterPlugin(opts.agent) + ) { return new Error( - `${opts.benchmarkId} cannot use switchyardAlgorithm: the ${opts.agent} agent calls the model from inside the sandbox, which does not forward the switchyard-router plugin` + `${opts.benchmarkId} cannot use switchyardAlgorithm: the ${opts.agent} agent calls the model from inside the sandbox and does not forward the switchyard-router plugin (supported agents: ${SWITCHYARD_ROUTER_PLUGIN_AGENTS.join(", ")})` ); } return undefined; diff --git a/src/benchmarks/agent-cli/request-plugins.test.ts b/src/benchmarks/agent-cli/request-plugins.test.ts index f530d898..bd62f8b8 100644 --- a/src/benchmarks/agent-cli/request-plugins.test.ts +++ b/src/benchmarks/agent-cli/request-plugins.test.ts @@ -1,7 +1,9 @@ import { describe, expect, it } from "bun:test"; -import { getOriHarness } from "./harness"; +import { SWITCHYARD_ALGORITHMS } from "../../harness/constants"; +import { getOriHarness, ORI_HARNESSES } from "./harness"; import { sandboxAgentRequestPlugins } from "./request-plugins"; +import { SWITCHYARD_ROUTER_PLUGIN_AGENTS } from "./schema"; const BASE = { benchmarkId: "terminal_bench", @@ -83,4 +85,50 @@ describe("sandboxAgentRequestPlugins", () => { "terminal_bench cannot use costQualityTradeoff, pinModel: the omp agent" ); }); + + it("builds the switchyard-router plugin for every algorithm on nvidia/switchyard", () => { + for (const switchyardAlgorithm of SWITCHYARD_ALGORITHMS) { + expect( + sandboxAgentRequestPlugins({ + ...BASE, + harness: getOriHarness("pi"), + model: "nvidia/switchyard:nitro", + switchyardAlgorithm, + }) + ).toEqual([{ id: "switchyard-router", algorithm: switchyardAlgorithm }]); + } + }); + + it("returns no switchyard-router plugin off nvidia/switchyard", () => { + expect( + sandboxAgentRequestPlugins({ + ...BASE, + harness: getOriHarness("claude"), + model: "openai/gpt-5", + switchyardAlgorithm: "stage", + }) + ).toEqual([]); + }); + + it("rejects a switchyard algorithm for agents that cannot forward the plugin", () => { + for (const agent of ["claude", "prime-agent", "omp"] as const) { + const result = sandboxAgentRequestPlugins({ + ...BASE, + harness: getOriHarness(agent), + model: "nvidia/switchyard", + switchyardAlgorithm: "stage", + }); + expect(result instanceof Error ? result.message : "").toBe( + `terminal_bench cannot use switchyardAlgorithm: the ${agent} agent calls the model from inside the sandbox and does not forward the switchyard-router plugin (supported agents: pi)` + ); + } + }); + + it("lists exactly the harnesses that forward request plugins as switchyard-router agents", () => { + expect( + Object.values(ORI_HARNESSES) + .filter((harness) => harness.forwardsRequestPlugins) + .map((harness) => harness.id) + ).toEqual([...SWITCHYARD_ROUTER_PLUGIN_AGENTS]); + }); }); diff --git a/src/benchmarks/agent-cli/request-plugins.ts b/src/benchmarks/agent-cli/request-plugins.ts index e771679a..70a13d20 100644 --- a/src/benchmarks/agent-cli/request-plugins.ts +++ b/src/benchmarks/agent-cli/request-plugins.ts @@ -1,9 +1,11 @@ +import type { SwitchyardAlgorithm } from "../../harness/constants"; import { stripVariantSuffix } from "../../harness/constants"; import type { GenerateConfig } from "../../harness/model"; import { buildAutoRouterPlugin, toWireAutoRouterPlugin, } from "../../providers/auto-router-plugin"; +import { buildSwitchyardRouterPlugin } from "../../providers/switchyard-router-plugin"; import type { OriHarnessDef } from "./harness"; import { ORI_HARNESSES } from "./harness"; import type { AgentRequestPlugin } from "./runner"; @@ -15,29 +17,45 @@ export function sandboxAgentRequestPlugins(opts: { readonly costTier: GenerateConfig["costTier"]; readonly costQualityTradeoff: number | undefined; readonly pinModel: boolean | undefined; + readonly switchyardAlgorithm?: SwitchyardAlgorithm | undefined; }): readonly AgentRequestPlugin[] | Error { - const plugin = buildAutoRouterPlugin(stripVariantSuffix(opts.model), { + const baseModel = stripVariantSuffix(opts.model); + const autoRouterPlugin = buildAutoRouterPlugin(baseModel, { costTier: opts.costTier, costQualityTradeoff: opts.costQualityTradeoff, pinModel: opts.pinModel, }); - if (plugin === undefined) { - return []; + const switchyardRouterPlugin = buildSwitchyardRouterPlugin( + baseModel, + opts.switchyardAlgorithm + ); + const plugins: AgentRequestPlugin[] = [ + ...(autoRouterPlugin === undefined + ? [] + : [toWireAutoRouterPlugin(autoRouterPlugin)]), + ...(switchyardRouterPlugin === undefined + ? [] + : [{ ...switchyardRouterPlugin }]), + ]; + if (plugins.length === 0 || opts.harness.forwardsRequestPlugins) { + return plugins; } - if (!opts.harness.forwardsRequestPlugins) { - const fields = [ - opts.costTier !== undefined ? "costTier" : undefined, - opts.costQualityTradeoff !== undefined - ? "costQualityTradeoff" - : undefined, - opts.pinModel === true ? "pinModel" : undefined, - ].filter((field) => field !== undefined); - const forwardingAgents = Object.values(ORI_HARNESSES) - .filter((harness) => harness.forwardsRequestPlugins) - .map((harness) => harness.id); - return new Error( - `${opts.benchmarkId} cannot use ${fields.join(", ")}: the ${opts.harness.id} agent calls the model from inside the sandbox and does not forward the ${plugin.id} plugin (supported agents: ${forwardingAgents.join(", ")})` - ); - } - return [toWireAutoRouterPlugin(plugin)]; + const fields = [ + autoRouterPlugin !== undefined && opts.costTier !== undefined + ? "costTier" + : undefined, + autoRouterPlugin !== undefined && opts.costQualityTradeoff !== undefined + ? "costQualityTradeoff" + : undefined, + autoRouterPlugin !== undefined && opts.pinModel === true + ? "pinModel" + : undefined, + switchyardRouterPlugin !== undefined ? "switchyardAlgorithm" : undefined, + ].filter((field) => field !== undefined); + const forwardingAgents = Object.values(ORI_HARNESSES) + .filter((harness) => harness.forwardsRequestPlugins) + .map((harness) => harness.id); + return new Error( + `${opts.benchmarkId} cannot use ${fields.join(", ")}: the ${opts.harness.id} agent calls the model from inside the sandbox and does not forward the ${plugins.map((plugin) => plugin.id).join(", ")} plugin (supported agents: ${forwardingAgents.join(", ")})` + ); } diff --git a/src/benchmarks/agent-cli/schema.ts b/src/benchmarks/agent-cli/schema.ts index bdd69e91..b0441061 100644 --- a/src/benchmarks/agent-cli/schema.ts +++ b/src/benchmarks/agent-cli/schema.ts @@ -6,6 +6,16 @@ export const ORI_AGENTS = ["pi", "claude", "prime-agent", "omp"] as const; export type OriAgent = ValueOf; +export const SWITCHYARD_ROUTER_PLUGIN_AGENTS = [ + "pi", +] as const satisfies readonly OriAgent[]; + +export function forwardsSwitchyardRouterPlugin(agent: string): boolean { + return SWITCHYARD_ROUTER_PLUGIN_AGENTS.some( + (candidate) => candidate === agent + ); +} + export const HARBOR_AGENTS = ["mini_swe", ...ORI_AGENTS] as const; export type HarborAgent = ValueOf; diff --git a/src/benchmarks/deep-swe/benchmark.ts b/src/benchmarks/deep-swe/benchmark.ts index 07a6fb00..d447f9d5 100644 --- a/src/benchmarks/deep-swe/benchmark.ts +++ b/src/benchmarks/deep-swe/benchmark.ts @@ -21,6 +21,8 @@ import { makeModalSandboxLayer } from "../../sandbox/modal"; import { resolveModalRegions } from "../../sandbox/modal-regions"; import { SandboxSession } from "../../sandbox/session"; import { sandboxAgentPluginError } from "../agent-cli/candidate-models"; +import { getOriHarness } from "../agent-cli/harness"; +import { sandboxAgentRequestPlugins } from "../agent-cli/request-plugins"; import { isOriAgent } from "../agent-cli/schema"; import { DEEP_SWE_META } from "../benchmark-meta"; import type { Benchmark, BenchmarkRunInput } from "../types"; @@ -50,6 +52,20 @@ function makeDeepSweLayer( if (candidateModelsError !== undefined) { return layerFail(candidateModelsError); } + const requestPlugins = isOriAgent(benchmarkConfig.agent) + ? sandboxAgentRequestPlugins({ + benchmarkId: benchmarkConfig.benchmarkId, + harness: getOriHarness(benchmarkConfig.agent), + model: benchmarkConfig.model, + costTier: undefined, + costQualityTradeoff: undefined, + pinModel: undefined, + switchyardAlgorithm: benchmarkConfig.switchyardAlgorithm, + }) + : []; + if (requestPlugins instanceof Error) { + return layerFail(requestPlugins); + } const datasetLayer = makeDeepSweDatasetLayer( definedValues({ taskSubset: benchmarkConfig.taskSubset, @@ -104,6 +120,8 @@ function makeDeepSweLayer( allowedTools: benchmarkConfig.allowedTools, disallowedTools: benchmarkConfig.disallowedTools, isolateAgentConfig: benchmarkConfig.isolateAgentConfig, + requestPlugins: + requestPlugins.length > 0 ? requestPlugins : undefined, }), endpointId: benchmarkConfig.endpointId, sessionId: input.sessionId, diff --git a/src/benchmarks/swe-atlas/benchmark.ts b/src/benchmarks/swe-atlas/benchmark.ts index 44a9a60b..9cfc5245 100644 --- a/src/benchmarks/swe-atlas/benchmark.ts +++ b/src/benchmarks/swe-atlas/benchmark.ts @@ -21,6 +21,8 @@ import { makeModalSandboxLayer } from "../../sandbox/modal"; import { resolveModalRegions } from "../../sandbox/modal-regions"; import { SandboxSession } from "../../sandbox/session"; import { sandboxAgentPluginError } from "../agent-cli/candidate-models"; +import { getOriHarness } from "../agent-cli/harness"; +import { sandboxAgentRequestPlugins } from "../agent-cli/request-plugins"; import { isOriAgent } from "../agent-cli/schema"; import { SWE_ATLAS_QA_META, @@ -57,6 +59,20 @@ function makeSweAtlasLayer( if (candidateModelsError !== undefined) { return layerFail(candidateModelsError); } + const requestPlugins = isOriAgent(benchmarkConfig.agent) + ? sandboxAgentRequestPlugins({ + benchmarkId: benchmarkConfig.benchmarkId, + harness: getOriHarness(benchmarkConfig.agent), + model: benchmarkConfig.model, + costTier: undefined, + costQualityTradeoff: undefined, + pinModel: undefined, + switchyardAlgorithm: benchmarkConfig.switchyardAlgorithm, + }) + : []; + if (requestPlugins instanceof Error) { + return layerFail(requestPlugins); + } const datasetLayer = makeSweAtlasDatasetLayer( definedValues({ track, @@ -114,6 +130,8 @@ function makeSweAtlasLayer( allowedTools: benchmarkConfig.allowedTools, disallowedTools: benchmarkConfig.disallowedTools, isolateAgentConfig: benchmarkConfig.isolateAgentConfig, + requestPlugins: + requestPlugins.length > 0 ? requestPlugins : undefined, }), endpointId: benchmarkConfig.endpointId, inference: { diff --git a/src/benchmarks/terminal-bench/benchmark.test.ts b/src/benchmarks/terminal-bench/benchmark.test.ts index 5d26c5e6..44397058 100644 --- a/src/benchmarks/terminal-bench/benchmark.test.ts +++ b/src/benchmarks/terminal-bench/benchmark.test.ts @@ -38,4 +38,16 @@ describe("terminal_bench layer", () => { "terminal_bench cannot use costTier: the claude agent calls the model from inside the sandbox and does not forward the auto-router plugin (supported agents: pi)" ); }); + + it("rejects a switchyard algorithm for an agent that cannot forward the switchyard-router plugin", async () => { + const exit = await buildLayer({ + agent: "claude", + model: "nvidia/switchyard", + switchyardAlgorithm: "stage", + }); + assertFailure(exit); + expect(getOrThrow(failureOption(exit.cause)).message).toBe( + "terminal_bench cannot use switchyardAlgorithm: the claude agent calls the model from inside the sandbox and does not forward the switchyard-router plugin (supported agents: pi)" + ); + }); }); diff --git a/src/benchmarks/terminal-bench/benchmark.ts b/src/benchmarks/terminal-bench/benchmark.ts index a778ae43..577c7571 100644 --- a/src/benchmarks/terminal-bench/benchmark.ts +++ b/src/benchmarks/terminal-bench/benchmark.ts @@ -57,6 +57,7 @@ function makeTerminalBenchLayer( costTier: benchmarkConfig.costTier, costQualityTradeoff: benchmarkConfig.costQualityTradeoff, pinModel: benchmarkConfig.pinModel, + switchyardAlgorithm: benchmarkConfig.switchyardAlgorithm, }); if (requestPlugins instanceof Error) { return layerFail(requestPlugins); From 96bf7bef0322eeec61da6f1011558a41286f131b Mon Sep 17 00:00:00 2001 From: Ayush Patel Date: Thu, 1 Oct 2026 09:15:24 -0400 Subject: [PATCH 2/3] feat: forward candidate models to the in-sandbox pi agent --- .../agent-cli/candidate-models.test.ts | 14 ++++- src/benchmarks/agent-cli/candidate-models.ts | 7 ++- src/benchmarks/agent-cli/harness.test.ts | 49 +++++++++++++++- src/benchmarks/agent-cli/harness.ts | 15 +++-- src/benchmarks/agent-cli/runner.ts | 18 +++++- src/benchmarks/deep-swe/benchmark.ts | 1 + src/benchmarks/swe-atlas/benchmark.ts | 1 + .../terminal-bench/benchmark.test.ts | 22 ++++++++ src/benchmarks/terminal-bench/benchmark.ts | 1 + .../terminal-bench/ori-solver.test.ts | 56 +++++++++++++++++++ 10 files changed, 171 insertions(+), 13 deletions(-) diff --git a/src/benchmarks/agent-cli/candidate-models.test.ts b/src/benchmarks/agent-cli/candidate-models.test.ts index 6a4cab04..796121e3 100644 --- a/src/benchmarks/agent-cli/candidate-models.test.ts +++ b/src/benchmarks/agent-cli/candidate-models.test.ts @@ -16,12 +16,12 @@ describe("sandboxAgentPluginError", () => { it("names the benchmark and agent when a candidate list would be dropped", () => { const error = sandboxAgentPluginError({ benchmarkId: "terminal_bench", - agent: "pi", + agent: "claude", models: ["openai/gpt-4.1-nano", "anthropic/claude-sonnet-4.5"], switchyardAlgorithm: undefined, }); expect(error?.message).toBe( - "terminal_bench cannot use candidate models: the pi agent calls the model from inside the sandbox, which does not forward the models list" + "terminal_bench cannot use candidate models: the claude agent calls the model from inside the sandbox and does not forward the models list (supported agents: pi)" ); }); it("names the benchmark and agent when a switchyard algorithm would be dropped", () => { @@ -35,6 +35,16 @@ describe("sandboxAgentPluginError", () => { "terminal_bench cannot use switchyardAlgorithm: the claude agent calls the model from inside the sandbox and does not forward the switchyard-router plugin (supported agents: pi)" ); }); + it("accepts candidate models and an algorithm for an agent that forwards them", () => { + expect( + sandboxAgentPluginError({ + benchmarkId: "terminal_bench", + agent: "pi", + models: ["z-ai/glm-5.3-flash", "anthropic/claude-opus-5.5"], + switchyardAlgorithm: "stage", + }) + ).toBeUndefined(); + }); it("accepts a switchyard algorithm for an agent that forwards the plugin", () => { expect( sandboxAgentPluginError({ diff --git a/src/benchmarks/agent-cli/candidate-models.ts b/src/benchmarks/agent-cli/candidate-models.ts index 313c38ee..3af03ed4 100644 --- a/src/benchmarks/agent-cli/candidate-models.ts +++ b/src/benchmarks/agent-cli/candidate-models.ts @@ -10,9 +10,12 @@ export function sandboxAgentPluginError(opts: { readonly models: readonly string[] | undefined; readonly switchyardAlgorithm: SwitchyardAlgorithm | undefined; }): Error | undefined { - if (opts.models !== undefined) { + if ( + opts.models !== undefined && + !forwardsSwitchyardRouterPlugin(opts.agent) + ) { return new Error( - `${opts.benchmarkId} cannot use candidate models: the ${opts.agent} agent calls the model from inside the sandbox, which does not forward the models list` + `${opts.benchmarkId} cannot use candidate models: the ${opts.agent} agent calls the model from inside the sandbox and does not forward the models list (supported agents: ${SWITCHYARD_ROUTER_PLUGIN_AGENTS.join(", ")})` ); } if ( diff --git a/src/benchmarks/agent-cli/harness.test.ts b/src/benchmarks/agent-cli/harness.test.ts index 8a16d6a1..04bfdef6 100644 --- a/src/benchmarks/agent-cli/harness.test.ts +++ b/src/benchmarks/agent-cli/harness.test.ts @@ -4,6 +4,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { + AGENT_CANDIDATE_MODELS_ENV, AGENT_REQUEST_PLUGINS_ENV, getOriHarness, ORI_HARNESSES, @@ -26,13 +27,20 @@ const RUN_OPTIONS: OriRunScriptOptions = { type ProviderRequestHandler = (event: { readonly payload: unknown }) => unknown; async function loadRequestPluginsHandler( - plugins: readonly unknown[] + plugins: readonly unknown[], + models?: readonly string[] ): Promise { const dir = mkdtempSync(join(tmpdir(), "pi-request-plugins-")); const path = join(dir, "openrouter-request-plugins.ts"); writeFileSync(path, PI_REQUEST_PLUGINS_EXTENSION_SOURCE); const previous = process.env[AGENT_REQUEST_PLUGINS_ENV]; + const previousModels = process.env[AGENT_CANDIDATE_MODELS_ENV]; process.env[AGENT_REQUEST_PLUGINS_ENV] = JSON.stringify(plugins); + if (models === undefined) { + Reflect.deleteProperty(process.env, AGENT_CANDIDATE_MODELS_ENV); + } else { + process.env[AGENT_CANDIDATE_MODELS_ENV] = JSON.stringify(models); + } try { const extension: { readonly default: (pi: { @@ -56,6 +64,11 @@ async function loadRequestPluginsHandler( } else { process.env[AGENT_REQUEST_PLUGINS_ENV] = previous; } + if (previousModels === undefined) { + Reflect.deleteProperty(process.env, AGENT_CANDIDATE_MODELS_ENV); + } else { + process.env[AGENT_CANDIDATE_MODELS_ENV] = previousModels; + } } } @@ -93,6 +106,36 @@ describe("pi request plugins extension", () => { }); }); + it("adds the candidate models and the switchyard-router plugin together", async () => { + const handler = await loadRequestPluginsHandler( + [{ id: "switchyard-router", algorithm: "stage" }], + ["z-ai/glm-5.3-flash", "anthropic/claude-opus-5.5"] + ); + expect( + handler({ payload: { model: "nvidia/switchyard", input: [] } }) + ).toEqual({ + model: "nvidia/switchyard", + input: [], + models: ["z-ai/glm-5.3-flash", "anthropic/claude-opus-5.5"], + plugins: [{ id: "switchyard-router", algorithm: "stage" }], + }); + }); + + it("adds only the candidate models when no plugins are configured", async () => { + const handler = await loadRequestPluginsHandler( + [], + ["deepseek/deepseek-v4.1-flash", "openai/gpt-6-sol"] + ); + expect( + handler({ + payload: { model: "nvidia/switchyard", models: ["stale/model"] }, + }) + ).toEqual({ + model: "nvidia/switchyard", + models: ["deepseek/deepseek-v4.1-flash", "openai/gpt-6-sol"], + }); + }); + it("leaves non-object payloads unchanged", async () => { const handler = await loadRequestPluginsHandler([ { id: "auto-router", cost_tier: "low" }, @@ -107,7 +150,7 @@ describe("pi request plugins run script", () => { const pi = getOriHarness("pi"); const script = pi.buildRunScript({ ...RUN_OPTIONS, - hasRequestPlugins: true, + loadsRequestExtension: true, }); expect(script).toContain( `cat > ${PI_REQUEST_PLUGINS_EXTENSION_PATH} <<'TB_PI_REQUEST_PLUGINS_EXTENSION'\n${PI_REQUEST_PLUGINS_EXTENSION_SOURCE}\nTB_PI_REQUEST_PLUGINS_EXTENSION\n` @@ -126,7 +169,7 @@ describe("pi request plugins run script", () => { it("loads the extension explicitly even when agent config is isolated", () => { const script = getOriHarness("pi").buildRunScript({ ...RUN_OPTIONS, - hasRequestPlugins: true, + loadsRequestExtension: true, isolateAgentConfig: true, }); expect(script).toContain("--no-extensions"); diff --git a/src/benchmarks/agent-cli/harness.ts b/src/benchmarks/agent-cli/harness.ts index db3c9d8a..6efb8b2c 100644 --- a/src/benchmarks/agent-cli/harness.ts +++ b/src/benchmarks/agent-cli/harness.ts @@ -47,6 +47,8 @@ export const DEFAULT_AGENT_RUNTIME_SHA256 = export const AGENT_REQUEST_PLUGINS_ENV = "TB_OPENROUTER_PLUGINS" as const; +export const AGENT_CANDIDATE_MODELS_ENV = "TB_OPENROUTER_MODELS" as const; + const PI_REQUEST_PLUGINS_EXTENSION_DIR = "/root/.bench-harness"; export const PI_REQUEST_PLUGINS_EXTENSION_PATH = @@ -56,6 +58,7 @@ const PI_REQUEST_PLUGINS_EXTENSION_EOF = "TB_PI_REQUEST_PLUGINS_EXTENSION"; export const PI_REQUEST_PLUGINS_EXTENSION_SOURCE = [ `const plugins = JSON.parse(process.env.${AGENT_REQUEST_PLUGINS_ENV} ?? "[]");`, + `const models = JSON.parse(process.env.${AGENT_CANDIDATE_MODELS_ENV} ?? "[]");`, "const pluginIds = new Set(plugins.map((plugin) => plugin.id));", "export default function registerRequestPlugins(pi) {", ' pi.on("before_provider_request", (event) => {', @@ -63,10 +66,14 @@ export const PI_REQUEST_PLUGINS_EXTENSION_SOURCE = [ ' if (typeof payload !== "object" || payload === null || Array.isArray(payload)) {', " return undefined;", " }", + " const withModels = models.length > 0 ? { ...payload, models } : payload;", + " if (plugins.length === 0) {", + " return withModels;", + " }", " const callerPlugins = Array.isArray(payload.plugins)", " ? payload.plugins.filter((plugin) => !pluginIds.has(plugin?.id))", " : [];", - " return { ...payload, plugins: [...callerPlugins, ...plugins] };", + " return { ...withModels, plugins: [...callerPlugins, ...plugins] };", " });", "}", ].join("\n"); @@ -89,7 +96,7 @@ export interface OriRunScriptOptions { readonly hasAllowedTools: boolean; readonly hasDisallowedTools: boolean; readonly isolateAgentConfig: boolean; - readonly hasRequestPlugins?: boolean; + readonly loadsRequestExtension?: boolean; } export interface OriImageStepsOptions { @@ -437,13 +444,13 @@ const ORI_PI_HARNESS: OriHarnessDef = { "set -euo pipefail", "export HOME=/root", "mkdir -p /logs/agent", - ...(options.hasRequestPlugins === true + ...(options.loadsRequestExtension === true ? writePiRequestPluginsExtension() : []), 'ori pi --model "$TB_MODEL" \\', ` --reasoning-effort ${options.reasoningEffort} -- \\`, " --print --mode json --no-session \\", - ...(options.hasRequestPlugins === true + ...(options.loadsRequestExtension === true ? [` --extension ${PI_REQUEST_PLUGINS_EXTENSION_PATH} \\`] : []), ...(options.hasSystemPrompt diff --git a/src/benchmarks/agent-cli/runner.ts b/src/benchmarks/agent-cli/runner.ts index 54cc37a9..f5c472a6 100644 --- a/src/benchmarks/agent-cli/runner.ts +++ b/src/benchmarks/agent-cli/runner.ts @@ -24,7 +24,10 @@ import { import { getCurrentEpoch } from "../../runtime/response-cache"; import type { SandboxSessionInstance } from "../../sandbox/session"; import type { OriAgentRun, OriHarnessDef } from "./harness"; -import { AGENT_REQUEST_PLUGINS_ENV } from "./harness"; +import { + AGENT_CANDIDATE_MODELS_ENV, + AGENT_REQUEST_PLUGINS_ENV, +} from "./harness"; import type { OriChannel, OriReasoningEffort } from "./schema"; import { DEFAULT_ORI_CHANNEL, DEFAULT_ORI_INSTALL_URL } from "./schema"; @@ -150,6 +153,7 @@ export interface AgentCliOpts { readonly disallowedTools?: readonly string[]; readonly isolateAgentConfig?: boolean; readonly requestPlugins?: readonly AgentRequestPlugin[]; + readonly models?: readonly string[]; } export interface AgentCliRunResult extends OriAgentRun { @@ -196,6 +200,10 @@ export function buildAgentCliEnv(opts: AgentCliOpts): Record { if (requestPlugins.length > 0) { env[AGENT_REQUEST_PLUGINS_ENV] = JSON.stringify(requestPlugins); } + const models = opts.models ?? []; + if (models.length > 0) { + env[AGENT_CANDIDATE_MODELS_ENV] = JSON.stringify(models); + } return env; } @@ -241,6 +249,7 @@ export function runAgentCli(input: { }): Effect { const { session, harness, opts, instructionPath, timeoutMs } = input; const requestPlugins = opts.requestPlugins ?? []; + const models = opts.models ?? []; const script = harness.buildRunScript({ instructionPath, logPath: harness.remoteLogPath, @@ -250,7 +259,7 @@ export function runAgentCli(input: { hasAllowedTools: (opts.allowedTools ?? []).length > 0, hasDisallowedTools: (opts.disallowedTools ?? []).length > 0, isolateAgentConfig: opts.isolateAgentConfig === true, - hasRequestPlugins: requestPlugins.length > 0, + loadsRequestExtension: requestPlugins.length > 0 || models.length > 0, }); return gen(function* () { if (requestPlugins.length > 0 && !harness.forwardsRequestPlugins) { @@ -258,6 +267,11 @@ export function runAgentCli(input: { message: `the ${harness.id} agent does not forward OpenRouter request plugins (${requestPlugins.map((plugin) => plugin.id).join(", ")}) from inside the sandbox`, }); } + if (models.length > 0 && !harness.forwardsRequestPlugins) { + return yield* new SolverError({ + message: `the ${harness.id} agent does not forward the OpenRouter models list from inside the sandbox`, + }); + } if ( opts.sessionId !== undefined && opts.sessionId.length > 0 && diff --git a/src/benchmarks/deep-swe/benchmark.ts b/src/benchmarks/deep-swe/benchmark.ts index d447f9d5..2d029bc6 100644 --- a/src/benchmarks/deep-swe/benchmark.ts +++ b/src/benchmarks/deep-swe/benchmark.ts @@ -122,6 +122,7 @@ function makeDeepSweLayer( isolateAgentConfig: benchmarkConfig.isolateAgentConfig, requestPlugins: requestPlugins.length > 0 ? requestPlugins : undefined, + models: benchmarkConfig.models, }), endpointId: benchmarkConfig.endpointId, sessionId: input.sessionId, diff --git a/src/benchmarks/swe-atlas/benchmark.ts b/src/benchmarks/swe-atlas/benchmark.ts index 9cfc5245..d036a9b6 100644 --- a/src/benchmarks/swe-atlas/benchmark.ts +++ b/src/benchmarks/swe-atlas/benchmark.ts @@ -132,6 +132,7 @@ function makeSweAtlasLayer( isolateAgentConfig: benchmarkConfig.isolateAgentConfig, requestPlugins: requestPlugins.length > 0 ? requestPlugins : undefined, + models: benchmarkConfig.models, }), endpointId: benchmarkConfig.endpointId, inference: { diff --git a/src/benchmarks/terminal-bench/benchmark.test.ts b/src/benchmarks/terminal-bench/benchmark.test.ts index 44397058..c1205285 100644 --- a/src/benchmarks/terminal-bench/benchmark.test.ts +++ b/src/benchmarks/terminal-bench/benchmark.test.ts @@ -50,4 +50,26 @@ describe("terminal_bench layer", () => { "terminal_bench cannot use switchyardAlgorithm: the claude agent calls the model from inside the sandbox and does not forward the switchyard-router plugin (supported agents: pi)" ); }); + + it("rejects candidate models for an agent that cannot forward them", async () => { + const exit = await buildLayer({ + agent: "claude", + model: "nvidia/switchyard", + models: ["z-ai/glm-5.3-flash", "anthropic/claude-opus-5.5"], + }); + assertFailure(exit); + expect(getOrThrow(failureOption(exit.cause)).message).toBe( + "terminal_bench cannot use candidate models: the claude agent calls the model from inside the sandbox and does not forward the models list (supported agents: pi)" + ); + }); + + it("accepts candidate models and a switchyard algorithm for pi", async () => { + const exit = await buildLayer({ + agent: "pi", + model: "nvidia/switchyard", + models: ["z-ai/glm-5.3-flash", "anthropic/claude-opus-5.5"], + switchyardAlgorithm: "stage", + }); + expect(exit._tag).toBe("Success"); + }); }); diff --git a/src/benchmarks/terminal-bench/benchmark.ts b/src/benchmarks/terminal-bench/benchmark.ts index 577c7571..352081c3 100644 --- a/src/benchmarks/terminal-bench/benchmark.ts +++ b/src/benchmarks/terminal-bench/benchmark.ts @@ -77,6 +77,7 @@ function makeTerminalBenchLayer( disallowedTools: benchmarkConfig.disallowedTools, isolateAgentConfig: benchmarkConfig.isolateAgentConfig, requestPlugins: requestPlugins.length > 0 ? requestPlugins : undefined, + models: benchmarkConfig.models, }); const datasetLayer = makeTerminalBenchDatasetLayer( definedValues({ diff --git a/src/benchmarks/terminal-bench/ori-solver.test.ts b/src/benchmarks/terminal-bench/ori-solver.test.ts index b6a0622e..672a23ea 100644 --- a/src/benchmarks/terminal-bench/ori-solver.test.ts +++ b/src/benchmarks/terminal-bench/ori-solver.test.ts @@ -49,6 +49,7 @@ import { GenerationResolver } from "../../runtime/generation-resolver"; import { setCurrentSampleId } from "../../runtime/request-session-id"; import { setCurrentEpoch } from "../../runtime/response-cache"; import { + AGENT_CANDIDATE_MODELS_ENV, AGENT_REQUEST_PLUGINS_ENV, BUN_RELEASE_SHA256, BUN_RELEASE_URL, @@ -988,6 +989,60 @@ describe("terminal-bench pi via ori", () => { ); }); + it("forwards candidate models and the switchyard-router plugin to pi", async () => { + const execCalls: ExecCalls = []; + const layer = makeTerminalBenchFakeSandboxLayer({ + reward: 1, + execCalls, + agentExitCode: 0, + }); + const solverLayer = layerEffect(Solver)( + gen(function* () { + const sessionFactory = yield* SandboxSession; + return Solver.of( + oriSolver( + sessionFactory, + { + ...SOLVER_OPTS, + model: "nvidia/switchyard", + models: ["z-ai/glm-5.3-flash", "anthropic/claude-opus-5.5"], + requestPlugins: [{ id: "switchyard-router", algorithm: "stage" }], + }, + getOriHarness("pi") + ) + ); + }) + ); + await runPromise( + gen(function* () { + const solver = yield* Solver; + return yield* solver(sampleState()); + }).pipe( + provide( + layerMergeAll( + solverLayer.pipe(layerProvide(layer)), + noopProgressLayer, + noopCheckpointLayer + ) + ) + ) + ); + const agentCall = execCalls[0]; + if (agentCall === undefined) { + throw new Error("fake sandbox did not capture the agent invocation"); + } + expect(agentCall.env["TB_MODEL"]).toBe("nvidia/switchyard"); + expect(agentCall.env[AGENT_CANDIDATE_MODELS_ENV]).toBe( + '["z-ai/glm-5.3-flash","anthropic/claude-opus-5.5"]' + ); + expect(agentCall.env[AGENT_REQUEST_PLUGINS_ENV]).toBe( + '[{"id":"switchyard-router","algorithm":"stage"}]' + ); + expect(agentCall.argv[2]).toContain( + `--extension ${PI_REQUEST_PLUGINS_EXTENSION_PATH}` + ); + }); + it("does not configure request plugins for pi when none are requested", async () => { const execCalls: ExecCalls = []; const layer = makeTerminalBenchFakeSandboxLayer({ @@ -1019,6 +1074,7 @@ describe("terminal-bench pi via ori", () => { ); const agentCall = execCalls[0]; expect(agentCall?.env[AGENT_REQUEST_PLUGINS_ENV]).toBeUndefined(); + expect(agentCall?.env[AGENT_CANDIDATE_MODELS_ENV]).toBeUndefined(); expect(agentCall?.argv[2]).not.toContain(PI_REQUEST_PLUGINS_EXTENSION_PATH); }); From 1410e8ce75ea5a5d592ee1ae8c3db537f455bb24 Mon Sep 17 00:00:00 2001 From: Ayush Patel Date: Thu, 1 Oct 2026 09:42:07 -0400 Subject: [PATCH 3/3] fix: drop pi's catalog output cap on Switchyard requests --- src/benchmarks/agent-cli/harness.test.ts | 23 ++++++++++++++++++++++- src/benchmarks/agent-cli/harness.ts | 3 ++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/src/benchmarks/agent-cli/harness.test.ts b/src/benchmarks/agent-cli/harness.test.ts index 04bfdef6..6dc90590 100644 --- a/src/benchmarks/agent-cli/harness.test.ts +++ b/src/benchmarks/agent-cli/harness.test.ts @@ -112,7 +112,13 @@ describe("pi request plugins extension", () => { ["z-ai/glm-5.3-flash", "anthropic/claude-opus-5.5"] ); expect( - handler({ payload: { model: "nvidia/switchyard", input: [] } }) + handler({ + payload: { + model: "nvidia/switchyard", + input: [], + max_output_tokens: 235_929, + }, + }) ).toEqual({ model: "nvidia/switchyard", input: [], @@ -136,6 +142,21 @@ describe("pi request plugins extension", () => { }); }); + it("keeps max_output_tokens when no candidate models are configured", async () => { + const handler = await loadRequestPluginsHandler([ + { id: "auto-router", cost_tier: "low" }, + ]); + expect( + handler({ + payload: { model: "openrouter/auto", max_output_tokens: 235_929 }, + }) + ).toEqual({ + model: "openrouter/auto", + max_output_tokens: 235_929, + plugins: [{ id: "auto-router", cost_tier: "low" }], + }); + }); + it("leaves non-object payloads unchanged", async () => { const handler = await loadRequestPluginsHandler([ { id: "auto-router", cost_tier: "low" }, diff --git a/src/benchmarks/agent-cli/harness.ts b/src/benchmarks/agent-cli/harness.ts index 6efb8b2c..64e2f2a6 100644 --- a/src/benchmarks/agent-cli/harness.ts +++ b/src/benchmarks/agent-cli/harness.ts @@ -66,7 +66,8 @@ export const PI_REQUEST_PLUGINS_EXTENSION_SOURCE = [ ' if (typeof payload !== "object" || payload === null || Array.isArray(payload)) {', " return undefined;", " }", - " const withModels = models.length > 0 ? { ...payload, models } : payload;", + " const { max_output_tokens: _maxOutputTokens, ...uncapped } = payload;", + " const withModels = models.length > 0 ? { ...uncapped, models } : payload;", " if (plugins.length === 0) {", " return withModels;", " }",