feat(agent-cli): forward Switchyard candidate models to the in-sandbox pi agent (2/2)

src/benchmarks/agent-cli/candidate-models.test.ts

@@ -16,12 +16,12 @@ describe("sandboxAgentPluginError", () => {
   it("names the benchmark and agent when a candidate list would be dropped", () => {
     const error = sandboxAgentPluginError({
       benchmarkId: "terminal_bench",
-      agent: "pi",
+      agent: "claude",
       models: ["openai/gpt-4.1-nano", "anthropic/claude-sonnet-4.5"],
       switchyardAlgorithm: undefined,
     });
     expect(error?.message).toBe(
-      "terminal_bench cannot use candidate models: the pi agent calls the model from inside the sandbox, which does not forward the models list"
+      "terminal_bench cannot use candidate models: the claude agent calls the model from inside the sandbox and does not forward the models list (supported agents: pi)"
     );
   });
   it("names the benchmark and agent when a switchyard algorithm would be dropped", () => {
@@ -35,6 +35,16 @@ describe("sandboxAgentPluginError", () => {
       "terminal_bench cannot use switchyardAlgorithm: the claude agent calls the model from inside the sandbox and does not forward the switchyard-router plugin (supported agents: pi)"
     );
   });
+  it("accepts candidate models and an algorithm for an agent that forwards them", () => {
+    expect(
+      sandboxAgentPluginError({
+        benchmarkId: "terminal_bench",
+        agent: "pi",
+        models: ["z-ai/glm-5.3-flash", "anthropic/claude-opus-5.5"],
+        switchyardAlgorithm: "stage",
+      })
+    ).toBeUndefined();
+  });
   it("accepts a switchyard algorithm for an agent that forwards the plugin", () => {
     expect(
       sandboxAgentPluginError({

src/benchmarks/agent-cli/candidate-models.ts

@@ -10,9 +10,12 @@ export function sandboxAgentPluginError(opts: {
   readonly models: readonly string[] | undefined;
   readonly switchyardAlgorithm: SwitchyardAlgorithm | undefined;
 }): Error | undefined {
-  if (opts.models !== undefined) {
+  if (
+    opts.models !== undefined &&
+    !forwardsSwitchyardRouterPlugin(opts.agent)
+  ) {
     return new Error(
-      `${opts.benchmarkId} cannot use candidate models: the ${opts.agent} agent calls the model from inside the sandbox, which does not forward the models list`
+      `${opts.benchmarkId} cannot use candidate models: the ${opts.agent} agent calls the model from inside the sandbox and does not forward the models list (supported agents: ${SWITCHYARD_ROUTER_PLUGIN_AGENTS.join(", ")})`
     );
   }
   if (

src/benchmarks/agent-cli/harness.test.ts

@@ -4,6 +4,7 @@ import { tmpdir } from "node:os";
 import { join } from "node:path";
 
 import {
+  AGENT_CANDIDATE_MODELS_ENV,
   AGENT_REQUEST_PLUGINS_ENV,
   getOriHarness,
   ORI_HARNESSES,
@@ -28,13 +29,20 @@ const RUN_OPTIONS: OriRunScriptOptions = {
 type ProviderRequestHandler = (event: { readonly payload: unknown }) => unknown;
 
 async function loadRequestPluginsHandler(
-  plugins: readonly unknown[]
+  plugins: readonly unknown[],
+  models?: readonly string[]
 ): Promise<ProviderRequestHandler> {
   const dir = mkdtempSync(join(tmpdir(), "pi-request-plugins-"));
   const path = join(dir, "openrouter-request-plugins.ts");
   writeFileSync(path, PI_REQUEST_PLUGINS_EXTENSION_SOURCE);
   const previous = process.env[AGENT_REQUEST_PLUGINS_ENV];
+  const previousModels = process.env[AGENT_CANDIDATE_MODELS_ENV];
   process.env[AGENT_REQUEST_PLUGINS_ENV] = JSON.stringify(plugins);
+  if (models === undefined) {
+    Reflect.deleteProperty(process.env, AGENT_CANDIDATE_MODELS_ENV);
+  } else {
+    process.env[AGENT_CANDIDATE_MODELS_ENV] = JSON.stringify(models);
+  }
   try {
     const extension: {
       readonly default: (pi: {
@@ -58,6 +66,11 @@ async function loadRequestPluginsHandler(
     } else {
       process.env[AGENT_REQUEST_PLUGINS_ENV] = previous;
     }
+    if (previousModels === undefined) {
+      Reflect.deleteProperty(process.env, AGENT_CANDIDATE_MODELS_ENV);
+    } else {
+      process.env[AGENT_CANDIDATE_MODELS_ENV] = previousModels;
+    }
   }
 }
 
@@ -95,6 +108,57 @@ describe("pi request plugins extension", () => {
     });
   });
 
+  it("adds the candidate models and the switchyard-router plugin together", async () => {
+    const handler = await loadRequestPluginsHandler(
+      [{ id: "switchyard-router", algorithm: "stage" }],
+      ["z-ai/glm-5.3-flash", "anthropic/claude-opus-5.5"]
+    );
+    expect(
+      handler({
+        payload: {
+          model: "nvidia/switchyard",
+          input: [],
+          max_output_tokens: 235_929,
+        },
+      })
+    ).toEqual({
+      model: "nvidia/switchyard",
+      input: [],
+      models: ["z-ai/glm-5.3-flash", "anthropic/claude-opus-5.5"],
+      plugins: [{ id: "switchyard-router", algorithm: "stage" }],
+    });
+  });
+
+  it("adds only the candidate models when no plugins are configured", async () => {
+    const handler = await loadRequestPluginsHandler(
+      [],
+      ["deepseek/deepseek-v4.1-flash", "openai/gpt-6-sol"]
+    );
+    expect(
+      handler({
+        payload: { model: "nvidia/switchyard", models: ["stale/model"] },
+      })
+    ).toEqual({
+      model: "nvidia/switchyard",
+      models: ["deepseek/deepseek-v4.1-flash", "openai/gpt-6-sol"],
+    });
+  });
+
+  it("keeps max_output_tokens when no candidate models are configured", async () => {
+    const handler = await loadRequestPluginsHandler([
+      { id: "auto-router", cost_tier: "low" },
+    ]);
+    expect(
+      handler({
+        payload: { model: "openrouter/auto", max_output_tokens: 235_929 },
+      })
+    ).toEqual({
+      model: "openrouter/auto",
+      max_output_tokens: 235_929,
+      plugins: [{ id: "auto-router", cost_tier: "low" }],
+    });
+  });
+
   it("leaves non-object payloads unchanged", async () => {
     const handler = await loadRequestPluginsHandler([
       { id: "auto-router", cost_tier: "low" },
@@ -109,7 +173,7 @@ describe("pi request plugins run script", () => {
     const pi = getOriHarness("pi");
     const script = pi.buildRunScript({
       ...RUN_OPTIONS,
-      hasRequestPlugins: true,
+      loadsRequestExtension: true,
     });
     expect(script).toContain(
       `cat > ${PI_REQUEST_PLUGINS_EXTENSION_PATH} <<'TB_PI_REQUEST_PLUGINS_EXTENSION'\n${PI_REQUEST_PLUGINS_EXTENSION_SOURCE}\nTB_PI_REQUEST_PLUGINS_EXTENSION\n`
@@ -128,7 +192,7 @@ describe("pi request plugins run script", () => {
   it("loads the extension explicitly even when agent config is isolated", () => {
     const script = getOriHarness("pi").buildRunScript({
       ...RUN_OPTIONS,
-      hasRequestPlugins: true,
+      loadsRequestExtension: true,
       isolateAgentConfig: true,
     });
     expect(script).toContain("--no-extensions");

src/benchmarks/agent-cli/harness.ts

@@ -47,6 +47,8 @@ export const DEFAULT_AGENT_RUNTIME_SHA256 =
 
 export const AGENT_REQUEST_PLUGINS_ENV = "TB_OPENROUTER_PLUGINS" as const;
 
+export const AGENT_CANDIDATE_MODELS_ENV = "TB_OPENROUTER_MODELS" as const;
+
 const PI_REQUEST_PLUGINS_EXTENSION_DIR = "/root/.bench-harness";
 
 export const PI_REQUEST_PLUGINS_EXTENSION_PATH =
@@ -56,17 +58,23 @@ const PI_REQUEST_PLUGINS_EXTENSION_EOF = "TB_PI_REQUEST_PLUGINS_EXTENSION";
 
 export const PI_REQUEST_PLUGINS_EXTENSION_SOURCE = [
   `const plugins = JSON.parse(process.env.${AGENT_REQUEST_PLUGINS_ENV} ?? "[]");`,
+  `const models = JSON.parse(process.env.${AGENT_CANDIDATE_MODELS_ENV} ?? "[]");`,
   "const pluginIds = new Set(plugins.map((plugin) => plugin.id));",
   "export default function registerRequestPlugins(pi) {",
   '  pi.on("before_provider_request", (event) => {',
   "    const payload = event.payload;",
   '    if (typeof payload !== "object" || payload === null || Array.isArray(payload)) {',
   "      return undefined;",
   "    }",
+  "    const { max_output_tokens: _maxOutputTokens, ...uncapped } = payload;",
+  "    const withModels = models.length > 0 ? { ...uncapped, models } : payload;",
+  "    if (plugins.length === 0) {",
+  "      return withModels;",
+  "    }",
   "    const callerPlugins = Array.isArray(payload.plugins)",
   "      ? payload.plugins.filter((plugin) => !pluginIds.has(plugin?.id))",
   "      : [];",
-  "    return { ...payload, plugins: [...callerPlugins, ...plugins] };",
+  "    return { ...withModels, plugins: [...callerPlugins, ...plugins] };",
   "  });",
   "}",
 ].join("\n");
@@ -162,7 +170,7 @@ export interface OriRunScriptOptions {
   readonly hasAllowedTools: boolean;
   readonly hasDisallowedTools: boolean;
   readonly isolateAgentConfig: boolean;
-  readonly hasRequestPlugins?: boolean;
+  readonly loadsRequestExtension?: boolean;
 }
 
 export interface OriImageStepsOptions {
@@ -512,13 +520,13 @@ const ORI_PI_HARNESS: OriHarnessDef = {
       "export HOME=/root",
       "mkdir -p /logs/agent",
       ...writePiUsageExtension(),
-      ...(options.hasRequestPlugins === true
+      ...(options.loadsRequestExtension === true
         ? writePiRequestPluginsExtension()
         : []),
       'ori pi --model "$TB_MODEL" \\',
       `  --reasoning-effort ${options.reasoningEffort} -- \\`,
       "  --print --mode json --no-session \\",
-      ...(options.hasRequestPlugins === true
+      ...(options.loadsRequestExtension === true
         ? [`  --extension ${PI_REQUEST_PLUGINS_EXTENSION_PATH} \\`]
         : []),
       `  --extension ${PI_USAGE_EXTENSION_PATH} \\`,

src/benchmarks/agent-cli/runner.ts

@@ -24,7 +24,10 @@ import {
 import { getCurrentEpoch } from "../../runtime/response-cache";
 import type { SandboxSessionInstance } from "../../sandbox/session";
 import type { OriAgentRun, OriHarnessDef } from "./harness";
-import { AGENT_REQUEST_PLUGINS_ENV } from "./harness";
+import {
+  AGENT_CANDIDATE_MODELS_ENV,
+  AGENT_REQUEST_PLUGINS_ENV,
+} from "./harness";
 import type { OriChannel, OriReasoningEffort } from "./schema";
 import { DEFAULT_ORI_CHANNEL, DEFAULT_ORI_INSTALL_URL } from "./schema";
 
@@ -157,6 +160,7 @@ export interface AgentCliOpts {
   readonly disallowedTools?: readonly string[];
   readonly isolateAgentConfig?: boolean;
   readonly requestPlugins?: readonly AgentRequestPlugin[];
+  readonly models?: readonly string[];
 }
 
 export interface AgentCliRunResult extends OriAgentRun {
@@ -203,6 +207,10 @@ export function buildAgentCliEnv(opts: AgentCliOpts): Record<string, string> {
   if (requestPlugins.length > 0) {
     env[AGENT_REQUEST_PLUGINS_ENV] = JSON.stringify(requestPlugins);
   }
+  const models = opts.models ?? [];
+  if (models.length > 0) {
+    env[AGENT_CANDIDATE_MODELS_ENV] = JSON.stringify(models);
+  }
   return env;
 }
 
@@ -248,6 +256,7 @@ export function runAgentCli(input: {
 }): Effect<AgentCliRunResult, SolverError> {
   const { session, harness, opts, instructionPath, timeoutMs } = input;
   const requestPlugins = opts.requestPlugins ?? [];
+  const models = opts.models ?? [];
   const script = harness.buildRunScript({
     instructionPath,
     logPath: harness.remoteLogPath,
@@ -257,14 +266,19 @@ export function runAgentCli(input: {
     hasAllowedTools: (opts.allowedTools ?? []).length > 0,
     hasDisallowedTools: (opts.disallowedTools ?? []).length > 0,
     isolateAgentConfig: opts.isolateAgentConfig === true,
-    hasRequestPlugins: requestPlugins.length > 0,
+    loadsRequestExtension: requestPlugins.length > 0 || models.length > 0,
   });
   return gen(function* () {
     if (requestPlugins.length > 0 && !harness.forwardsRequestPlugins) {
       return yield* new SolverError({
         message: `the ${harness.id} agent does not forward OpenRouter request plugins (${requestPlugins.map((plugin) => plugin.id).join(", ")}) from inside the sandbox`,
       });
     }
+    if (models.length > 0 && !harness.forwardsRequestPlugins) {
+      return yield* new SolverError({
+        message: `the ${harness.id} agent does not forward the OpenRouter models list from inside the sandbox`,
+      });
+    }
     if (
       opts.sessionId !== undefined &&
       opts.sessionId.length > 0 &&

src/benchmarks/deep-swe/benchmark.ts

@@ -122,6 +122,7 @@ function makeDeepSweLayer(
               isolateAgentConfig: benchmarkConfig.isolateAgentConfig,
               requestPlugins:
                 requestPlugins.length > 0 ? requestPlugins : undefined,
+              models: benchmarkConfig.models,
             }),
             endpointId: benchmarkConfig.endpointId,
             sessionId: input.sessionId,

src/benchmarks/swe-atlas/benchmark.ts

@@ -132,6 +132,7 @@ function makeSweAtlasLayer(
               isolateAgentConfig: benchmarkConfig.isolateAgentConfig,
               requestPlugins:
                 requestPlugins.length > 0 ? requestPlugins : undefined,
+              models: benchmarkConfig.models,
             }),
             endpointId: benchmarkConfig.endpointId,
             inference: {

src/benchmarks/terminal-bench/benchmark.test.ts

@@ -50,4 +50,26 @@ describe("terminal_bench layer", () => {
       "terminal_bench cannot use switchyardAlgorithm: the claude agent calls the model from inside the sandbox and does not forward the switchyard-router plugin (supported agents: pi)"
     );
   });
+
+  it("rejects candidate models for an agent that cannot forward them", async () => {
+    const exit = await buildLayer({
+      agent: "claude",
+      model: "nvidia/switchyard",
+      models: ["z-ai/glm-5.3-flash", "anthropic/claude-opus-5.5"],
+    });
+    assertFailure(exit);
+    expect(getOrThrow(failureOption(exit.cause)).message).toBe(
+      "terminal_bench cannot use candidate models: the claude agent calls the model from inside the sandbox and does not forward the models list (supported agents: pi)"
+    );
+  });
+
+  it("accepts candidate models and a switchyard algorithm for pi", async () => {
+    const exit = await buildLayer({
+      agent: "pi",
+      model: "nvidia/switchyard",
+      models: ["z-ai/glm-5.3-flash", "anthropic/claude-opus-5.5"],
+      switchyardAlgorithm: "stage",
+    });
+    expect(exit._tag).toBe("Success");
+  });
 });

src/benchmarks/terminal-bench/benchmark.ts

@@ -77,6 +77,7 @@ function makeTerminalBenchLayer(
     disallowedTools: benchmarkConfig.disallowedTools,
     isolateAgentConfig: benchmarkConfig.isolateAgentConfig,
     requestPlugins: requestPlugins.length > 0 ? requestPlugins : undefined,
+    models: benchmarkConfig.models,
   });
   const datasetLayer = makeTerminalBenchDatasetLayer(
     definedValues({