refactor(benchmarks): retire Tau3 Banking from Kepler (2/2)

src/benchmarks/benchmark-config.ts

@@ -10,13 +10,9 @@ import { ProviderSort } from "../internal/enums";
 import type { ValueOf } from "../internal/guards";
 import { z, zDefaultedText } from "../internal/zod";
 import { SWITCHYARD_MODEL } from "../providers/switchyard-router-plugin";
-import {
-  TAU3_BENCH_BANKING_META,
-  TAU_BENCH_AIRLINE_META,
-} from "./benchmark-meta";
+import { TAU_BENCH_AIRLINE_META } from "./benchmark-meta";
 import { DracoPanelConfigSchema } from "./draco/schemas";
 import { SearchLaneConfigSchema } from "./search/core/config";
-import { BankingRetrievalConfigSchema } from "./tau3-bench-banking/retrieval-config";
 
 export const GEMINI_MEDIA_RESOLUTIONS = [
   "MEDIA_RESOLUTION_UNSPECIFIED",
@@ -103,22 +99,6 @@ export const TauBenchAirlineConfigSchema = z.object({
 
 export type TauBenchAirlineConfig = z.infer<typeof TauBenchAirlineConfigSchema>;
 
-export const Tau3BenchBankingOptionsSchema = z.object({
-  userModel: zDefaultedText(TAU3_BENCH_BANKING_META.userModel),
-  userReasoningEffort: z.enum(REASONING_EFFORTS).default("medium"),
-  retrievalConfig: BankingRetrievalConfigSchema,
-});
-
-export const Tau3BenchBankingConfigSchema = z.object({
-  benchmarkId: z.literal("tau3_bench_banking"),
-  ...FixedTemperatureBenchmarkBaseSchema.shape,
-  ...Tau3BenchBankingOptionsSchema.shape,
-});
-
-export type Tau3BenchBankingConfig = z.infer<
-  typeof Tau3BenchBankingConfigSchema
->;
-
 export const MmmuProVisionOptionsSchema = z.object({
   imageDetail: z.enum(IMAGE_DETAIL_VALUES).optional(),
   mediaResolution: z.enum(GEMINI_MEDIA_RESOLUTIONS).optional(),
@@ -240,7 +220,6 @@ export const KeplerBenchmarkRunConfigSchema = z.discriminatedUnion(
     GpqaBenchmarkConfigSchema,
     MmluProBenchmarkConfigSchema,
     TauBenchAirlineConfigSchema,
-    Tau3BenchBankingConfigSchema,
     MmmuProVisionBenchmarkConfigSchema,
     DracoBenchmarkConfigSchema,
     IfStructBenchmarkConfigSchema,
@@ -270,7 +249,6 @@ export const BENCHMARK_OPTIONS_SCHEMAS = {
   gpqa_diamond: GpqaOptionsSchema,
   mmlu_pro: MmluProOptionsSchema,
   tau_bench_verified_airline: TauBenchOptionsSchema,
-  tau3_bench_banking: Tau3BenchBankingOptionsSchema,
   mmmu_pro_vision: MmmuProVisionOptionsSchema,
   ifstruct: IfStructOptionsSchema,
   toolcall_formats: ToolCallFormatsOptionsSchema,

src/benchmarks/benchmark-meta.ts

@@ -28,12 +28,6 @@ export const TAU_BENCH_AIRLINE_META = {
   userModel: "google/gemini-2.5-flash",
 } as const satisfies BenchmarkMeta;
 
-export const TAU3_BENCH_BANKING_META = {
-  id: "tau3_bench_banking",
-  defaultEpochs: 5,
-  userModel: "openai/gpt-5.4-mini",
-} as const satisfies BenchmarkMeta;
-
 export const DRACO_META = {
   id: "draco",
   defaultEpochs: 1,
@@ -80,7 +74,6 @@ const BENCHMARK_META: Readonly<Record<string, BenchmarkMeta>> = {
   [MMLU_PRO_META.id]: MMLU_PRO_META,
   [MMMU_PRO_VISION_META.id]: MMMU_PRO_VISION_META,
   [TAU_BENCH_AIRLINE_META.id]: TAU_BENCH_AIRLINE_META,
-  [TAU3_BENCH_BANKING_META.id]: TAU3_BENCH_BANKING_META,
   [DRACO_META.id]: DRACO_META,
   [IFSTRUCT_META.id]: IFSTRUCT_META,
   [TOOLCALL_FORMATS_META.id]: TOOLCALL_FORMATS_META,

src/benchmarks/native-eval-migration.test.ts

@@ -0,0 +1,44 @@
+import { describe, expect, it } from "bun:test";
+
+import { assertLeft, assertRight } from "../internal/testing";
+import { parseSchema } from "../internal/zod";
+import {
+  BENCHMARK_OPTIONS_SCHEMAS,
+  KeplerBenchmarkRunConfigSchema,
+} from "./benchmark-config";
+import { getBenchmarkMeta } from "./benchmark-meta";
+import { getBenchmark } from "./registry";
+
+describe("native eval migration", () => {
+  it("does not expose Tau3 Banking through Kepler", () => {
+    const benchmarkId = "tau3_bench_banking";
+    expect(getBenchmark(benchmarkId)).toBeUndefined();
+    expect(getBenchmarkMeta(benchmarkId)).toBeUndefined();
+    expect(Object.keys(BENCHMARK_OPTIONS_SCHEMAS)).not.toContain(benchmarkId);
+    assertLeft(
+      parseSchema(KeplerBenchmarkRunConfigSchema, {
+        benchmarkId,
+        model: "openai/gpt-5",
+        reasoningEffort: "high",
+      })
+    );
+  });
+
+  it.each([
+    "gpqa_diamond",
+    "mmlu_pro",
+    "mmmu_pro_vision",
+    "tau_bench_verified_airline",
+    "vgi_bench",
+  ])("retains %s", (benchmarkId) => {
+    expect(getBenchmark(benchmarkId)).toBeDefined();
+    expect(getBenchmarkMeta(benchmarkId)).toBeDefined();
+    assertRight(
+      parseSchema(KeplerBenchmarkRunConfigSchema, {
+        benchmarkId,
+        model: "openai/gpt-5",
+        reasoningEffort: "high",
+      })
+    );
+  });
+});

src/benchmarks/registry.ts

@@ -8,7 +8,6 @@ import { DSQA_BENCHMARK } from "./search/dsqa/benchmark";
 import { HLE_BENCHMARK } from "./search/hle/benchmark";
 import { WIDESEARCH_BENCHMARK } from "./search/widesearch/benchmark";
 import { TAU_BENCH_AIRLINE_BENCHMARK } from "./tau-bench-airline/benchmark";
-import { TAU3_BENCH_BANKING_BENCHMARK } from "./tau3-bench-banking/benchmark";
 import { TOOLCALL_FORMATS_BENCHMARK } from "./toolcall-formats/benchmark";
 import type { Benchmark } from "./types";
 import { VGI_BENCH_BENCHMARK } from "./vgi-bench/benchmark";
@@ -17,7 +16,6 @@ const BENCHMARKS: Record<string, Benchmark> = {
   [GPQA_BENCHMARK.id]: GPQA_BENCHMARK,
   [MMLU_PRO_BENCHMARK.id]: MMLU_PRO_BENCHMARK,
   [TAU_BENCH_AIRLINE_BENCHMARK.id]: TAU_BENCH_AIRLINE_BENCHMARK,
-  [TAU3_BENCH_BANKING_BENCHMARK.id]: TAU3_BENCH_BANKING_BENCHMARK,
   [MMMU_PRO_VISION_BENCHMARK.id]: MMMU_PRO_VISION_BENCHMARK,
   [DRACO_BENCHMARK.id]: DRACO_BENCHMARK,
   [IFSTRUCT_BENCHMARK.id]: IFSTRUCT_BENCHMARK,