From 3d6e33d154951622e5969e95633fcff737a32fd0 Mon Sep 17 00:00:00 2001 From: Lin Jia Date: Wed, 19 Aug 2026 15:35:52 -0700 Subject: [PATCH] docs(benchmark): add v0.2.0 TB-2.1 escalation routing profile Signed-off-by: Lin Jia --- .../tb21-escalation-opus-glm-deepseek.toml | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 benchmark/routing-profiles/tb21-escalation-opus-glm-deepseek.toml diff --git a/benchmark/routing-profiles/tb21-escalation-opus-glm-deepseek.toml b/benchmark/routing-profiles/tb21-escalation-opus-glm-deepseek.toml new file mode 100644 index 000000000..c2c13a0dd --- /dev/null +++ b/benchmark/routing-profiles/tb21-escalation-opus-glm-deepseek.toml @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Escalation-router deployment behind the v0.2.0 Terminal-Bench 2.1 efficiency results +# (https://developer.nvidia.com/blog/route-ai-agent-workloads-across-models-with-nvidia-nemo-switchyard/). +# +# This is the configuration as benchmarked, with one change: the published runs were +# served through NVIDIA-internal inference endpoints, replaced here with OpenRouter +# equivalents so the deployment is publicly runnable. Routing-algorithm parameters +# are exactly as run; absolute solve rates may shift slightly across serving stacks. +# As run, the strong tier used Anthropic adaptive thinking with output effort `high`, +# the weak tier ran with reasoning effort `high`, and the judge ran with thinking +# disabled — mirrored below with OpenRouter's `reasoning` controls. +# +# The `escalation` block is part of the v0.2.0 server config schema only; it is not +# accepted by the current `switchyard-server`. Run from the tag: +# git checkout v0.2.0-rc.1 +# switchyard-server --config benchmark/routing-profiles/tb21-escalation-opus-glm-deepseek.toml --port 4000 + +schema_version = 1 + +[llm_clients.openrouter] +format = "openai_chat" +base_url = "https://openrouter.ai/api/v1" +api_key_env = "OPENROUTER_API_KEY" + +[targets.strong] +id = "anthropic/claude-opus-4.8" +llm_client = "openrouter" + +[targets.strong.extra_body.reasoning] +effort = "high" + +[targets.weak] +id = "z-ai/glm-5.2" +llm_client = "openrouter" + +[targets.weak.extra_body.reasoning] +effort = "high" + +[targets.judge] +id = "deepseek/deepseek-v4-flash" +llm_client = "openrouter" + +[targets.judge.extra_body.reasoning] +enabled = false + +# Every conversation starts on the weak tier. The judge reviews the trajectory each +# turn, and two consecutive escalate verdicts latch the session onto the strong tier. +[routes.switchyard] +id = "switchyard" +type = "llm_classifier" +classifier_target = "judge" +strong_target = "strong" +weak_target = "weak" +base_threshold = 0.5 +escalation = { confirmations = 2, recent_turn_window = 28, window_message_chars = 500 }