From 16516fe5bf8d91c0c0764bc62dd9a9953c3ced6c Mon Sep 17 00:00:00 2001 From: derek <33986799+dereklarmstrong@users.noreply.github.com> Date: Wed, 19 Aug 2026 13:14:40 -0400 Subject: [PATCH] docs: correct metrics reference, examples, and stale references - metrics_reference: remove phantom tier label (tier lives in the JSONL routing log), seed 404 among canonical outcome codes, document the previously undocumented run/decision/call counters, stage-router, and advisor-gate metric families, update the cardinality table - getting_started: mark X-Switchyard-Version and the telemetry opt-out as not yet implemented, cross-reference known_issues - DEVELOPMENT: --extra server -> cli, replace missing secrets/ template with the actual env-var mechanism - examples/libsy.py: replace non-existent Step.Decision / call.decision with the real binding API - release_workflow: fix crate publish order per publish.yml - toml_schema: add the advisor route section (13 knobs) and correct the duplicate (client, id) behavior to a warning - escalation_router_routing: drop cost from the snapshot description and reword judge attribution to the JSONL-log behavior - core_concepts: add noop and advisor rows to the algorithm table - known_issues: mark the x-switchyard-session-id item as fixed --- DEVELOPMENT.md | 9 +-- docs/core_concepts.md | 6 +- docs/getting_started.md | 12 ++-- docs/internal/metrics_reference.md | 68 +++++++++++++++++-- docs/internal/release_workflow.md | 4 +- docs/known_issues.md | 2 +- docs/reference/toml_schema.md | 23 +++++++ .../escalation_router_routing.md | 17 ++--- examples/libsy.py | 28 ++++---- 9 files changed, 127 insertions(+), 42 deletions(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index e9fae8ba5..d54605139 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -34,8 +34,8 @@ uv sync --extra cli # add prompt-toolkit uv sync --all-extras # everything (dev group is still included by default) ``` -(`server` is already pulled in by the dev group, so you don't need -`--extra server` for local development.) +(There is no `server` extra: the native server is compiled into the wheel by +maturin through `switchyard-py`.) ## Project Structure @@ -97,8 +97,9 @@ export OPENAI_API_KEY="sk-..." uv run pytest tests/your_e2e_test.py -v -x ``` -`secrets/secrets.template.json` shows the structure expected by -`secrets/secrets.json` if you prefer a credential file over env vars. +The server reads API keys from the environment variables named by each +client's `api_key_env` key; it does not load a credential file itself. +`examples/secrets.template.json` is a sample of what those values look like. ## Human-AI Development Convention diff --git a/docs/core_concepts.md b/docs/core_concepts.md index 79e29e2b2..a5d67fcd1 100644 --- a/docs/core_concepts.md +++ b/docs/core_concepts.md @@ -74,15 +74,17 @@ and reasoning. ## Routing Algorithms An algorithm receives the normalized request, publishes a routing decision, -and serves the selected target. The standalone server supports these primary -route types: +and serves the selected target. The standalone server supports these route +types: | Route type | Behavior | |---|---| +| `noop` | Returns a canned `OK` response without calling an upstream. | | `passthrough` | Sends every request to one target. | | `random` | Selects among targets using optional relative weights. | | `llm_classifier` | Uses a classifier target to choose between weak and strong targets. | | `stage_router` | Uses tool-result and progress signals to choose an efficient or capable target. | +| `advisor` | Serves turns through an executor while an advisor judge gates terminal turns for rework. | Strong, weak, capable, and efficient are roles within an algorithm, not fixed properties of a model. The same upstream model can serve different roles in diff --git a/docs/getting_started.md b/docs/getting_started.md index 58b59efe6..59b8458f1 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -225,15 +225,19 @@ references, and route construction without starting the server. Check health: `curl http://localhost:4000/health` -**Telemetry header opt-out** +**Telemetry opt-out** -Switchyard adds an `X-Switchyard-Version` header to outbound LLM calls for -release attribution. No request or response content is included. To disable: +The server exports traces and metrics with OpenTelemetry. Remote OTLP export +is off by default and only activates when you set an OTLP endpoint such as +`OTEL_EXPORTER_OTLP_ENDPOINT`. To disable telemetry entirely: ```bash -export SWITCHYARD_TELEMETRY_OPT_OUT=1 +export OTEL_SDK_DISABLED=1 ``` +Or disable a single signal: `OTEL_TRACES_EXPORTER=none` or +`OTEL_METRICS_EXPORTER=none`. + --- ## Library Path diff --git a/docs/internal/metrics_reference.md b/docs/internal/metrics_reference.md index 0d29bb59f..3b2882f11 100644 --- a/docs/internal/metrics_reference.md +++ b/docs/internal/metrics_reference.md @@ -25,13 +25,35 @@ A JSON summary of the same traffic lives at `GET /v1/stats`. | `switchyard_total_requests` | gauge | Successful and failed routed model calls since process start. Classifier and judge calls are excluded; a context-window fallback can add another routed call. | | `switchyard_total_errors` | gauge | Failed routed model calls since process start. | +## Run, decision, and call counters + +Instrument names in the code use the OTel dotted form (`switchyard.runs`, +`switchyard.stage_router.score`, ...); the Prometheus exporter sanitizes the +dots to underscores and appends `_total` to counters, same as the other +families in this document. + +| Metric | Type | Meaning | +|---|---|---| +| `switchyard_build_info{version}` | gauge | `1` at process start, with the server version as the `version` label. | +| `switchyard_runs_total{algorithm,outcome}` | counter | One per completed run of a routing algorithm; `outcome` is `ok` or `error`. | +| `switchyard_run_duration_ms{algorithm,outcome}` | histogram | Wall-clock duration of one routing-algorithm run, in milliseconds. | +| `switchyard_decisions_total{algorithm,selected_model}` | counter | One per run that resolves a target model. | +| `switchyard_llm_calls_total{algorithm,selected_model,outcome}` | counter | One per model call the router makes — algorithm-layer calls (classifier, judge, advisor) plus the terminal routed answer call. | +| `switchyard_llm_call_duration_ms{algorithm,selected_model,outcome}` | histogram | Wall-clock duration of one of those model calls, in milliseconds. | + +`algorithm` is the routing algorithm's name (e.g. `stage_router`, +`llm_classifier`). `selected_model` is the model ID the call targeted — always +a real model, never `none`. + ## Per-endpoint counters The `model` label is the configured endpoint id (`openai/gpt-5.5`, `azure_openai/gpt-5.5`, etc.). -The `tier` label is optional. It is present when an algorithm defines a stable -tier for the selected model, such as `strong` or `weak` for a two-tier classifier. +The `tier` label is not exported on any of these families. The routing tier +(`strong`/`weak`) is a per-request routing decision and is recorded, when the +routing log is enabled, in the server's JSONL routing log (`tier` field) — not as +a Prometheus label. | Metric | Type | Meaning | |---|---|---| @@ -69,6 +91,36 @@ Each histogram emits `_bucket`, `_sum`, and `_count` series. Use `upstream_5xx`, `upstream_non_5xx`, `invalid_response`, `parse_error`, `client_error`, or `call_error`. The labels never include request or response text. +## Stage router metrics + +Present when the routing algorithm is `stage_router`: one decision counter and +six distributions per turn. + +| Metric | Type | Meaning | +|---|---|---| +| `switchyard_stage_router_routing_decisions_total{decision_source,target_name}` | counter | One per turn's final routing choice. `decision_source` is one of `override`, `tests_passed`, `dimensions`, `ambiguous`, `llm-classifier`, or `fall_open`; `target_name` is the model name the turn routed to (one of the router's two tier endpoints). | +| `switchyard_stage_router_score` | histogram | The stage scorer's signed routing score (positive favors the capable side, negative the efficient side). | +| `switchyard_stage_router_confidence` | histogram | The decision confidence used to resolve or defer the turn. | +| `switchyard_stage_router_severity` | histogram | Detected tool-failure severity for the turn. | +| `switchyard_stage_router_spinning` | histogram | Repeated unproductive tool activity for the turn. | +| `switchyard_stage_router_exploring` | histogram | Exploratory tool activity for the turn. | +| `switchyard_stage_router_production_intensity` | histogram | Production-oriented tool activity for the turn. | + +The six histograms carry no labels. The score uses buckets from `-1` to `1` +in 0.25 steps; the other five use `[0, 0.1, 0.25, 0.5, 0.75, 0.9, 1]`. + +## Advisor gate metrics + +Present when the routing algorithm is `advisor_gate` (opt-in); otherwise +absent from the scrape. + +| Metric | Type | Meaning | +|---|---|---| +| `switchyard_advisor_gate_reviews_total{verdict,trigger}` | counter | One per advisor-model consultation. `verdict` is `approve`, `redo`, or `unparseable`; `trigger` is `pattern`, `no_tool_call`, or `stall`. | +| `switchyard_advisor_gate_consult_failures_total{reason}` | counter | One per advisor call that failed before a verdict; `reason` from the same bounded set as the classifier fail-open reasons, minus `parse_error`. | +| `switchyard_advisor_gate_discarded_turns_total` | counter | One per executor turn discarded on a `redo` verdict. | +| `switchyard_advisor_gate_discarded_tokens_total{kind}` | counter | Tokens consumed by that discarded turn; `kind` is `input`, `cached`, `cache_creation`, or `output`. | + ## Outcome counters for error-rate ratios The `outcome` label takes exactly three values: @@ -97,7 +149,7 @@ The `outcome` label takes exactly three values: `outcome` is fully determined by `code`, so adding the label does not multiply series. You get one series per distinct code either way. The -canonical codes (`200`, `429`, `500`, `504`, `none`) are seeded at `0` so +canonical codes (`200`, `404`, `429`, `500`, `504`, `none`) are seeded at `0` so their time series exist from process start (a `rate()` over a never-seen counter reads as "no data", not zero). @@ -149,10 +201,14 @@ into label space. | `outcome` | Exactly 3: `success`, `retryable_error`, `other_error`. | Outcome counters | | `code` | Bounded: the known-code allowlist (`200`, `400`, `401`, `403`, `404`, `408`, `409`, `422`, `429`, `500`, `502`, `503`, `504`), plus `none` and the per-class buckets `1xx`/`2xx`/`3xx`/`4xx`/`5xx`/`other`. About 20 values max. | `switchyard_upstream_attempts_total` | | `le` | The configured histogram bucket boundaries. | Histogram buckets | -| `algorithm` | One stable value per configured algorithm. | Routing-overhead histogram | -| `tier` | Small enumerated set, optional. | Per-endpoint counters and histograms on algorithms that supply it | +| `algorithm` | One stable value per configured algorithm. | Routing-overhead histogram, run and call counters | +| `decision_source` | Exactly 6: `override`, `tests_passed`, `dimensions`, `ambiguous`, `llm-classifier`, `fall_open`. | Stage-router decision counter | +| `target_name` | One per configured endpoint. | Stage-router decision counter | +| `selected_model` | One per configured endpoint (always a real model ID). | Decision and call counters | +| `reason` | Bounded error categories: `timeout`, `transport`, `upstream_5xx`, `upstream_non_5xx`, `invalid_response`, `parse_error`, `client_error`, `call_error` (consult failures use that set minus `parse_error`). | Classifier fail-open counter, advisor-gate consult-failure counter | +| `kind` | Exactly 4: `input`, `cached`, `cache_creation`, `output`. | Advisor-gate discarded-token counter | | `judge_model` | One per configured judge target. | Classifier fail-open counter | -| `reason` | Exactly 8 fixed error categories. | Classifier fail-open counter | +| `version` | One value: the server version. | `switchyard_build_info` | ## Triage cheatsheet diff --git a/docs/internal/release_workflow.md b/docs/internal/release_workflow.md index aebf12ee4..49bd8b3fa 100644 --- a/docs/internal/release_workflow.md +++ b/docs/internal/release_workflow.md @@ -90,8 +90,8 @@ ready. The same tag publishes these crates to crates.io in dependency order: 1. `switchyard-protocol` -2. `switchyard-libsy` -3. `switchyard-translation` +2. `switchyard-translation` +3. `switchyard-libsy` 4. `switchyard-llm-client` 5. `switchyard-server` diff --git a/docs/known_issues.md b/docs/known_issues.md index d1ed129f9..169d04683 100644 --- a/docs/known_issues.md +++ b/docs/known_issues.md @@ -4,7 +4,7 @@ 1. Buffered upstream work continues after the client disconnects, so a cancelled request can still incur provider cost. 2. Routing-tier attribution is missing from `GET /v1/stats` and `/metrics` for LLM-classifier judge failures that route to the default target, escalation decisions, and `stage_router` fallback decisions. 3. The retry recovery counter stays at zero after a successful upstream retry. -4. `x-switchyard-session-id` is not recorded in native session stats. +4. (Fixed) `x-switchyard-session-id` is now recorded in native session stats, with fallback correlation from harness session-id headers (`x-session-id`, Codex `session_id` path, generic `session-id`). 5. The native server does not send the documented `X-Switchyard-Version` header upstream. ## 0.1.0 diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index 68d01bc54..615d98412 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -208,6 +208,29 @@ optional `handoff_notes` and `classifier` tables and for tuning. | `efficient_system_prompt` | No | unset | System prompt handed to the efficient tier. | | `classifier.response_format_type` | No | `json_schema` | Structured-output mode for the optional classifier judge. Use `json_object` when the classifier provider does not support JSON Schema; Switchyard adds the schema to the prompt and validates the verdict locally. | +### `advisor` + +The executor serves every turn; an advisor judge can send a terminal turn +back for rework. See +[Advisor-Gate Routing](../routing_algorithms/advisor_gate_routing.md) for gate +behavior and tuning. + +| Key | Required | Default | Meaning | +|---|:---:|---|---| +| `executor_target` | Yes | — | Target that serves every turn and answers `count_tokens` calls. | +| `advisor_target` | Yes | — | Judge-only target. Not a routing destination. | +| `reviewer_system_prompt` | No | packaged prompt | System prompt for the advisor's review call (the APPROVE/REDO contract). | +| `redo_feedback_prefix` | No | packaged prefix | Text prepended to the advisor's REDO plan when it is fed back to the executor. | +| `gate_trigger` | No | `no_tool_call` | `no_tool_call` reviews the first turn without tool calls; `pattern` reviews the first turn whose visible text matches `gate_trigger_pattern`. | +| `gate_trigger_pattern` | No | unset | Regex (searched, not anchored) for the `pattern` trigger. Required when `gate_trigger = "pattern"`; rejected while `no_tool_call` is in effect. | +| `max_reviews` | No | `1` | Reviews allowed per budget scope. Must be at least `1`. | +| `gate_stall_turns` | No | `0` | When above `0`, also review (once per conversation) the first request already carrying at least this many assistant turns. `0` disables it. | +| `gate_min_tool_results` | No | `0` | For the `no_tool_call` trigger: only review once the conversation carries at least this many tool results. `0` reviews from the first terminal turn. | +| `advisor_max_tokens` | No | `2048` | Cap on the advisor's output per consult. | +| `advisor_temperature` | No | unset | Sampling temperature for the consult. Unset omits it from the wire request. | +| `transcript_max_chars` | No | `200000` | Cap on the serialized transcript handed to the advisor; the middle of an over-cap conversation is dropped. | +| `fail_open` | No | `true` | `true` degrades a failed consult to APPROVE; `false` propagates it as the turn's error. | + ## Validation Errors `--dry-run` prefixes configuration failures with diff --git a/docs/routing_algorithms/escalation_router_routing.md b/docs/routing_algorithms/escalation_router_routing.md index 1ed9e8085..130a6ce6d 100644 --- a/docs/routing_algorithms/escalation_router_routing.md +++ b/docs/routing_algorithms/escalation_router_routing.md @@ -150,14 +150,15 @@ Read the standard stats endpoint: curl -s http://localhost:4000/v1/stats ``` -The snapshot reports per-model calls, tokens, latency, and cost for the strong -and weak tiers. Judge calls are recorded in the classifier stats bucket, so their -token cost and latency remain visible as routing overhead. - -When the server runs with a routing log, successful judge calls also appear in -per-session routing stats under the judge's model id, tagged with the -`classifier` tier — so per-session token accounting includes judge overhead -alongside the tiers the session was served by. +The snapshot reports calls, tokens, and latency per model, so the strong and +weak tiers show up as their own target-model entries. Judge calls are recorded in +the classifier stats bucket, so their token usage and latency remain visible as +routing overhead. + +When the server runs with a routing log, each successful judge call is written +to the log under the judge's model id, labeled with the `classifier` tier — so +per-session token accounting counts judge overhead in the judge model's entry, +alongside the models that actually served the session. ## When not to use escalation routing diff --git a/examples/libsy.py b/examples/libsy.py index 3a1536331..ef6c00c68 100644 --- a/examples/libsy.py +++ b/examples/libsy.py @@ -5,23 +5,22 @@ """Drive a libsy algorithm stream from Python.""" import asyncio -from collections.abc import Mapping from switchyard.libsy import Step, algorithms class EchoClient: - """Return a fixed completion for any selected target.""" + """Return a fixed completion for the model on the request.""" - async def call( - self, - request: Mapping[str, object], - model: str, - ) -> Mapping[str, object]: + async def call(self, request): return { - "model": model, + "model": request["model"], "outputs": [ - {"role": "assistant", "content": [{"type": "text", "text": "Hello"}]} + { + "role": "assistant", + "content": [{"type": "text", "text": "Hello"}], + "stop_reason": "end_turn", + } ], } @@ -41,12 +40,11 @@ async def main() -> None: async for step in algorithm.run_stream(request): match step: - case Step.Decision(decision): - print("Decision:", decision.selected_model_id, decision.reasoning) - case Step.CallModel(call): - model = call.decision.selected_model_id - call.respond(await client.call(call.request, model)) - case Step.Done(response): + case Step.Done(outcome): + response = outcome.response + if response is None: + response = await client.call(outcome.request) + print("Selected model:", outcome.selected_model_id) print("Response:", response)