diff --git a/README.md b/README.md index 2cd0020..0fe1fc2 100644 --- a/README.md +++ b/README.md @@ -136,6 +136,18 @@ models: - id: large-prompt target_model: qwen/qwen3.6-35b-a3b min_prompt_chars: 12000 + preclassifier: + min_confidence: 0.8 + targets: + - target_model: qwen/qwen3.6-35b-a3b + confidence: 0.95 + keywords: + - architecture + - migration + - production + - root cause + - security + - terraform classifier: backend: lmstudio model: qwen3-coder-30b-a3b-instruct diff --git a/configs/router.example.yaml b/configs/router.example.yaml index 5744f61..cf58f90 100644 --- a/configs/router.example.yaml +++ b/configs/router.example.yaml @@ -49,6 +49,18 @@ models: - id: large-prompt target_model: qwen/qwen3.6-35b-a3b min_prompt_chars: 12000 + preclassifier: + min_confidence: 0.8 + targets: + - target_model: qwen/qwen3.6-35b-a3b + confidence: 0.95 + keywords: + - architecture + - migration + - production + - root cause + - security + - terraform classifier: backend: lmstudio model: qwen3-coder-30b-a3b-instruct diff --git a/docs/benchmarking.md b/docs/benchmarking.md index ee7107a..81b23d6 100644 --- a/docs/benchmarking.md +++ b/docs/benchmarking.md @@ -103,3 +103,38 @@ go run ./cmd/devrail-router bench \ The deep alias may spend early tokens on reasoning before emitting normal content, so use a larger token cap than a smoke test. Compare both timings and the `content_sample` field before making a slower backend automatic. + +## Routing Classifier Benchmarks + +Use the route-classifier benchmark to compare fast-vs-strong selection policies +without sending full generation requests through the router: + +```sh +python3 tools/route_classifier_bench.py \ + --cases test/bench/router-routing.cases.json \ + --candidates guardrails,keywords +``` + +Add `openai` to compare a live OpenAI-compatible classifier model: + +```sh +python3 tools/route_classifier_bench.py \ + --candidates guardrails,keywords,openai \ + --openai-base-url http://llm-srv-01.mfsoho.linkridge.net:18080/v1 \ + --openai-model local-coder-fast +``` + +The tool writes one JSON object per candidate/case to stdout and prints per +candidate accuracy/timing summaries to stderr. + +Router selection order is: + +1. explicit routing rules, such as prompt-size and output-token guardrails +2. optional keyword preclassifier for high-confidence cheap decisions +3. optional OpenAI-compatible LLM classifier for ambiguous requests +4. the model alias default target + +Use the benchmark corpus to tune the preclassifier keyword list before enabling +it in deployed config. Negated phrases such as `no production` and +`without security impact` should remain fall-through cases so they can reach the +LLM classifier or default route. diff --git a/internal/config/config.go b/internal/config/config.go index 1ba0a80..248b003 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -52,8 +52,9 @@ type EnsureConfig struct { } type RoutingConfig struct { - Rules []RoutingRuleConfig `yaml:"rules"` - Classifier RoutingClassifierConfig `yaml:"classifier"` + Rules []RoutingRuleConfig `yaml:"rules"` + Preclassifier RoutingPreclassifierConfig `yaml:"preclassifier"` + Classifier RoutingClassifierConfig `yaml:"classifier"` } type RoutingRuleConfig struct { @@ -75,6 +76,18 @@ type RoutingClassifierConfig struct { MaxTokens int `yaml:"max_tokens"` } +type RoutingPreclassifierConfig struct { + Targets []RoutingPreclassifierTargetConfig `yaml:"targets"` + MinConfidence float64 `yaml:"min_confidence"` + NegationPhrases []string `yaml:"negation_phrases"` +} + +type RoutingPreclassifierTargetConfig struct { + TargetModel string `yaml:"target_model"` + Keywords []string `yaml:"keywords"` + Confidence float64 `yaml:"confidence"` +} + type CommandArgs []string func (args *CommandArgs) UnmarshalYAML(value *yaml.Node) error { @@ -236,10 +249,51 @@ func (routing RoutingConfig) Validate() error { if err := routing.Classifier.Validate(); err != nil { return fmt.Errorf("classifier is invalid: %w", err) } + if err := routing.Preclassifier.Validate(); err != nil { + return fmt.Errorf("preclassifier is invalid: %w", err) + } return nil } +func (preclassifier RoutingPreclassifierConfig) Validate() error { + if !preclassifier.Enabled() { + return nil + } + if len(preclassifier.Targets) == 0 { + return errors.New("targets is required") + } + if preclassifier.MinConfidence < 0 || preclassifier.MinConfidence > 1 { + return errors.New("min_confidence must be between 0 and 1") + } + for phraseIndex, phrase := range preclassifier.NegationPhrases { + if strings.TrimSpace(phrase) == "" { + return fmt.Errorf("negation_phrases[%d] must not be empty", phraseIndex) + } + } + for index, target := range preclassifier.Targets { + if strings.TrimSpace(target.TargetModel) == "" { + return fmt.Errorf("targets[%d].target_model is required", index) + } + if len(target.Keywords) == 0 { + return fmt.Errorf("targets[%d].keywords is required", index) + } + if target.Confidence <= 0 || target.Confidence > 1 { + return fmt.Errorf("targets[%d].confidence must be greater than 0 and less than or equal to 1", index) + } + for keywordIndex, keyword := range target.Keywords { + if strings.TrimSpace(keyword) == "" { + return fmt.Errorf("targets[%d].keywords[%d] must not be empty", index, keywordIndex) + } + } + } + return nil +} + +func (preclassifier RoutingPreclassifierConfig) Enabled() bool { + return len(preclassifier.Targets) > 0 || preclassifier.MinConfidence != 0 || len(preclassifier.NegationPhrases) > 0 +} + func (classifier RoutingClassifierConfig) Validate() error { if classifier.Backend == "" && classifier.Model == "" && len(classifier.TargetModels) == 0 && classifier.SystemPrompt == "" && classifier.Timeout == "" && classifier.MaxTokens == 0 { return nil diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 4f66ed2..c59a783 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -127,6 +127,70 @@ func TestValidateRoutingRuleRequiresCondition(t *testing.T) { } } +func TestValidateRoutingPreclassifier(t *testing.T) { + t.Parallel() + + cfg := Config{ + Models: []ModelConfig{{ + ID: "local-coder", + Backend: "lmstudio", + TargetModel: "fast-model", + Routing: RoutingConfig{ + Preclassifier: RoutingPreclassifierConfig{ + MinConfidence: 0.8, + Targets: []RoutingPreclassifierTargetConfig{{ + TargetModel: "deep-model", + Keywords: []string{"production", "terraform"}, + Confidence: 0.95, + }}, + }, + }, + }}, + Backends: []BackendConfig{{ + ID: "lmstudio", + BaseURL: "http://127.0.0.1:1234/v1", + }}, + } + + if err := cfg.Validate(); err != nil { + t.Fatalf("validate config: %v", err) + } +} + +func TestValidateRoutingPreclassifierRejectsInvalidConfidence(t *testing.T) { + t.Parallel() + + cfg := Config{ + Models: []ModelConfig{{ + ID: "local-coder", + Backend: "lmstudio", + TargetModel: "fast-model", + Routing: RoutingConfig{ + Preclassifier: RoutingPreclassifierConfig{ + MinConfidence: 1.2, + Targets: []RoutingPreclassifierTargetConfig{{ + TargetModel: "deep-model", + Keywords: []string{"production"}, + Confidence: 0.95, + }}, + }, + }, + }}, + Backends: []BackendConfig{{ + ID: "lmstudio", + BaseURL: "http://127.0.0.1:1234/v1", + }}, + } + + err := cfg.Validate() + if err == nil { + t.Fatal("expected validation error") + } + if !strings.Contains(err.Error(), "preclassifier") { + t.Fatalf("expected preclassifier error, got: %v", err) + } +} + func TestLoadEnsureCommandString(t *testing.T) { t.Parallel() diff --git a/internal/server/server.go b/internal/server/server.go index 4c46db1..f38f641 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -922,6 +922,14 @@ func (s *Server) selectTargetModel(ctx context.Context, body []byte, model confi } } + if model.Routing.Preclassifier.Enabled() { + target, ok := preclassifyTargetModel(model.Routing.Preclassifier, features) + if ok { + slog.Info("routing preclassifier selected target", "request_id", requestID, "alias", model.ID, "target_model", target) + return target, "preclassifier" + } + } + if model.Routing.Classifier.Enabled() { target, ok := s.classifyTargetModel(ctx, model, features, requestID) if ok { @@ -932,6 +940,63 @@ func (s *Server) selectTargetModel(ctx context.Context, body []byte, model confi return model.TargetModel, "default" } +func preclassifyTargetModel(preclassifier config.RoutingPreclassifierConfig, features routingFeatures) (string, bool) { + minConfidence := preclassifier.MinConfidence + if minConfidence == 0 { + minConfidence = 0.75 + } + + bestTarget := "" + bestConfidence := 0.0 + for _, target := range preclassifier.Targets { + if !preclassifierTargetMatches(target, features, preclassifier.NegationPhrases) { + continue + } + if target.Confidence > bestConfidence { + bestTarget = target.TargetModel + bestConfidence = target.Confidence + } + } + if bestTarget == "" || bestConfidence < minConfidence { + return "", false + } + return bestTarget, true +} + +func preclassifierTargetMatches(target config.RoutingPreclassifierTargetConfig, features routingFeatures, negationPhrases []string) bool { + for _, keyword := range target.Keywords { + normalized := strings.ToLower(strings.TrimSpace(keyword)) + if normalized == "" || !strings.Contains(features.Text, normalized) { + continue + } + if preclassifierKeywordNegated(features.Text, normalized, negationPhrases) { + continue + } + return true + } + return false +} + +func preclassifierKeywordNegated(text string, keyword string, configuredPhrases []string) bool { + phrases := configuredPhrases + if len(phrases) == 0 { + phrases = []string{"no ", "not ", "without ", "non-"} + } + for _, phrase := range phrases { + normalized := strings.ToLower(strings.TrimSpace(phrase)) + if normalized == "" { + continue + } + if strings.Contains(text, normalized+keyword) { + return true + } + if !strings.HasSuffix(normalized, "-") && strings.Contains(text, normalized+" "+keyword) { + return true + } + } + return false +} + func (s *Server) classifyTargetModel(ctx context.Context, model config.ModelConfig, features routingFeatures, requestID string) (string, bool) { classifier := model.Routing.Classifier backend, ok := s.cfg.Backend(classifier.Backend) diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 9316ad1..7217a01 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -369,6 +369,143 @@ func TestRoutingClassifierFallsBackToDefaultTarget(t *testing.T) { } } +func TestRoutingPreclassifierSelectsTargetBeforeClassifier(t *testing.T) { + t.Parallel() + + var classifierCalls int + classifier := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + classifierCalls++ + writeJSON(w, http.StatusOK, map[string]any{ + "choices": []map[string]any{{ + "message": map[string]string{ + "role": "assistant", + "content": `{"target_model":"fast-model"}`, + }, + }}, + }) + })) + t.Cleanup(classifier.Close) + + var backendModel string + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var payload struct { + Model string `json:"model"` + } + if err := json.NewDecoder(r.Body).Decode(&payload); err != nil { + t.Errorf("decode backend request: %v", err) + http.Error(w, "bad request", http.StatusBadRequest) + return + } + backendModel = payload.Model + writeJSON(w, http.StatusOK, map[string]string{"model": payload.Model}) + })) + t.Cleanup(backend.Close) + + srv, err := New(config.Config{ + Server: config.ServerConfig{Address: "127.0.0.1:0"}, + Models: []config.ModelConfig{{ + ID: "local-coder-auto", + Backend: "lmstudio", + TargetModel: "fast-model", + Routing: config.RoutingConfig{ + Preclassifier: config.RoutingPreclassifierConfig{ + MinConfidence: 0.8, + Targets: []config.RoutingPreclassifierTargetConfig{{ + TargetModel: "deep-model", + Keywords: []string{"production", "incident"}, + Confidence: 0.95, + }}, + }, + Classifier: config.RoutingClassifierConfig{ + Backend: "classifier", + Model: "classifier-model", + TargetModels: []string{"fast-model", "deep-model"}, + }, + }, + }}, + Backends: []config.BackendConfig{ + {ID: "lmstudio", BaseURL: backend.URL}, + {ID: "classifier", BaseURL: classifier.URL}, + }, + }) + if err != nil { + t.Fatalf("create server: %v", err) + } + + req := httptest.NewRequest( + http.MethodPost, + "/v1/chat/completions", + strings.NewReader(`{"model":"local-coder-auto","messages":[{"role":"user","content":"Investigate a production incident."}]}`), + ) + rec := httptest.NewRecorder() + srv.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("unexpected status: %d", rec.Code) + } + if classifierCalls != 0 { + t.Fatalf("expected preclassifier to bypass classifier, got %d classifier calls", classifierCalls) + } + if backendModel != "deep-model" { + t.Fatalf("unexpected backend model: %q", backendModel) + } +} + +func TestRoutingPreclassifierFallsThroughForNegatedKeyword(t *testing.T) { + t.Parallel() + + features := routingFeatures{Text: "update frontend copy. no production services are involved."} + target, ok := preclassifyTargetModel(config.RoutingPreclassifierConfig{ + MinConfidence: 0.8, + Targets: []config.RoutingPreclassifierTargetConfig{{ + TargetModel: "deep-model", + Keywords: []string{"production"}, + Confidence: 0.95, + }}, + }, features) + + if ok { + t.Fatalf("expected negated keyword to fall through, got target %q", target) + } +} + +func TestRoutingPreclassifierFallsThroughForCustomNegationPhrase(t *testing.T) { + t.Parallel() + + features := routingFeatures{Text: "quick docs update without security impact."} + target, ok := preclassifyTargetModel(config.RoutingPreclassifierConfig{ + MinConfidence: 0.8, + NegationPhrases: []string{"without"}, + Targets: []config.RoutingPreclassifierTargetConfig{{ + TargetModel: "deep-model", + Keywords: []string{"security"}, + Confidence: 0.95, + }}, + }, features) + + if ok { + t.Fatalf("expected custom-negated keyword to fall through, got target %q", target) + } +} + +func TestRoutingPreclassifierFallsThroughBelowConfidence(t *testing.T) { + t.Parallel() + + features := routingFeatures{Text: "review a terraform plan"} + target, ok := preclassifyTargetModel(config.RoutingPreclassifierConfig{ + MinConfidence: 0.9, + Targets: []config.RoutingPreclassifierTargetConfig{{ + TargetModel: "deep-model", + Keywords: []string{"terraform"}, + Confidence: 0.7, + }}, + }, features) + + if ok { + t.Fatalf("expected low-confidence keyword to fall through, got target %q", target) + } +} + func TestRewriteModelClampsOversizedTargetAliasOutput(t *testing.T) { t.Parallel() diff --git a/test/bench/router-routing.cases.json b/test/bench/router-routing.cases.json new file mode 100644 index 0000000..39361da --- /dev/null +++ b/test/bench/router-routing.cases.json @@ -0,0 +1,104 @@ +[ + { + "id": "small-git-cleanup", + "expected": "fast", + "prompt": "Clean up a stale local git branch, remove an unused temporary file, and summarize the diff. No production services are involved." + }, + { + "id": "frontend-copy", + "expected": "fast", + "prompt": "Fix a button label and adjust spacing in a small React component, then run the focused unit test." + }, + { + "id": "status-question", + "expected": "fast", + "prompt": "Check whether the local working tree is clean and tell me the current branch." + }, + { + "id": "formatting-only", + "expected": "fast", + "prompt": "Run the formatter on the touched Go files and report whether anything changed." + }, + { + "id": "small-doc-edit", + "expected": "fast", + "prompt": "Add one sentence to the README explaining how to run the local smoke test." + }, + { + "id": "unit-test-fix", + "expected": "fast", + "prompt": "A focused unit test has a changed expected string after a copy edit. Update the assertion and run that one package test." + }, + { + "id": "production-incident", + "expected": "strong", + "prompt": "Diagnose a production Kubernetes outage with database connection failures, identify likely rollback options, and propose the safest recovery sequence." + }, + { + "id": "terraform-firewall", + "expected": "strong", + "prompt": "Update Terraform-managed OPNsense firewall policy for cross-site BGP, prove the plan scope, apply only the approved rules, and verify route exchange." + }, + { + "id": "security-review", + "expected": "strong", + "prompt": "Review an authentication middleware change for token leakage, replay risk, and missing audit coverage before it ships." + }, + { + "id": "database-migration", + "expected": "strong", + "prompt": "Design and implement a Postgres migration that backfills tenant entitlement state without downtime and with rollback instructions." + }, + { + "id": "broad-refactor", + "expected": "strong", + "prompt": "Refactor the request routing layer across multiple packages while preserving streaming behavior, queue limits, metrics, and error responses." + }, + { + "id": "root-cause-debug", + "expected": "strong", + "prompt": "Find the root cause of intermittent SSE stream truncation under concurrent load and propose a production-safe fix." + }, + { + "id": "large-output-request", + "expected": "strong", + "max_tokens": 12000, + "prompt": "Generate a detailed implementation plan for a multi-service migration." + }, + { + "id": "large-prompt-session", + "expected": "strong", + "repeat": 9000, + "prompt": "Existing session context includes many prior file reads, logs, and partial patches. Decide the safest next implementation step.\n" + }, + { + "id": "ambiguous-architecture", + "expected": "strong", + "prompt": "Should we change the architecture so opencode can use one alias while the router dynamically chooses fast, strong, and deep backends?" + }, + { + "id": "dependency-bump", + "expected": "fast", + "prompt": "Bump one dev-only lint dependency, run the lockfile update, and report the package test result." + }, + { + "id": "billing-risk", + "expected": "strong", + "prompt": "Change billing entitlement enforcement so no-charge tenants bypass Stripe while paid tenants still require active subscriptions." + }, + { + "id": "dns-change", + "expected": "strong", + "prompt": "Update public DNS and reverse proxy routing for a customer-facing application, then prove the deployed host and TLS route." + }, + { + "id": "simple-command", + "expected": "fast", + "prompt": "Run date and tell me the current time." + }, + { + "id": "concurrency-bug", + "expected": "strong", + "prompt": "Investigate a concurrency bug where queued model requests sometimes time out after an active request finishes." + } +] diff --git a/tools/route_classifier_bench.py b/tools/route_classifier_bench.py new file mode 100644 index 0000000..314cc41 --- /dev/null +++ b/tools/route_classifier_bench.py @@ -0,0 +1,260 @@ +#!/usr/bin/env python3 +"""Benchmark router-classifier candidates against labeled routing cases.""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +import time +import urllib.error +import urllib.request +from dataclasses import dataclass +from pathlib import Path +from typing import Any + + +STRONG_KEYWORDS = { + "architecture", + "audit", + "billing", + "bgp", + "concurrency", + "customer-facing", + "database", + "dns", + "firewall", + "incident", + "kubernetes", + "migration", + "opnsense", + "production", + "refactor", + "root cause", + "security", + "stripe", + "terraform", +} + + +@dataclass(frozen=True) +class Case: + id: str + expected: str + prompt: str + max_tokens: int + + @property + def prompt_chars(self) -> int: + return len(self.prompt) + + +def load_cases(path: Path) -> list[Case]: + raw_cases = json.loads(path.read_text()) + cases: list[Case] = [] + for raw in raw_cases: + prompt = str(raw.get("prompt", "")) + repeat = int(raw.get("repeat", 1)) + cases.append( + Case( + id=str(raw["id"]), + expected=str(raw["expected"]), + prompt=prompt * repeat, + max_tokens=int(raw.get("max_tokens", 512)), + ) + ) + return cases + + +def guardrail_candidate(case: Case, *, large_prompt_chars: int, high_output_tokens: int) -> tuple[str, float]: + if case.prompt_chars >= large_prompt_chars or case.max_tokens > high_output_tokens: + return "strong", 1.0 + return "fast", 0.55 + + +def keyword_candidate(case: Case, *, large_prompt_chars: int, high_output_tokens: int) -> tuple[str, float]: + label, confidence = guardrail_candidate( + case, + large_prompt_chars=large_prompt_chars, + high_output_tokens=high_output_tokens, + ) + if label == "strong": + return label, confidence + + text = case.prompt.lower() + hits = [ + keyword + for keyword in STRONG_KEYWORDS + if keyword in text and not negated_keyword(text, keyword) + ] + if hits: + return "strong", min(0.99, 0.68 + len(hits) * 0.06) + return "fast", 0.72 + + +def negated_keyword(text: str, keyword: str) -> bool: + return any( + phrase in text + for phrase in ( + f"no {keyword}", + f"not {keyword}", + f"non-{keyword}", + f"without {keyword}", + ) + ) + + +def llm_system_prompt(targets: list[str]) -> str: + return ( + "You route coding-agent requests to the cheapest adequate model. " + "Return only JSON with target_model set to one of: " + + ", ".join(targets) + + ". Choose the stronger model for architecture, debugging, security, migrations, " + "production risk, broad refactors, ambiguous planning, long-context synthesis, or high requested output. " + "Choose the faster model for routine edits, short questions, status checks, formatting, simple commands, " + "and low-risk local changes." + ) + + +def llm_user_prompt(case: Case, targets: list[str]) -> str: + text = case.prompt.lower() + if len(text) > 6000: + text = text[:3000] + "\n...[middle omitted]...\n" + text[-3000:] + return ( + "/no_think\n" + f"Allowed targets: {', '.join(targets)}\n" + f"Prompt chars: {case.prompt_chars}\n" + f"Requested max output tokens: {case.max_tokens}\n" + f"Request text:\n{text}\n\n" + 'Return only: {"target_model":""}' + ) + + +def openai_candidate(case: Case, args: argparse.Namespace) -> tuple[str, float, str, str]: + targets = ["fast", "strong"] + payload = { + "model": args.openai_model, + "messages": [ + {"role": "system", "content": llm_system_prompt(targets)}, + {"role": "user", "content": llm_user_prompt(case, targets)}, + ], + "temperature": 0, + "stream": False, + "max_tokens": args.openai_max_tokens, + } + data = json.dumps(payload).encode() + url = args.openai_base_url.rstrip("/") + "/chat/completions" + request = urllib.request.Request(url, data=data, headers={"Content-Type": "application/json"}) + if args.openai_api_key: + request.add_header("Authorization", "Bearer " + args.openai_api_key) + + with urllib.request.urlopen(request, timeout=args.timeout) as response: + body = response.read().decode() + + target, parse_source = parse_target(body, targets) + return target or "unknown", 1.0 if target else 0.0, body[:2000], parse_source + + +def parse_target(text: str, targets: list[str]) -> tuple[str | None, str]: + try: + payload = json.loads(text) + choices = payload.get("choices") or [] + combined = "\n".join( + str(choice.get("text", "")) + + "\n" + + str((choice.get("message") or {}).get("content", "")) + + "\n" + + str((choice.get("message") or {}).get("reasoning_content", "")) + for choice in choices + ) + except json.JSONDecodeError: + combined = text + + match = re.search(r'\{\s*"target_model"\s*:\s*"([^"]+)"\s*\}', combined) + if match and match.group(1) in targets: + return match.group(1), "json" + for target in targets: + if target in combined: + return target, "substring" + return None, "none" + + +def run_candidate(name: str, case: Case, args: argparse.Namespace) -> dict[str, Any]: + started = time.perf_counter() + error = "" + raw = "" + parse_source = "" + try: + if name == "guardrails": + prediction, confidence = guardrail_candidate( + case, + large_prompt_chars=args.large_prompt_chars, + high_output_tokens=args.high_output_tokens, + ) + elif name == "keywords": + prediction, confidence = keyword_candidate( + case, + large_prompt_chars=args.large_prompt_chars, + high_output_tokens=args.high_output_tokens, + ) + elif name == "openai": + prediction, confidence, raw, parse_source = openai_candidate(case, args) + else: + raise ValueError(f"unknown candidate {name!r}") + except (OSError, TimeoutError, urllib.error.URLError, ValueError) as exc: + prediction = "error" + confidence = 0.0 + error = str(exc) + + duration_ms = round((time.perf_counter() - started) * 1000, 3) + return { + "case_id": case.id, + "candidate": name, + "expected": case.expected, + "prediction": prediction, + "correct": prediction == case.expected, + "confidence": confidence, + "duration_ms": duration_ms, + "prompt_chars": case.prompt_chars, + "max_tokens": case.max_tokens, + "error": error, + "parse_source": parse_source, + "raw_sample": raw, + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--cases", default="test/bench/router-routing.cases.json") + parser.add_argument("--candidates", default="guardrails,keywords") + parser.add_argument("--large-prompt-chars", type=int, default=80000) + parser.add_argument("--high-output-tokens", type=int, default=8192) + parser.add_argument("--openai-base-url", default="http://llm-srv-01.mfsoho.linkridge.net:18080/v1") + parser.add_argument("--openai-model", default="local-coder-fast") + parser.add_argument("--openai-api-key", default="") + parser.add_argument("--openai-max-tokens", type=int, default=192) + parser.add_argument("--timeout", type=float, default=120) + args = parser.parse_args() + + cases = load_cases(Path(args.cases)) + candidates = [candidate.strip() for candidate in args.candidates.split(",") if candidate.strip()] + results = [run_candidate(candidate, case, args) for candidate in candidates for case in cases] + + for result in results: + print(json.dumps(result, separators=(",", ":"))) + + for candidate in candidates: + subset = [result for result in results if result["candidate"] == candidate] + correct = sum(1 for result in subset if result["correct"]) + total = len(subset) + avg_ms = sum(float(result["duration_ms"]) for result in subset) / total if total else 0 + print( + f"summary candidate={candidate} accuracy={correct}/{total} avg_ms={avg_ms:.3f}", + file=sys.stderr, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())