bench: add route classifier benchmark and preclassifier

README.md

@@ -136,6 +136,18 @@ models:
         - id: large-prompt
           target_model: qwen/qwen3.6-35b-a3b
           min_prompt_chars: 12000
+      preclassifier:
+        min_confidence: 0.8
+        targets:
+          - target_model: qwen/qwen3.6-35b-a3b
+            confidence: 0.95
+            keywords:
+              - architecture
+              - migration
+              - production
+              - root cause
+              - security
+              - terraform
       classifier:
         backend: lmstudio
         model: qwen3-coder-30b-a3b-instruct

configs/router.example.yaml

@@ -49,6 +49,18 @@ models:
         - id: large-prompt
           target_model: qwen/qwen3.6-35b-a3b
           min_prompt_chars: 12000
+      preclassifier:
+        min_confidence: 0.8
+        targets:
+          - target_model: qwen/qwen3.6-35b-a3b
+            confidence: 0.95
+            keywords:
+              - architecture
+              - migration
+              - production
+              - root cause
+              - security
+              - terraform
       classifier:
         backend: lmstudio
         model: qwen3-coder-30b-a3b-instruct

docs/benchmarking.md

@@ -103,3 +103,38 @@ go run ./cmd/devrail-router bench \
 The deep alias may spend early tokens on reasoning before emitting normal
 content, so use a larger token cap than a smoke test. Compare both timings and
 the `content_sample` field before making a slower backend automatic.
+
+## Routing Classifier Benchmarks
+
+Use the route-classifier benchmark to compare fast-vs-strong selection policies
+without sending full generation requests through the router:
+
+```sh
+python3 tools/route_classifier_bench.py \
+  --cases test/bench/router-routing.cases.json \
+  --candidates guardrails,keywords
+```
+
+Add `openai` to compare a live OpenAI-compatible classifier model:
+
+```sh
+python3 tools/route_classifier_bench.py \
+  --candidates guardrails,keywords,openai \
+  --openai-base-url http://llm-srv-01.mfsoho.linkridge.net:18080/v1 \
+  --openai-model local-coder-fast
+```
+
+The tool writes one JSON object per candidate/case to stdout and prints per
+candidate accuracy/timing summaries to stderr.
+
+Router selection order is:
+
+1. explicit routing rules, such as prompt-size and output-token guardrails
+2. optional keyword preclassifier for high-confidence cheap decisions
+3. optional OpenAI-compatible LLM classifier for ambiguous requests
+4. the model alias default target
+
+Use the benchmark corpus to tune the preclassifier keyword list before enabling
+it in deployed config. Negated phrases such as `no production` and
+`without security impact` should remain fall-through cases so they can reach the
+LLM classifier or default route.

internal/config/config.go

@@ -52,8 +52,9 @@ type EnsureConfig struct {
 }
 
 type RoutingConfig struct {
-	Rules      []RoutingRuleConfig     `yaml:"rules"`
-	Classifier RoutingClassifierConfig `yaml:"classifier"`
+	Rules         []RoutingRuleConfig        `yaml:"rules"`
+	Preclassifier RoutingPreclassifierConfig `yaml:"preclassifier"`
+	Classifier    RoutingClassifierConfig    `yaml:"classifier"`
 }
 
 type RoutingRuleConfig struct {
@@ -75,6 +76,18 @@ type RoutingClassifierConfig struct {
 	MaxTokens    int      `yaml:"max_tokens"`
 }
 
+type RoutingPreclassifierConfig struct {
+	Targets         []RoutingPreclassifierTargetConfig `yaml:"targets"`
+	MinConfidence   float64                            `yaml:"min_confidence"`
+	NegationPhrases []string                           `yaml:"negation_phrases"`
+}
+
+type RoutingPreclassifierTargetConfig struct {
+	TargetModel string   `yaml:"target_model"`
+	Keywords    []string `yaml:"keywords"`
+	Confidence  float64  `yaml:"confidence"`
+}
+
 type CommandArgs []string
 
 func (args *CommandArgs) UnmarshalYAML(value *yaml.Node) error {
@@ -236,10 +249,51 @@ func (routing RoutingConfig) Validate() error {
 	if err := routing.Classifier.Validate(); err != nil {
 		return fmt.Errorf("classifier is invalid: %w", err)
 	}
+	if err := routing.Preclassifier.Validate(); err != nil {
+		return fmt.Errorf("preclassifier is invalid: %w", err)
+	}
 
 	return nil
 }
 
+func (preclassifier RoutingPreclassifierConfig) Validate() error {
+	if !preclassifier.Enabled() {
+		return nil
+	}
+	if len(preclassifier.Targets) == 0 {
+		return errors.New("targets is required")
+	}
+	if preclassifier.MinConfidence < 0 || preclassifier.MinConfidence > 1 {
+		return errors.New("min_confidence must be between 0 and 1")
+	}
+	for phraseIndex, phrase := range preclassifier.NegationPhrases {
+		if strings.TrimSpace(phrase) == "" {
+			return fmt.Errorf("negation_phrases[%d] must not be empty", phraseIndex)
+		}
+	}
+	for index, target := range preclassifier.Targets {
+		if strings.TrimSpace(target.TargetModel) == "" {
+			return fmt.Errorf("targets[%d].target_model is required", index)
+		}
+		if len(target.Keywords) == 0 {
+			return fmt.Errorf("targets[%d].keywords is required", index)
+		}
+		if target.Confidence <= 0 || target.Confidence > 1 {
+			return fmt.Errorf("targets[%d].confidence must be greater than 0 and less than or equal to 1", index)
+		}
+		for keywordIndex, keyword := range target.Keywords {
+			if strings.TrimSpace(keyword) == "" {
+				return fmt.Errorf("targets[%d].keywords[%d] must not be empty", index, keywordIndex)
+			}
+		}
+	}
+	return nil
+}
+
+func (preclassifier RoutingPreclassifierConfig) Enabled() bool {
+	return len(preclassifier.Targets) > 0 || preclassifier.MinConfidence != 0 || len(preclassifier.NegationPhrases) > 0
+}
+
 func (classifier RoutingClassifierConfig) Validate() error {
 	if classifier.Backend == "" && classifier.Model == "" && len(classifier.TargetModels) == 0 && classifier.SystemPrompt == "" && classifier.Timeout == "" && classifier.MaxTokens == 0 {
 		return nil

internal/config/config_test.go

@@ -127,6 +127,70 @@ func TestValidateRoutingRuleRequiresCondition(t *testing.T) {
 	}
 }
 
+func TestValidateRoutingPreclassifier(t *testing.T) {
+	t.Parallel()
+
+	cfg := Config{
+		Models: []ModelConfig{{
+			ID:          "local-coder",
+			Backend:     "lmstudio",
+			TargetModel: "fast-model",
+			Routing: RoutingConfig{
+				Preclassifier: RoutingPreclassifierConfig{
+					MinConfidence: 0.8,
+					Targets: []RoutingPreclassifierTargetConfig{{
+						TargetModel: "deep-model",
+						Keywords:    []string{"production", "terraform"},
+						Confidence:  0.95,
+					}},
+				},
+			},
+		}},
+		Backends: []BackendConfig{{
+			ID:      "lmstudio",
+			BaseURL: "http://127.0.0.1:1234/v1",
+		}},
+	}
+
+	if err := cfg.Validate(); err != nil {
+		t.Fatalf("validate config: %v", err)
+	}
+}
+
+func TestValidateRoutingPreclassifierRejectsInvalidConfidence(t *testing.T) {
+	t.Parallel()
+
+	cfg := Config{
+		Models: []ModelConfig{{
+			ID:          "local-coder",
+			Backend:     "lmstudio",
+			TargetModel: "fast-model",
+			Routing: RoutingConfig{
+				Preclassifier: RoutingPreclassifierConfig{
+					MinConfidence: 1.2,
+					Targets: []RoutingPreclassifierTargetConfig{{
+						TargetModel: "deep-model",
+						Keywords:    []string{"production"},
+						Confidence:  0.95,
+					}},
+				},
+			},
+		}},
+		Backends: []BackendConfig{{
+			ID:      "lmstudio",
+			BaseURL: "http://127.0.0.1:1234/v1",
+		}},
+	}
+
+	err := cfg.Validate()
+	if err == nil {
+		t.Fatal("expected validation error")
+	}
+	if !strings.Contains(err.Error(), "preclassifier") {
+		t.Fatalf("expected preclassifier error, got: %v", err)
+	}
+}
+
 func TestLoadEnsureCommandString(t *testing.T) {
 	t.Parallel()
 

internal/server/server.go

@@ -922,6 +922,14 @@ func (s *Server) selectTargetModel(ctx context.Context, body []byte, model confi
 		}
 	}
 
+	if model.Routing.Preclassifier.Enabled() {
+		target, ok := preclassifyTargetModel(model.Routing.Preclassifier, features)
+		if ok {
+			slog.Info("routing preclassifier selected target", "request_id", requestID, "alias", model.ID, "target_model", target)
+			return target, "preclassifier"
+		}
+	}
+
 	if model.Routing.Classifier.Enabled() {
 		target, ok := s.classifyTargetModel(ctx, model, features, requestID)
 		if ok {
@@ -932,6 +940,63 @@ func (s *Server) selectTargetModel(ctx context.Context, body []byte, model confi
 	return model.TargetModel, "default"
 }
 
+func preclassifyTargetModel(preclassifier config.RoutingPreclassifierConfig, features routingFeatures) (string, bool) {
+	minConfidence := preclassifier.MinConfidence
+	if minConfidence == 0 {
+		minConfidence = 0.75
+	}
+
+	bestTarget := ""
+	bestConfidence := 0.0
+	for _, target := range preclassifier.Targets {
+		if !preclassifierTargetMatches(target, features, preclassifier.NegationPhrases) {
+			continue
+		}
+		if target.Confidence > bestConfidence {
+			bestTarget = target.TargetModel
+			bestConfidence = target.Confidence
+		}
+	}
+	if bestTarget == "" || bestConfidence < minConfidence {
+		return "", false
+	}
+	return bestTarget, true
+}
+
+func preclassifierTargetMatches(target config.RoutingPreclassifierTargetConfig, features routingFeatures, negationPhrases []string) bool {
+	for _, keyword := range target.Keywords {
+		normalized := strings.ToLower(strings.TrimSpace(keyword))
+		if normalized == "" || !strings.Contains(features.Text, normalized) {
+			continue
+		}
+		if preclassifierKeywordNegated(features.Text, normalized, negationPhrases) {
+			continue
+		}
+		return true
+	}
+	return false
+}
+
+func preclassifierKeywordNegated(text string, keyword string, configuredPhrases []string) bool {
+	phrases := configuredPhrases
+	if len(phrases) == 0 {
+		phrases = []string{"no ", "not ", "without ", "non-"}
+	}
+	for _, phrase := range phrases {
+		normalized := strings.ToLower(strings.TrimSpace(phrase))
+		if normalized == "" {
+			continue
+		}
+		if strings.Contains(text, normalized+keyword) {
+			return true
+		}
+		if !strings.HasSuffix(normalized, "-") && strings.Contains(text, normalized+" "+keyword) {
+			return true
+		}
+	}
+	return false
+}
+
 func (s *Server) classifyTargetModel(ctx context.Context, model config.ModelConfig, features routingFeatures, requestID string) (string, bool) {
 	classifier := model.Routing.Classifier
 	backend, ok := s.cfg.Backend(classifier.Backend)