feat(judge): validated named metrics and persistence (script judge, slice 1)

internal/cli/run_events.go

@@ -167,10 +167,33 @@ func runEvalWithEventLog(
 
 func installEvaluationSignalContext(cmd *cobra.Command) func() {
 	parent := cmd.Context()
+	// Register the observer before NotifyContext so the buffered signal is already
+	// queued when the cancel path runs, and drain it non-blockingly on ctx.Done so a
+	// signal that arrived first is still attributed. A cancellation with no queued
+	// signal stays "unknown" rather than being guessed as user-initiated.
+	observer := &evaluator.SignalObserver{}
+	observed := make(chan os.Signal, 1)
+	signal.Notify(observed, os.Interrupt, syscall.SIGTERM)
 	ctx, stop := signal.NotifyContext(parent, os.Interrupt, syscall.SIGTERM)
-	cmd.SetContext(ctx)
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		select {
+		case <-observed:
+			observer.MarkSignalObserved()
+		case <-ctx.Done():
+			select {
+			case <-observed:
+				observer.MarkSignalObserved()
+			default:
+			}
+		}
+	}()
+	cmd.SetContext(evaluator.ContextWithSignalObserver(ctx, observer))
 	return func() {
 		stop()
+		<-done
+		signal.Stop(observed)
 		cmd.SetContext(parent)
 	}
 }

internal/config/metrics_validation_test.go

@@ -0,0 +1,136 @@
+package config
+
+import (
+	"math"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func TestValidateJudgeMetrics(t *testing.T) {
+	cases := []struct {
+		name     string
+		metrics  []MetricConfig
+		wantErrs []string
+		wantOK   bool
+	}{
+		{name: "absent registry is valid legacy configuration", metrics: nil, wantOK: true},
+		{
+			name:    "two well-formed slots",
+			metrics: []MetricConfig{{ID: "correctness", Required: true}, {ID: "completeness", MinScore: new(0.7)}},
+			wantOK:  true,
+		},
+		{
+			name:    "a zero threshold is distinguishable from an absent one and stays valid",
+			metrics: []MetricConfig{{ID: "correctness", MinScore: new(0.0)}},
+			wantOK:  true,
+		},
+		{
+			name:    "upper bound is inclusive",
+			metrics: []MetricConfig{{ID: "correctness", MinScore: new(1.0)}},
+			wantOK:  true,
+		},
+		{
+			name:     "threshold above the score range is rejected",
+			metrics:  []MetricConfig{{ID: "correctness", MinScore: new(1.1)}},
+			wantErrs: []string{"min_score must be between 0.0 and 1.0"},
+		},
+		{
+			name:     "negative threshold is rejected",
+			metrics:  []MetricConfig{{ID: "correctness", MinScore: new(-0.5)}},
+			wantErrs: []string{"min_score must be between 0.0 and 1.0"},
+		},
+		{
+			name:     "duplicate slot ids are rejected",
+			metrics:  []MetricConfig{{ID: "correctness"}, {ID: "correctness"}},
+			wantErrs: []string{`metrics[1].id "correctness" is duplicated`},
+		},
+		{
+			name:     "a NaN threshold is rejected because every comparison against it is false",
+			metrics:  []MetricConfig{{ID: "correctness", MinScore: new(math.NaN())}},
+			wantErrs: []string{"min_score must be between 0.0 and 1.0"},
+		},
+		{
+			name:     "an infinite threshold is rejected",
+			metrics:  []MetricConfig{{ID: "correctness", MinScore: new(math.Inf(1))}},
+			wantErrs: []string{"min_score must be between 0.0 and 1.0"},
+		},
+		{
+			name:     "an unsafe identifier is rejected",
+			metrics:  []MetricConfig{{ID: "Correctness!"}},
+			wantErrs: []string{"id must be a safe lowercase identifier"},
+		},
+		{
+			name:     "an empty identifier is rejected",
+			metrics:  []MetricConfig{{ID: ""}},
+			wantErrs: []string{"id must be a safe lowercase identifier"},
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			errs := validateJudgeMetrics("judge", tc.metrics)
+			if tc.wantOK {
+				if len(errs) != 0 {
+					t.Fatalf("want valid, got %v", errs)
+				}
+				return
+			}
+			if len(errs) != len(tc.wantErrs) {
+				t.Fatalf("errors = %v, want %v", errs, tc.wantErrs)
+			}
+			for i, want := range tc.wantErrs {
+				if !strings.Contains(errs[i], want) {
+					t.Fatalf("error %d = %q, want substring %q", i, errs[i], want)
+				}
+			}
+		})
+	}
+}
+
+func TestMetricRegistryKeepsAnAbsentThresholdDistinctFromZero(t *testing.T) {
+	// A nil MinScore means "no threshold decision configured", which the quality
+	// derivation reports as NOT_CONFIGURED rather than UNKNOWN; a zero threshold is
+	// a real requirement. The two must not collapse during configuration loading.
+	absent := MetricConfig{ID: "correctness"}
+	zero := MetricConfig{ID: "completeness", MinScore: new(0.0)}
+	if absent.MinScore != nil {
+		t.Fatalf("unset threshold must stay nil, got %v", *absent.MinScore)
+	}
+	if zero.MinScore == nil || *zero.MinScore != 0 {
+		t.Fatal("an explicit zero threshold was lost")
+	}
+}
+
+// TestYAMLRegistryParsesNanThreshold pins the premise behind the non-finite
+// rejection: the real YAML loader turns `min_score: .nan` into a NaN *float64, and
+// every comparison against NaN is false, so an unvalidated registry would reach
+// quality derivation and report MET for any available score.
+func TestYAMLRegistryParsesNanThreshold(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "eval.yaml")
+	content := `schema_version: v1alpha1
+judge:
+  type: script
+  script_path: score.sh
+  metrics:
+    - id: correctness
+      min_score: .nan
+cases:
+  files: []
+`
+	if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	cfg, err := NewLoader(path).LoadEvalConfig()
+	if err != nil {
+		t.Fatalf("load: %v", err)
+	}
+	metric := cfg.Judge.Metrics[0]
+	if metric.MinScore == nil || !math.IsNaN(*metric.MinScore) {
+		t.Fatalf("min_score = %+v, want NaN parsed from `.nan`", metric.MinScore)
+	}
+	if errs := validateJudgeMetrics("judge.metrics", cfg.Judge.Metrics); len(errs) == 0 || !strings.Contains(errs[0], "min_score") {
+		t.Fatalf("validation errors = %v, want the NaN threshold rejected", errs)
+	}
+}

internal/config/schema.go

@@ -200,6 +200,23 @@ type JudgeConfig struct {
 	TimeoutSeconds *int   `json:"timeout_seconds,omitempty" yaml:"timeout_seconds,omitempty"`
 	Success        []Rule `json:"success,omitempty"        yaml:"success,omitempty"`
 	Failure        []Rule `json:"failure,omitempty"        yaml:"failure,omitempty"`
+	// Metrics declares the planned named-metric slots for this judge. The list is
+	// the registry: one slot per entry, fixed before execution, and the coverage
+	// denominator never moves. Omitted means no named metrics for this judge.
+	Metrics []MetricConfig `json:"metrics,omitempty" yaml:"metrics,omitempty"`
+}
+
+// MetricConfig is one planned named-metric registry slot.
+type MetricConfig struct {
+	// ID is a stable configuration identifier, never a display label produced by a judge.
+	ID string `json:"id" yaml:"id"`
+	// Required marks the metric as part of the declared scoring requirements.
+	Required bool `json:"required,omitempty" yaml:"required,omitempty"`
+	// MinScore is an optional quality threshold. Nil means no threshold decision is
+	// configured for this metric, which is distinct from a threshold of zero.
+	MinScore *float64 `json:"min_score,omitempty" yaml:"min_score,omitempty"`
+	// EvidenceRequires makes non-empty evidence mandatory for an AVAILABLE score.
+	EvidenceRequired bool `json:"evidence_required,omitempty" yaml:"evidence_required,omitempty"`
 }
 
 // JudgeContextConfig controls how agent_judge receives evaluation materials.

internal/config/validator.go

@@ -3,6 +3,7 @@ package config
 import (
 	"fmt"
 	"maps"
+	"math"
 	"path"
 	"regexp"
 	"slices"
@@ -277,6 +278,7 @@ func validateJudgeTypeAndLocalFields(judge JudgeConfig) []string {
 	errs = append(errs, validateJudgeEngine(judge)...)
 
 	errs = append(errs, validatePassThreshold(judge.PassThreshold)...)
+	errs = append(errs, validateJudgeMetrics("judge.metrics", judge.Metrics)...)
 	errs = append(errs, validateJudgeContext(judge.Context)...)
 
 	if judge.TimeoutSeconds != nil && *judge.TimeoutSeconds < 0 {
@@ -632,6 +634,37 @@ func validateJudgeList(members *[]JudgeConfig) []string {
 		for _, detail := range validateJudgeTypeAndRequiredFields(member) {
 			errs = append(errs, fmt.Sprintf("judges[%d]: %s", i, detail))
 		}
+		for _, detail := range validateJudgeMetrics("metrics", member.Metrics) {
+			errs = append(errs, fmt.Sprintf("judges[%d].%s", i, detail))
+		}
+	}
+	return errs
+}
+
+// validateJudgeMetrics checks the planned named-metric registry. Ids are stable
+// configuration identifiers, must be unique, and a configured threshold has to sit
+// inside the declared [0,1] score range.
+func validateJudgeMetrics(configPath string, metrics []MetricConfig) []string {
+	if len(metrics) == 0 {
+		return nil
+	}
+	var errs []string
+	seen := make(map[string]bool, len(metrics))
+	for i, metric := range metrics {
+		if !judgeIDPattern.MatchString(metric.ID) {
+			errs = append(errs, fmt.Sprintf("%s.metrics[%d].id must be a safe lowercase identifier", configPath, i))
+		}
+		if seen[metric.ID] {
+			errs = append(errs, fmt.Sprintf("%s.metrics[%d].id %q is duplicated", configPath, i, metric.ID))
+		}
+		seen[metric.ID] = true
+		// NaN slips through a bare range check because every comparison against it
+		// is false, and a non-finite threshold would then make the quality
+		// derivation report MET for any score.
+		if metric.MinScore != nil && (math.IsNaN(*metric.MinScore) || math.IsInf(*metric.MinScore, 0) ||
+			*metric.MinScore < 0.0 || *metric.MinScore > 1.0) {
+			errs = append(errs, fmt.Sprintf("%s.metrics[%d].min_score must be between 0.0 and 1.0", configPath, i))
+		}
 	}
 	return errs
 }

internal/evaluator/cancellation.go

@@ -0,0 +1,80 @@
+package evaluator
+
+import (
+	"context"
+	"sync"
+)
+
+// Execution-provenance context values used to classify why a judge stopped.
+// They exist so a cause is never reconstructed from an error string, and so an
+// unattributable cancellation stays "unknown" instead of being guessed as
+// user-initiated.
+
+type (
+	signalObserverKey struct{}
+	deadlineSourceKey struct{}
+)
+
+// SignalObserver records whether this process actually observed an interrupt.
+// The flag is set by the signal handler path, never inferred afterwards.
+type SignalObserver struct {
+	mu    sync.Mutex
+	fired bool
+}
+
+// MarkSignalObserved is called from the interrupt delivery path.
+func (o *SignalObserver) MarkSignalObserved() {
+	if o == nil {
+		return
+	}
+	o.mu.Lock()
+	defer o.mu.Unlock()
+	o.fired = true
+}
+
+// Observed reports whether a signal was seen. A false result with no observer
+// attached means the source is unknown, not that no signal occurred.
+func (o *SignalObserver) Observed() bool {
+	if o == nil {
+		return false
+	}
+	o.mu.Lock()
+	defer o.mu.Unlock()
+	return o.fired
+}
+
+// ContextWithSignalObserver attaches the observer to ctx for downstream
+// classification.
+func ContextWithSignalObserver(ctx context.Context, observer *SignalObserver) context.Context {
+	if observer == nil {
+		return ctx
+	}
+	return context.WithValue(ctx, signalObserverKey{}, observer)
+}
+
+// SignalObserved reports whether a process interrupt was observed for this ctx.
+func SignalObserved(ctx context.Context) bool {
+	if ctx == nil {
+		return false
+	}
+	observer, ok := ctx.Value(signalObserverKey{}).(*SignalObserver)
+	return ok && observer.Observed()
+}
+
+// ContextWithDeadlineSource names the configuration knob that armed the deadline
+// carried by this context. An empty source stays unattributed downstream.
+func ContextWithDeadlineSource(ctx context.Context, source string) context.Context {
+	if source == "" {
+		return ctx
+	}
+	return context.WithValue(ctx, deadlineSourceKey{}, source)
+}
+
+// DeadlineSource returns the recorded knob name, or "" when unknown.
+func DeadlineSource(ctx context.Context) string {
+	if ctx == nil {
+		return ""
+	}
+	source, _ := ctx.Value(deadlineSourceKey{}).(string)
+	return source
+}