From 5488fbf230bce9a33c01995e9886ae26c7befb70 Mon Sep 17 00:00:00 2001 From: Max_Guobaixin Date: Tue, 27 Oct 2026 18:24:17 +0800 Subject: [PATCH 01/12] feat(judge): validated named-metric scoring and persistence (#317) Adds the first bounded slice of the #317 contract: a per-judge metric registry, four independent state axes (execution stage, completion, termination cause, slot availability), a derived quality state, whole-payload versus single-slot validation, and group completion folding that preserves earlier member detail. Execution metadata and planned metric slots are recorded on the member outcome and the grouped path even when no judge result exists, so a gate short-circuit or a cancellation never loses them. An accepted score survives a later cancellation or timeout, and the timeout, its source and any recovery stay visible. Existing PASS/FAIL/SKIP/ERROR verdicts, the #309 assertion and member denominators, script-timeout verdict mapping and the JSONL event schema are unchanged. --- internal/cli/run_events.go | 25 +- internal/config/metrics_validation_test.go | 92 ++++ internal/config/schema.go | 17 + internal/config/validator.go | 28 ++ internal/evaluator/cancellation.go | 78 ++++ internal/evaluator/evaluator.go | 3 + internal/evaluator/metrics_wiring_test.go | 407 +++++++++++++++++ internal/evaluator/multi_judge.go | 175 ++++++- internal/judge/README.md | 58 +++ internal/judge/compat.go | 67 +++ internal/judge/judge.go | 11 + internal/judge/metrics.go | 438 ++++++++++++++++++ internal/judge/metrics_test.go | 502 +++++++++++++++++++++ 13 files changed, 1895 insertions(+), 6 deletions(-) create mode 100644 internal/config/metrics_validation_test.go create mode 100644 internal/evaluator/cancellation.go create mode 100644 internal/evaluator/metrics_wiring_test.go create mode 100644 internal/judge/compat.go create mode 100644 internal/judge/metrics.go create mode 100644 internal/judge/metrics_test.go diff --git a/internal/cli/run_events.go b/internal/cli/run_events.go index 8a441177..3ac9c1b3 100644 --- a/internal/cli/run_events.go +++ b/internal/cli/run_events.go @@ -167,10 +167,33 @@ func runEvalWithEventLog( func installEvaluationSignalContext(cmd *cobra.Command) func() { parent := cmd.Context() + // Register the observer before NotifyContext so the buffered signal is already + // queued when the cancel path runs, and drain it non-blockingly on ctx.Done so a + // signal that arrived first is still attributed. A cancellation with no queued + // signal stays "unknown" rather than being guessed as user-initiated. + observer := &evaluator.SignalObserver{} + observed := make(chan os.Signal, 1) + signal.Notify(observed, os.Interrupt, syscall.SIGTERM) ctx, stop := signal.NotifyContext(parent, os.Interrupt, syscall.SIGTERM) - cmd.SetContext(ctx) + done := make(chan struct{}) + go func() { + defer close(done) + select { + case <-observed: + observer.MarkSignalObserved() + case <-ctx.Done(): + select { + case <-observed: + observer.MarkSignalObserved() + default: + } + } + }() + cmd.SetContext(evaluator.ContextWithSignalObserver(ctx, observer)) return func() { stop() + <-done + signal.Stop(observed) cmd.SetContext(parent) } } diff --git a/internal/config/metrics_validation_test.go b/internal/config/metrics_validation_test.go new file mode 100644 index 00000000..3ddf3a6e --- /dev/null +++ b/internal/config/metrics_validation_test.go @@ -0,0 +1,92 @@ +package config + +import ( + "strings" + "testing" +) + +func f(v float64) *float64 { return &v } + +func TestValidateJudgeMetrics(t *testing.T) { + cases := []struct { + name string + metrics []MetricConfig + wantErrs []string + wantOK bool + }{ + {name: "absent registry is valid legacy configuration", metrics: nil, wantOK: true}, + { + name: "two well-formed slots", + metrics: []MetricConfig{{ID: "correctness", Required: true}, {ID: "completeness", MinScore: f(0.7)}}, + wantOK: true, + }, + { + name: "a zero threshold is distinguishable from an absent one and stays valid", + metrics: []MetricConfig{{ID: "correctness", MinScore: f(0)}}, + wantOK: true, + }, + { + name: "upper bound is inclusive", + metrics: []MetricConfig{{ID: "correctness", MinScore: f(1)}}, + wantOK: true, + }, + { + name: "threshold above the score range is rejected", + metrics: []MetricConfig{{ID: "correctness", MinScore: f(1.1)}}, + wantErrs: []string{"min_score must be between 0.0 and 1.0"}, + }, + { + name: "negative threshold is rejected", + metrics: []MetricConfig{{ID: "correctness", MinScore: f(-0.5)}}, + wantErrs: []string{"min_score must be between 0.0 and 1.0"}, + }, + { + name: "duplicate slot ids are rejected", + metrics: []MetricConfig{{ID: "correctness"}, {ID: "correctness"}}, + wantErrs: []string{`metrics[1].id "correctness" is duplicated`}, + }, + { + name: "an unsafe identifier is rejected", + metrics: []MetricConfig{{ID: "Correctness!"}}, + wantErrs: []string{"id must be a safe lowercase identifier"}, + }, + { + name: "an empty identifier is rejected", + metrics: []MetricConfig{{ID: ""}}, + wantErrs: []string{"id must be a safe lowercase identifier"}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + errs := validateJudgeMetrics("judge", tc.metrics) + if tc.wantOK { + if len(errs) != 0 { + t.Fatalf("want valid, got %v", errs) + } + return + } + if len(errs) != len(tc.wantErrs) { + t.Fatalf("errors = %v, want %v", errs, tc.wantErrs) + } + for i, want := range tc.wantErrs { + if !strings.Contains(errs[i], want) { + t.Fatalf("error %d = %q, want substring %q", i, errs[i], want) + } + } + }) + } +} + +func TestMetricRegistryKeepsAnAbsentThresholdDistinctFromZero(t *testing.T) { + // A nil MinScore means "no threshold decision configured", which the quality + // derivation reports as NOT_CONFIGURED rather than UNKNOWN; a zero threshold is + // a real requirement. The two must not collapse during configuration loading. + absent := MetricConfig{ID: "correctness"} + zero := MetricConfig{ID: "completeness", MinScore: f(0)} + if absent.MinScore != nil { + t.Fatalf("unset threshold must stay nil, got %v", *absent.MinScore) + } + if zero.MinScore == nil || *zero.MinScore != 0 { + t.Fatal("an explicit zero threshold was lost") + } +} diff --git a/internal/config/schema.go b/internal/config/schema.go index c9475a63..2b679852 100644 --- a/internal/config/schema.go +++ b/internal/config/schema.go @@ -200,6 +200,23 @@ type JudgeConfig struct { TimeoutSeconds *int `json:"timeout_seconds,omitempty" yaml:"timeout_seconds,omitempty"` Success []Rule `json:"success,omitempty" yaml:"success,omitempty"` Failure []Rule `json:"failure,omitempty" yaml:"failure,omitempty"` + // Metrics declares the planned named-metric slots for this judge. The list is + // the registry: one slot per entry, fixed before execution, and the coverage + // denominator never moves. Omitted means no named metrics for this judge. + Metrics []MetricConfig `json:"metrics,omitempty" yaml:"metrics,omitempty"` +} + +// MetricConfig is one planned named-metric registry slot. +type MetricConfig struct { + // ID is a stable configuration identifier, never a display label produced by a judge. + ID string `json:"id" yaml:"id"` + // Required marks the metric as part of the declared scoring requirements. + Required bool `json:"required,omitempty" yaml:"required,omitempty"` + // MinScore is an optional quality threshold. Nil means no threshold decision is + // configured for this metric, which is distinct from a threshold of zero. + MinScore *float64 `json:"min_score,omitempty" yaml:"min_score,omitempty"` + // EvidenceRequires makes non-empty evidence mandatory for an AVAILABLE score. + EvidenceRequired bool `json:"evidence_required,omitempty" yaml:"evidence_required,omitempty"` } // JudgeContextConfig controls how agent_judge receives evaluation materials. diff --git a/internal/config/validator.go b/internal/config/validator.go index f32f8bbc..1fd310db 100644 --- a/internal/config/validator.go +++ b/internal/config/validator.go @@ -277,6 +277,7 @@ func validateJudgeTypeAndLocalFields(judge JudgeConfig) []string { errs = append(errs, validateJudgeEngine(judge)...) errs = append(errs, validatePassThreshold(judge.PassThreshold)...) + errs = append(errs, validateJudgeMetrics("judge.metrics", judge.Metrics)...) errs = append(errs, validateJudgeContext(judge.Context)...) if judge.TimeoutSeconds != nil && *judge.TimeoutSeconds < 0 { @@ -632,6 +633,33 @@ func validateJudgeList(members *[]JudgeConfig) []string { for _, detail := range validateJudgeTypeAndRequiredFields(member) { errs = append(errs, fmt.Sprintf("judges[%d]: %s", i, detail)) } + for _, detail := range validateJudgeMetrics("metrics", member.Metrics) { + errs = append(errs, fmt.Sprintf("judges[%d].%s", i, detail)) + } + } + return errs +} + +// validateJudgeMetrics checks the planned named-metric registry. Ids are stable +// configuration identifiers, must be unique, and a configured threshold has to sit +// inside the declared [0,1] score range. +func validateJudgeMetrics(path string, metrics []MetricConfig) []string { + if len(metrics) == 0 { + return nil + } + var errs []string + seen := make(map[string]bool, len(metrics)) + for i, metric := range metrics { + if !judgeIDPattern.MatchString(metric.ID) { + errs = append(errs, fmt.Sprintf("%s.metrics[%d].id must be a safe lowercase identifier", path, i)) + } + if seen[metric.ID] { + errs = append(errs, fmt.Sprintf("%s.metrics[%d].id %q is duplicated", path, i, metric.ID)) + } + seen[metric.ID] = true + if metric.MinScore != nil && (*metric.MinScore < 0.0 || *metric.MinScore > 1.0) { + errs = append(errs, fmt.Sprintf("%s.metrics[%d].min_score must be between 0.0 and 1.0", path, i)) + } } return errs } diff --git a/internal/evaluator/cancellation.go b/internal/evaluator/cancellation.go new file mode 100644 index 00000000..a1ed10b1 --- /dev/null +++ b/internal/evaluator/cancellation.go @@ -0,0 +1,78 @@ +package evaluator + +import ( + "context" + "sync" +) + +// Execution-provenance context values used to classify why a judge stopped. +// They exist so a cause is never reconstructed from an error string, and so an +// unattributable cancellation stays "unknown" instead of being guessed as +// user-initiated. + +type signalObserverKey struct{} +type deadlineSourceKey struct{} + +// SignalObserver records whether this process actually observed an interrupt. +// The flag is set by the signal handler path, never inferred afterwards. +type SignalObserver struct { + mu sync.Mutex + fired bool +} + +// MarkSignalObserved is called from the interrupt delivery path. +func (o *SignalObserver) MarkSignalObserved() { + if o == nil { + return + } + o.mu.Lock() + defer o.mu.Unlock() + o.fired = true +} + +// Observed reports whether a signal was seen. A false result with no observer +// attached means the source is unknown, not that no signal occurred. +func (o *SignalObserver) Observed() bool { + if o == nil { + return false + } + o.mu.Lock() + defer o.mu.Unlock() + return o.fired +} + +// ContextWithSignalObserver attaches the observer to ctx for downstream +// classification. +func ContextWithSignalObserver(ctx context.Context, observer *SignalObserver) context.Context { + if observer == nil { + return ctx + } + return context.WithValue(ctx, signalObserverKey{}, observer) +} + +// SignalObserved reports whether a process interrupt was observed for this ctx. +func SignalObserved(ctx context.Context) bool { + if ctx == nil { + return false + } + observer, ok := ctx.Value(signalObserverKey{}).(*SignalObserver) + return ok && observer.Observed() +} + +// ContextWithDeadlineSource names the configuration knob that armed the deadline +// carried by this context. An empty source stays unattributed downstream. +func ContextWithDeadlineSource(ctx context.Context, source string) context.Context { + if source == "" { + return ctx + } + return context.WithValue(ctx, deadlineSourceKey{}, source) +} + +// DeadlineSource returns the recorded knob name, or "" when unknown. +func DeadlineSource(ctx context.Context) string { + if ctx == nil { + return "" + } + source, _ := ctx.Value(deadlineSourceKey{}).(string) + return source +} diff --git a/internal/evaluator/evaluator.go b/internal/evaluator/evaluator.go index ef058cc3..ae4d918b 100644 --- a/internal/evaluator/evaluator.go +++ b/internal/evaluator/evaluator.go @@ -355,6 +355,9 @@ func (e *defaultEvaluator) executeCase(ctx context.Context, caseCfg *config.Case var result EvalResult for attempt := 1; attempt <= maxAttempts; attempt++ { attemptCtx, cancel, timeoutSource, timeoutSec := withCaseTimeout(ctx, e.evalCfg.Cases.Defaults.TimeoutSeconds, caseCfg.Constraints.TimeoutSeconds) + // Name the armed knob so a deadline that fires through the case context can + // report its own source instead of being attributed to a guess. + attemptCtx = ContextWithDeadlineSource(attemptCtx, timeoutSource) result = e.executeCaseOnce(attemptCtx, caseCfg, configName, overrideRT, overrideAgent) // Decide whether to annotate *before* cancel() so (a) cancel doesn't // overwrite Err with Canceled and erase the signal, and (b) a diff --git a/internal/evaluator/metrics_wiring_test.go b/internal/evaluator/metrics_wiring_test.go new file mode 100644 index 00000000..043680a7 --- /dev/null +++ b/internal/evaluator/metrics_wiring_test.go @@ -0,0 +1,407 @@ +package evaluator + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + "time" + + "github.com/alibaba/skill-up/internal/config" + "github.com/alibaba/skill-up/internal/judge" +) + +func timeout(n int) *int { return &n } + +func TestMemberStopAttributesSources(t *testing.T) { + expiredCaseDeadlineIntoMember := func(source string) func(*testing.T) (context.Context, context.Context) { + return func(t *testing.T) (context.Context, context.Context) { + parent := expiredDeadlineContext(source) + member, cancel := context.WithCancel(parent) + t.Cleanup(cancel) + return parent, member + } + } + cases := []struct { + name string + build func(t *testing.T) (parent, member context.Context) + memberTimeout bool + want judge.Stop + }{ + { + name: "no cancellation", + build: func(*testing.T) (context.Context, context.Context) { return context.Background(), context.Background() }, + memberTimeout: false, + want: judge.Stop{Cause: judge.CauseNone}, + }, + { + name: "member-level deadline names its own knob", + build: func(t *testing.T) (context.Context, context.Context) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + t.Cleanup(cancel) + <-ctx.Done() + return context.Background(), ctx + }, + memberTimeout: true, + want: judge.Stop{Cause: judge.CauseDeadlineExpiry, DeadlineSource: "judge.timeout_seconds", Detail: "context deadline exceeded"}, + }, + { + name: "a case deadline firing through the member is not called a judge timeout", + build: expiredCaseDeadlineIntoMember("case.constraints.timeout_seconds"), + memberTimeout: true, + want: judge.Stop{Cause: judge.CauseDeadlineExpiry, DeadlineSource: "case.constraints.timeout_seconds", Detail: "context deadline exceeded"}, + }, + { + name: "a parent deadline with an unattributed origin stays unknown", + build: func(t *testing.T) (context.Context, context.Context) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + t.Cleanup(cancel) + <-ctx.Done() + return ctx, ctx + }, + memberTimeout: true, + want: judge.Stop{Cause: judge.CauseDeadlineExpiry, DeadlineSource: judge.CancelUnspecified, Detail: "context deadline exceeded"}, + }, + { + name: "cancellation without an observed signal stays unknown", + build: func(*testing.T) (context.Context, context.Context) { return canceledContext(false), canceledContext(false) }, + want: judge.Stop{Cause: judge.CauseCancellation, CancelSource: judge.CancelUnspecified, Detail: "context canceled"}, + }, + { + name: "cancellation with an observed signal is attributed", + build: func(*testing.T) (context.Context, context.Context) { return canceledContext(true), canceledContext(true) }, + want: judge.Stop{Cause: judge.CauseCancellation, CancelSource: judge.CancelOSSignal, Detail: "context canceled"}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + parent, member := tc.build(t) + got := memberStop(parent, member, tc.memberTimeout) + if got != tc.want { + t.Fatalf("memberStop = %+v, want %+v", got, tc.want) + } + }) + } +} + +func expiredDeadlineContext(source string) context.Context { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + defer cancel() + <-ctx.Done() + return ContextWithDeadlineSource(ctx, source) +} + +// canceledContext returns a context whose Err is already context.Canceled. cancel() +// sets Err synchronously, so there is nothing to wait for here. +func canceledContext(signal bool) context.Context { + observer := &SignalObserver{} + if signal { + observer.MarkSignalObserved() + } + ctx, cancel := context.WithCancel(ContextWithSignalObserver(context.Background(), observer)) + cancel() + return ctx +} + +func TestRecordMemberContextErrorKeepsAcceptedResults(t *testing.T) { + cases := []struct { + name string + accepted bool + start judge.Status + wantStatus judge.Status + wantErr bool + }{ + {"accepted PASS survives a later cancellation", true, judge.StatusPass, judge.StatusPass, true}, + {"accepted FAIL survives a later cancellation", true, judge.StatusFail, judge.StatusFail, true}, + {"unaccepted PASS is escalated to ERROR", false, judge.StatusPass, judge.StatusError, true}, + {"an existing ERROR is left alone", false, judge.StatusError, judge.StatusError, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + outcome := JudgeOutcome{Status: tc.start} + err := recordMemberContextError(ctx, &outcome, tc.accepted) + if outcome.Status != tc.wantStatus { + t.Fatalf("status = %s, want %s", outcome.Status, tc.wantStatus) + } + if gotErr := err != nil; gotErr != tc.wantErr { + t.Fatalf("returned error = %v, want err %v", err, tc.wantErr) + } + if tc.accepted && outcome.Error != "" { + t.Fatalf("an accepted member must not gain an error string, got %q", outcome.Error) + } + }) + } +} + +func TestCompletionForContract(t *testing.T) { + env := func(states ...judge.Availability) *judge.MetricsEnvelope { + ids := []string{"correctness", "completeness", "instruction_following"} + out := &judge.MetricsEnvelope{} + for i, s := range states { + out.Metrics = append(out.Metrics, judge.MetricResult{MetricID: ids[i], Availability: s}) + } + return out + } + cases := []struct { + name string + env *judge.MetricsEnvelope + want judge.Completion + }{ + {"all available", env(judge.Available, judge.Available, judge.Available), judge.CompletionFinished}, + {"optional omitted is incomplete", env(judge.Available, judge.Missing, judge.Available), judge.CompletionIncomplete}, + {"required SKIP is incomplete", env(judge.Available, judge.Declined, judge.Available), judge.CompletionIncomplete}, + {"cancelled slot is incomplete", env(judge.Available, judge.NotProduced, judge.Available), judge.CompletionIncomplete}, + {"an errored metric fails the contract", env(judge.Available, judge.Errored, judge.Available), judge.CompletionFailed}, + {"an errored metric outranks an omission", env(judge.Missing, judge.Errored, judge.Available), judge.CompletionFailed}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := completionForContract(tc.env); got != tc.want { + t.Fatalf("completion = %s, want %s", got, tc.want) + } + }) + } +} + +func TestPreventedMetricsKeepsEveryPlannedSlot(t *testing.T) { + registry := []config.MetricConfig{{ID: "correctness", Required: true}, {ID: "completeness"}} + env := preventedMetrics("j1", registry) + if env == nil { + t.Fatal("expected an envelope for a configured registry") + } + if len(env.Metrics) != 2 || env.Coverage.Planned != 2 || env.Coverage.NotProduced != 2 { + t.Fatalf("envelope = %+v", env) + } + for _, metric := range env.Metrics { + if metric.Availability != judge.NotProduced || metric.Value != nil { + t.Fatalf("slot %s = %+v, want NOT_PRODUCED with no value", metric.MetricID, metric) + } + } + if env.Quality != judge.QualityNotAssessed { + t.Fatalf("quality = %s, want NOT_ASSESSED without thresholds", env.Quality) + } +} + +func TestPreventedMetricsWithThresholdsIsUnknown(t *testing.T) { + registry := []config.MetricConfig{{ID: "correctness", MinScore: ptr(0.7)}} + env := preventedMetrics("j1", registry) + if env.Quality != judge.QualityUnknown { + t.Fatalf("quality = %s, want UNKNOWN when a thresholded metric was never produced", env.Quality) + } +} + +func TestNoRegistryMeansNoEnvelope(t *testing.T) { + if got := preventedMetrics("j1", nil); got != nil { + t.Fatalf("legacy judge gained an envelope: %+v", got) + } +} + +func TestStopDetailKeepsLegacyGateReason(t *testing.T) { + cases := []struct { + stop judge.Stop + want string + }{ + {judge.Stop{Cause: judge.CauseExpectGate}, "gate_failed"}, + {judge.Stop{Cause: judge.CauseCancellation}, "cancelled before invocation"}, + {judge.Stop{Cause: judge.CauseDeadlineExpiry}, "case deadline expired"}, + {judge.Stop{Cause: judge.CauseNone}, ""}, + {judge.Stop{Cause: judge.CauseExecutionError, Detail: "boom"}, "boom"}, + } + for _, tc := range cases { + if got := stopDetail(tc.stop); got != tc.want { + t.Fatalf("stopDetail(%s) = %q, want %q", tc.stop.Cause, got, tc.want) + } + } +} + +func TestSkippedJudgesRetainSlotsPerCause(t *testing.T) { + members := []config.JudgeConfig{ + {ID: "a", Type: "agent_judge", Metrics: []config.MetricConfig{{ID: "correctness", Required: true}}}, + {ID: "b", Type: "script", Metrics: []config.MetricConfig{{ID: "completeness"}}}, + } + cases := []struct { + reason string + want judge.Cause + }{ + {"gate_failed", judge.CauseExpectGate}, + {"agent_execution_failed", judge.CauseExecutionError}, + {"invalid_judge_config", judge.CauseContractError}, + {"session_resumption_unsupported", judge.CauseContractError}, + } + for _, tc := range cases { + t.Run(tc.reason, func(t *testing.T) { + result := &EvalResult{} + recordSkippedJudges(result, config.JudgePlan{Judges: members, Multi: true}, tc.reason) + if len(result.JudgeResults) != 2 { + t.Fatalf("outcomes = %d, want 2", len(result.JudgeResults)) + } + for _, outcome := range result.JudgeResults { + if outcome.Status != judge.StatusSkip || outcome.SkipReason != tc.reason { + t.Fatalf("compat fields changed: %+v", outcome) + } + if outcome.Execution == nil || outcome.Execution.Completion != judge.CompletionInterrupted || + outcome.Execution.Cause != tc.want || outcome.Execution.Stage != judge.StageNotInvoked { + t.Fatalf("execution = %+v, want interrupted/not-invoked/%s", outcome.Execution, tc.want) + } + if outcome.Metrics == nil || outcome.Metrics.Coverage.Planned != 1 || + outcome.Metrics.Metrics[0].Availability != judge.NotProduced { + t.Fatalf("metrics = %+v, want one NOT_PRODUCED slot", outcome.Metrics) + } + } + }) + } +} + +func TestSkippedJudgesAreNoopForSingleJudgePlans(t *testing.T) { + result := &EvalResult{} + recordSkippedJudges(result, config.JudgePlan{Judges: []config.JudgeConfig{{ID: "a"}}}, "gate_failed") + if len(result.JudgeResults) != 0 { + t.Fatalf("single-judge plan gained member outcomes: %+v", result.JudgeResults) + } +} + +// TestMetadataSurvivesNilResult pins the rule that execution facts and planned +// slots are not stored only inside judge.Result. +func TestMetadataSurvivesNilResult(t *testing.T) { + member := config.JudgeConfig{ID: "slow", Type: "script", Metrics: []config.MetricConfig{{ID: "correctness", Required: true}}} + stop := judge.Stop{Cause: judge.CauseCancellation, CancelSource: judge.CancelUnspecified, Detail: "context canceled"} + graded := EvalResult{Status: judge.StatusError, Grading: nil, Error: context.Canceled} + outcome := JudgeOutcome{ + ID: member.ID, Type: member.Type, Status: graded.Status, Result: graded.Grading, + Metrics: gradedMetrics(member, graded, stop), + Execution: memberExecution(graded, stop, false), + } + if outcome.Result != nil { + t.Fatal("no Result may be fabricated to carry metadata") + } + if outcome.Execution.Completion != judge.CompletionInterrupted || outcome.Execution.Stage != judge.StageStarted { + t.Fatalf("execution = %+v", outcome.Execution) + } + if outcome.Metrics == nil || outcome.Metrics.Metrics[0].Availability != judge.NotProduced { + t.Fatalf("metrics = %+v", outcome.Metrics) + } + encoded, err := json.Marshal(outcome) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{`"completion_state":"INTERRUPTED"`, `"termination_cause":"cancellation"`, `"NOT_PRODUCED"`, `"correctness"`} { + if !strings.Contains(string(encoded), want) { + t.Fatalf("persisted JSON missing %s: %s", want, encoded) + } + } + var decoded JudgeOutcome + if err := json.Unmarshal(encoded, &decoded); err != nil { + t.Fatal(err) + } + if decoded.Execution.Cause != judge.CauseCancellation || decoded.Execution.CancelSource != judge.CancelUnspecified { + t.Fatalf("round-trip lost cause/source: %+v", decoded.Execution) + } + if decoded.Metrics.Quality != judge.QualityNotAssessed || decoded.Metrics.Coverage.Planned != 1 { + t.Fatalf("round-trip lost derived state: %+v", decoded.Metrics) + } +} + +func TestAcceptedResultKeepsScoresAfterLaterStop(t *testing.T) { + member := config.JudgeConfig{ID: "j1", Type: "agent_judge", Metrics: []config.MetricConfig{{ID: "correctness"}, {ID: "completeness"}}} + env := judge.MaterializeSlots("j1", judge.SlotsFromConfig(member.Metrics), []judge.EmittedMetric{ + {MetricID: "correctness", Availability: "AVAILABLE", Value: ptr(0.8)}, + }, judge.ProblemNone, false) + graded := EvalResult{Status: judge.StatusPass, Grading: &judge.Result{Status: judge.StatusPass, Metrics: env}} + stop := judge.Stop{Cause: judge.CauseCancellation, CancelSource: judge.CancelOSSignal} + outcome := JudgeOutcome{Status: graded.Status, Result: graded.Grading} + execution := memberExecution(graded, stop, true) + if execution.Completion != judge.CompletionIncomplete { + t.Fatalf("completion = %s, want INCOMPLETE because one required slot was never reached", execution.Completion) + } + if execution.Cause != judge.CauseCancellation || execution.CancelSource != judge.CancelOSSignal { + t.Fatalf("the later stop must stay visible: %+v", execution) + } + if outcome.Result.Metrics.Metrics[0].Availability != judge.Available { + t.Fatal("an accepted score was downgraded") + } + if got := recordMemberContextError(canceledPlain(), &outcome, true); got == nil { + t.Fatal("expected the cancellation error to still be reported") + } + if outcome.Status != judge.StatusPass { + t.Fatalf("accepted verdict was rewritten: %s", outcome.Status) + } +} + +func TestTimeoutRemainsVisibleOnAcceptedResult(t *testing.T) { + member := config.JudgeConfig{ID: "j1", Type: "agent_judge", Metrics: []config.MetricConfig{{ID: "correctness"}}} + env := judge.MaterializeSlots("j1", judge.SlotsFromConfig(member.Metrics), []judge.EmittedMetric{ + {MetricID: "correctness", Availability: "AVAILABLE", Value: ptr(0.9)}, + }, judge.ProblemNone, false) + graded := EvalResult{Status: judge.StatusPass, Grading: &judge.Result{Status: judge.StatusPass, Metrics: env}} + stop := judge.Stop{Cause: judge.CauseDeadlineExpiry, DeadlineSource: "judge.timeout_seconds", Detail: "context deadline exceeded"} + execution := memberExecution(graded, stop, true) + if execution.Timeout == nil || !execution.Timeout.Occurred || execution.Timeout.Source != "judge.timeout_seconds" { + t.Fatalf("timeout metadata missing: %+v", execution) + } + if execution.Completion != judge.CompletionFinished { + t.Fatalf("completion = %s, want COMPLETED for a fully scored accepted result", execution.Completion) + } +} + +func canceledPlain() context.Context { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + return ctx +} + +func TestSignalObserverReportsOnlyObservedSignals(t *testing.T) { + observer := &SignalObserver{} + ctx := ContextWithSignalObserver(context.Background(), observer) + if SignalObserved(ctx) { + t.Fatal("an un-triggered observer must not claim a signal") + } + observer.MarkSignalObserved() + if !SignalObserved(ctx) { + t.Fatal("an observed signal must be reported") + } + if SignalObserved(context.Background()) { + t.Fatal("a context without an observer reports no signal") + } + var nilObserver *SignalObserver + if nilObserver.Observed() { + t.Fatal("nil observer must not report a signal") + } + nilObserver.MarkSignalObserved() +} + +func TestGroupFoldOverMemberOutcomes(t *testing.T) { + cases := []struct { + name string + states []judge.Completion + want judge.Completion + wantErr bool + }{ + {"completed plus pending cancelled", []judge.Completion{judge.CompletionFinished, judge.CompletionInterrupted}, judge.CompletionInterrupted, false}, + {"failed plus pending cancelled keeps FAILED", []judge.Completion{judge.CompletionFailed, judge.CompletionInterrupted}, judge.CompletionFailed, false}, + {"incomplete plus pending cancelled keeps INCOMPLETE", []judge.Completion{judge.CompletionIncomplete, judge.CompletionInterrupted}, judge.CompletionIncomplete, false}, + {"empty group is refused", nil, "", true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + completions := make([]judge.Completion, 0, len(tc.states)) + for _, state := range tc.states { + completions = append(completions, state) + } + got, err := judge.FoldGroup(completions) + if tc.wantErr { + if !errors.Is(err, judge.ErrEmptyGroup) { + t.Fatalf("want ErrEmptyGroup, got %v", err) + } + return + } + if err != nil || got != tc.want { + t.Fatalf("fold = %s (%v), want %s", got, err, tc.want) + } + }) + } +} + +func ptr(v float64) *float64 { return &v } diff --git a/internal/evaluator/multi_judge.go b/internal/evaluator/multi_judge.go index c9b16031..52ed63de 100644 --- a/internal/evaluator/multi_judge.go +++ b/internal/evaluator/multi_judge.go @@ -36,9 +36,13 @@ func (e *defaultEvaluator) runMultipleJudges( var causes []error for i, member := range plan.Judges { if ctx.Err() != nil { + stop := judge.ClassifyStop(ctx.Err(), DeadlineSource(ctx), SignalObserved(ctx)) for _, pending := range plan.Judges[i:] { result.JudgeResults = append(result.JudgeResults, JudgeOutcome{ - ID: pending.ID, Type: pending.Type, Status: judge.StatusSkip, SkipReason: "case deadline or cancellation", + ID: pending.ID, Type: pending.Type, Status: judge.StatusSkip, + SkipReason: stopDetail(stop), + Execution: interruptedExecution(judge.StageNotInvoked, stop), + Metrics: preventedMetrics(pending.ID, pending.Metrics), }) } break @@ -47,8 +51,9 @@ func (e *defaultEvaluator) runMultipleJudges( outcome := JudgeOutcome{ID: member.ID, Type: member.Type, Artifacts: "judge/" + member.ID + "/run"} e.prepareOutputDir(ctx, configName, caseCfg.ID, outcome.Artifacts) memberCtx := ctx + memberTimeout := member.TimeoutSeconds != nil && *member.TimeoutSeconds > 0 cancel := func() {} - if member.TimeoutSeconds != nil && *member.TimeoutSeconds > 0 { + if memberTimeout { memberCtx, cancel = context.WithTimeout(ctx, time.Duration(*member.TimeoutSeconds)*time.Second) } memberResult := &EvalResult{SessionResult: result.SessionResult, CaseID: result.CaseID, CaseName: result.CaseName} @@ -62,7 +67,15 @@ func (e *defaultEvaluator) runMultipleJudges( if graded.Status == judge.StatusError && graded.Error != nil { causes = append(causes, graded.Error) } - if memberErr := recordMemberContextError(memberCtx, &outcome); memberErr != nil { + // An outcome that reached a final validated state before the stop is + // accepted and retained: a later cancellation or deadline never rewrites + // its verdict, and its scores are not downgraded because evaluation then + // stopped. Only members without such an outcome take the interrupt verdict. + accepted := graded.Grading != nil && graded.Status != judge.StatusError + stop := memberStop(ctx, memberCtx, memberTimeout) + outcome.Metrics = gradedMetrics(member, graded, stop) + outcome.Execution = memberExecution(graded, stop, accepted) + if memberErr := recordMemberContextError(memberCtx, &outcome, accepted); memberErr != nil { causes = append(causes, memberErr) } cancel() @@ -77,15 +90,148 @@ func (e *defaultEvaluator) runMultipleJudges( return *result } -func recordMemberContextError(ctx context.Context, outcome *JudgeOutcome) error { +func recordMemberContextError(ctx context.Context, outcome *JudgeOutcome, accepted bool) error { err := ctx.Err() - if err != nil && outcome.Status != judge.StatusError { + if err == nil { + return nil + } + // An accepted outcome keeps its own verdict; the stop is still recorded so the + // timeout or cancellation stays visible in the execution metadata. + if !accepted && outcome.Status != judge.StatusError { outcome.Status = judge.StatusError outcome.Error = err.Error() } return err } +// memberStop resolves why a member stopped. A member-level deadline is only +// attributed to judge.timeout_seconds when the parent context had not itself +// expired, so a case deadline firing through the member is never mislabeled. +func memberStop(parentCtx, memberCtx context.Context, memberTimeout bool) judge.Stop { + if memberCtx.Err() == nil { + return judge.Stop{Cause: judge.CauseNone} + } + source := DeadlineSource(parentCtx) + if memberTimeout && source == "" && parentCtx.Err() == nil && errors.Is(memberCtx.Err(), context.DeadlineExceeded) { + source = "judge.timeout_seconds" + } + return judge.ClassifyStop(memberCtx.Err(), source, SignalObserved(parentCtx)) +} + +// stopDetail renders a human-readable stop note. The machine-readable cause and +// source live in ExecutionMetadata and are never re-derived from this text. +func stopDetail(stop judge.Stop) string { + switch stop.Cause { + case judge.CauseDeadlineExpiry: + return "case deadline expired" + case judge.CauseCancellation: + return "cancelled before invocation" + case judge.CauseExpectGate: + return "gate_failed" + case judge.CauseNone: + return "" + default: + return stop.Detail + } +} + +func interruptedExecution(stage judge.Stage, stop judge.Stop) *judge.ExecutionMetadata { + return &judge.ExecutionMetadata{ + Stage: stage, + Completion: judge.CompletionInterrupted, + Cause: stop.Cause, + DeadlineSource: stop.DeadlineSource, + CancelSource: stop.CancelSource, + Detail: stop.Detail, + } +} + +// preventedMetrics materializes every planned slot as NOT_PRODUCED for a member +// whose evaluation never reached a final outcome. The planned denominator is the +// registry, so a member that never started still reports each slot. +func preventedMetrics(judgeID string, registry []config.MetricConfig) *judge.MetricsEnvelope { + slots := judge.SlotsFromConfig(registry) + if len(slots) == 0 { + return nil + } + return judge.MaterializeSlots(judgeID, slots, nil, judge.ProblemNone, true) +} + +// gradedMetrics builds the envelope from what a judge produced. A stop that +// happened after a usable score never discards that score; slots that were never +// reached stay NOT_PRODUCED rather than becoming zero. +func gradedMetrics(member config.JudgeConfig, graded EvalResult, stop judge.Stop) *judge.MetricsEnvelope { + slots := judge.SlotsFromConfig(member.Metrics) + if len(slots) == 0 { + return nil + } + if graded.Grading != nil && graded.Grading.Metrics != nil { + return graded.Grading.Metrics + } + prevented := stop.Cause != judge.CauseNone && graded.Grading == nil + return judge.MaterializeSlots(member.ID, slots, nil, judge.ProblemNone, prevented) +} + +// memberExecution describes one member's execution facts. Accepted results report +// their own completion; a stopped member without a final outcome is INTERRUPTED +// whether or not it had started, with the stage recording which of the two it was. +func memberExecution(graded EvalResult, stop judge.Stop, accepted bool) *judge.ExecutionMetadata { + if accepted { + completion := judge.CompletionFinished + if graded.Grading.Metrics != nil { + completion = completionForContract(graded.Grading.Metrics) + } + meta := &judge.ExecutionMetadata{ + Stage: judge.StageFinished, + Completion: completion, + Cause: judge.CauseNone, + } + if stop.Cause != judge.CauseNone { + meta.Cause, meta.DeadlineSource, meta.CancelSource, meta.Detail = stop.Cause, stop.DeadlineSource, stop.CancelSource, stop.Detail + if stop.Cause == judge.CauseDeadlineExpiry { + meta.Timeout = &judge.TimeoutMetadata{Occurred: true, Source: stop.DeadlineSource, Recovered: graded.Grading.Recovery != nil} + } + } + return meta + } + stage := judge.StageStarted + if graded.Grading == nil && graded.Status == judge.StatusSkip { + stage = judge.StageNotInvoked + } + execution := interruptedExecution(stage, stop) + if execution.Cause == judge.CauseNone { + execution.Cause = judge.CauseExecutionError + if graded.Error != nil { + execution.Detail = graded.Error.Error() + } + if graded.Grading != nil { + execution.Completion = judge.CompletionFailed + } + } + return execution +} + +// completionForContract maps the materialized slots onto the completion states: +// an errored slot failed the contract, an unfulfilled required slot or an +// unevaluable threshold leaves the outcome incomplete. +func completionForContract(env *judge.MetricsEnvelope) judge.Completion { + requiredMissing := false + for _, metric := range env.Metrics { + if metric.Availability == judge.Errored { + return judge.CompletionFailed + } + } + for _, metric := range env.Metrics { + if metric.Availability != judge.Available { + requiredMissing = true + } + } + if requiredMissing { + return judge.CompletionIncomplete + } + return judge.CompletionFinished +} + func multiJudgeError(ctx context.Context, outcomes []JudgeOutcome, causes []error) error { if ctx.Err() != nil { causes = append(causes, ctx.Err()) @@ -121,13 +267,32 @@ func summarizeJudgeErrors(outcomes []JudgeOutcome) string { return "judges failed to complete: " + strings.Join(details, "; ") } +// recordSkippedJudges materializes every planned member as skipped. Each member +// keeps execution metadata and its registry-derived metric slots even though no +// judge ran, so a gate or execution failure never loses that information. The +// reason strings are unchanged for compatibility; the machine-readable cause is +// the new source of truth. func recordSkippedJudges(result *EvalResult, plan config.JudgePlan, reason string) { if !plan.Multi { return } + cause := causeForReason(reason) for _, member := range plan.Judges { result.JudgeResults = append(result.JudgeResults, JudgeOutcome{ ID: member.ID, Type: member.Type, Status: judge.StatusSkip, SkipReason: reason, + Execution: interruptedExecution(judge.StageNotInvoked, judge.Stop{Cause: cause, Detail: reason}), + Metrics: preventedMetrics(member.ID, member.Metrics), }) } } + +func causeForReason(reason string) judge.Cause { + switch reason { + case "gate_failed": + return judge.CauseExpectGate + case "session_resumption_unsupported", "invalid_judge_config": + return judge.CauseContractError + default: + return judge.CauseExecutionError + } +} diff --git a/internal/judge/README.md b/internal/judge/README.md index 3327be58..32e32845 100644 --- a/internal/judge/README.md +++ b/internal/judge/README.md @@ -276,3 +276,61 @@ go test ./internal/judge/ -run TestAgentJudge -v | `agent_judge_test.go` | 11 | Mock LLM client + threshold edge cases + JSON parsing | | `factory_test.go` | 8 | Factory construction + configuration merge | | `e2e_test.go` | 12 | End-to-end pipeline integration tests | +| `metrics_test.go` | 8 tables | Group folding, quality derivation, availability states, payload-vs-slot validation, stop classification, definition compatibility | + +## Named metrics (#317, first slice: validated scoring and persistence) + +Configured per judge as a planned registry; omitted means the judge keeps today's +shape exactly: + +```yaml +judge: + type: agent_judge + metrics: + - id: correctness # stable config identifier, never a judge-generated label + required: true + evidence_required: true + - id: completeness + min_score: 0.7 # absent means no threshold decision, which is not a threshold of 0 +``` + +Four independent questions, four fields, disjoint value sets: + +| Field | Values | Question | +|---|---|---| +| `execution_stage` | `NOT_INVOKED` / `STARTED` / `COMPLETED` | did the member actually run | +| `completion_state` | `COMPLETED` / `INCOMPLETE` / `FAILED` / `INTERRUPTED` | how far evaluation got | +| `termination_cause` | `none` / `cancellation` / `deadline_expiry` / `expect_gate` / `process_exit` / `execution_error` / `contract_error` / `judge_declined` | why it stopped | +| `availability` | `AVAILABLE` / `MISSING` / `ERROR` / `SKIP` / `NOT_PRODUCED` | did this slot produce a usable score | +| `metric_quality_state` | `MET` / `NOT_MET` / `UNKNOWN` / `NOT_CONFIGURED` / `NOT_ASSESSED` | does a quality decision exist | + +Rules that the implementation enforces: + +- There is no `NOT_RUN` state. A judge that never started and one interrupted + mid-flight are both `INTERRUPTED`; `execution_stage` is what distinguishes them, + and no start timestamp is invented when none was recorded. +- Group completion is derived, never assigned: `FAILED > INCOMPLETE > INTERRUPTED > + COMPLETED`, so `failed + pending-cancelled` stays `FAILED` while the cancellation + and the unfinished members remain visible on their own members. +- `metric_quality_state` is always derived from slots by `DeriveQuality` and never + from `completion_state`. Thresholds configured and all thresholded values present + give `MET`/`NOT_MET`; thresholds with any thresholded value missing give + `UNKNOWN`; valid scores with no thresholds give `NOT_CONFIGURED`; no valid scores + give `NOT_ASSESSED`. +- A timeout does not invalidate a trustworthy score. An accepted score stays + `AVAILABLE`, and the timeout, its source and any recovery stay recorded. + A score is never fabricated as zero, and an accepted result is never rewritten + by a cancellation that arrives later. +- Whole-payload defects (unparseable, top-level shape, duplicate JSON keys, unknown + top-level field, schema version, unbound or stale result, size limit) make every + slot `ERROR`. Slot-level defects poison only their own slot and independently + valid siblings stay usable in storage, reports and comparisons alike. +- Unknown ids are output diagnostics and never new planned slots; a duplicated id + invalidates its planned slot once. The coverage denominator is the registry count + and never moves. +- Execution metadata and planned slots live on `Outcome` and the case result, not + only inside `judge.Result`, so a gate short-circuit or a cancellation with no + judge result still reports them. +- Compatibility is untouched: `PASS/FAIL/SKIP/ERROR`, the #309 assertion and member + denominators with zero-credit skipped members, the existing script-timeout verdict + mapping, and the JSONL event schema all behave as before. diff --git a/internal/judge/compat.go b/internal/judge/compat.go new file mode 100644 index 00000000..14e77972 --- /dev/null +++ b/internal/judge/compat.go @@ -0,0 +1,67 @@ +package judge + +// Definition compatibility for named metrics (#317). Mandatory equality covers +// the scoring definition itself; provider, model and revision are provenance and +// produce an advisory only, because different identifiers can name the same +// underlying model and identical aliases can drift. +// +// The decision is pure logic so comparisons can consume it in a later bounded PR +// without re-deriving the rules. + +// ScoringDefinition is the comparison-relevant identity of one metric series. +type ScoringDefinition struct { + JudgeID string + MetricID string + // RubricHash covers exact rubric text, anchors and version. + RubricHash string + // ScaleHash covers the declared scale, direction and range. + ScaleHash string + // Provenance only: never part of mandatory equality. + Provider string + Model string + // ModelRevision is recorded only when a provider explicitly supplied a + // concrete version. An empty value means unknown and is never inferred + // from an alias. + ModelRevision string +} + +// Compatibility is the outcome of comparing two definitions. +type Compatibility struct { + Comparable bool + // Reasons blocks pairing: mismatched scoring definition or scale. + Reasons []string + // Advisories inform, never block. + Advisories []string +} + +// CompareScoringDefinitions reports whether two metric series may be paired. +func CompareScoringDefinitions(a, b ScoringDefinition) Compatibility { + out := Compatibility{Comparable: true} + if a.JudgeID != b.JudgeID { + out.Reasons = append(out.Reasons, "judge_id_mismatch") + } + if a.MetricID != b.MetricID { + out.Reasons = append(out.Reasons, "metric_id_mismatch") + } + if a.RubricHash != b.RubricHash { + out.Reasons = append(out.Reasons, "rubric_mismatch") + } + if a.ScaleHash != b.ScaleHash { + out.Reasons = append(out.Reasons, "scoring_scale_mismatch") + } + if len(out.Reasons) > 0 { + out.Comparable = false + } + if a.Provider != b.Provider { + out.Advisories = append(out.Advisories, "provider_differs") + } + if a.Model != b.Model { + out.Advisories = append(out.Advisories, "model_alias_differs") + } + if a.ModelRevision == "" || b.ModelRevision == "" { + out.Advisories = append(out.Advisories, "model_revision_unknown") + } else if a.ModelRevision != b.ModelRevision { + out.Advisories = append(out.Advisories, "model_revision_differs") + } + return out +} diff --git a/internal/judge/judge.go b/internal/judge/judge.go index 1588c4d9..c4f80854 100644 --- a/internal/judge/judge.go +++ b/internal/judge/judge.go @@ -148,6 +148,11 @@ type Result struct { // agent call returned an error. It is carried into report.json separately so // grading.json remains compatible with the Anthropic grading contract. Recovery *RecoveryMetadata `json:"-"` + + // Metrics carries the materialized named-metric slots (#317). It is additive: + // judges without a configured metrics registry leave this nil, and no legacy + // field changes meaning. + Metrics *MetricsEnvelope `json:"metrics,omitempty"` } // RecoveryMetadata describes a recovered judge result without exposing the @@ -158,6 +163,10 @@ type RecoveryMetadata struct { } // Outcome preserves one independent judge's result and diagnostic session. +// +// Execution and Metrics are additive #317 metadata. They exist even when Result +// is nil, so a judge that was prevented from running, interrupted, or stopped by +// the expect gate never loses its execution facts or its planned metric slots. type Outcome struct { ID string `json:"id"` Type string `json:"type"` @@ -170,6 +179,8 @@ type Outcome struct { Recovery *RecoveryMetadata `json:"recovery,omitempty"` Session *agent.SessionResult `json:"-"` Skills []SkillInfo `json:"judge_skills,omitempty"` + Execution *ExecutionMetadata `json:"execution,omitempty"` + Metrics *MetricsEnvelope `json:"metrics,omitempty"` } // ContextMetadata is report-facing metadata for agent_judge context diff --git a/internal/judge/metrics.go b/internal/judge/metrics.go new file mode 100644 index 00000000..aa36bc42 --- /dev/null +++ b/internal/judge/metrics.go @@ -0,0 +1,438 @@ +package judge + +import ( + "context" + "errors" + "math" + "sort" + "strings" + + "github.com/alibaba/skill-up/internal/config" +) + +// Named-metrics scoring contract for #317, first slice (validated scoring and +// persistence). Semantics follow the maintainer decision in +// https://github.com/alibaba/skill-up/issues/317#issuecomment-6437995472 . +// +// Report integration and cross-run comparison rendering land in later bounded +// PRs. + +// Completion describes how far a member's evaluation got. There is deliberately +// no NOT_RUN state: a member that never started and a member stopped mid-flight +// are both INTERRUPTED, and ExecutionStage is what distinguishes them. +type Completion string + +const ( + // CompletionFinished means a final validated outcome was obtained and the + // declared scoring requirements were satisfied. It is not a quality claim. + CompletionFinished Completion = "COMPLETED" + // CompletionIncomplete means a final structurally valid outcome was obtained + // but a required metric or configured threshold could not be evaluated + // because its score was unavailable. + CompletionIncomplete Completion = "INCOMPLETE" + // CompletionFailed means an execution or scoring-contract error prevented a + // trustworthy outcome, or an explicit metric error occurred. Independently + // validated sibling scores may still be usable. + CompletionFailed Completion = "FAILED" + // CompletionInterrupted means evaluation was prevented or stopped before a + // final scoring outcome was obtained. + CompletionInterrupted Completion = "INTERRUPTED" +) + +// Cause records why evaluation ended, always separate from Completion so a cause +// never has to be inferred from a verdict. +type Cause string + +const ( + CauseNone Cause = "none" + CauseCancellation Cause = "cancellation" + CauseDeadlineExpiry Cause = "deadline_expiry" + CauseExpectGate Cause = "expect_gate" + CauseProcessExit Cause = "process_exit" + CauseExecutionError Cause = "execution_error" + CauseContractError Cause = "contract_error" + CauseJudgeDeclined Cause = "judge_declined" +) + +// Stage is the program-recorded execution stage. INTERRUPTED members that +// actually ran are distinguished from never-started ones here, and no start +// timestamp is invented when one was not recorded. +type Stage string + +const ( + StageNotInvoked Stage = "NOT_INVOKED" + StageStarted Stage = "STARTED" + StageFinished Stage = "COMPLETED" +) + +// Availability describes one planned metric slot. +type Availability string + +const ( + Available Availability = "AVAILABLE" + Missing Availability = "MISSING" + Errored Availability = "ERROR" + Declined Availability = "SKIP" + NotProduced Availability = "NOT_PRODUCED" +) + +// Quality is the threshold-decision state. It is always derived, never assigned. +type Quality string + +const ( + QualityMet Quality = "MET" + QualityNotMet Quality = "NOT_MET" + QualityUnknown Quality = "UNKNOWN" + QualityNotConfigured Quality = "NOT_CONFIGURED" + QualityNotAssessed Quality = "NOT_ASSESSED" +) + +// Cancellation and deadline sources are recorded only when actually observed. +// A bare context.Canceled does not establish user intent, so it stays unknown. +const ( + CancelOSSignal = "os_signal" + CancelParentContext = "parent_context" + CancelChildScope = "child_scope" + CancelUnspecified = "unknown" +) + +// ErrEmptyGroup guards group folding against inventing a state. +var ErrEmptyGroup = errors.New("judge: cannot fold completion over an empty member group") + +// Stop classifies why evaluation ended. The cause never comes from an error +// string: only the observed context error, the deadline knob that was configured +// and whether a process signal was actually seen. +type Stop struct { + Cause Cause + DeadlineSource string + CancelSource string + Detail string +} + +// ClassifyStop turns an observed context error into a cause plus its source. +// deadlineSource names the configuration knob when a deadline was configured and +// empty means the deadline's origin is unknown; signalObserved is true only when +// the process actually saw an interrupt, because a bare context.Canceled does not +// establish user-initiated cancellation. +func ClassifyStop(ctxErr error, deadlineSource string, signalObserved bool) Stop { + switch { + case ctxErr == nil: + return Stop{Cause: CauseNone} + case errors.Is(ctxErr, context.DeadlineExceeded): + source := deadlineSource + if source == "" { + source = CancelUnspecified + } + return Stop{Cause: CauseDeadlineExpiry, DeadlineSource: source, Detail: ctxErr.Error()} + case errors.Is(ctxErr, context.Canceled): + source := CancelUnspecified + if signalObserved { + source = CancelOSSignal + } + return Stop{Cause: CauseCancellation, CancelSource: source, Detail: ctxErr.Error()} + default: + return Stop{Cause: CauseExecutionError, Detail: ctxErr.Error()} + } +} + +// MetricSlot is one planned registry slot. The denominator is fixed by +// configuration before execution, never by what a judge happened to return. +type MetricSlot struct { + ID string + Required bool + MinScore *float64 + EvidenceRequired bool +} + +// MetricResult is the materialized state of one planned slot. +type MetricResult struct { + MetricID string `json:"metric_id"` + Availability Availability `json:"availability"` + Value *float64 `json:"value"` + Reason string `json:"reason,omitempty"` + Evidence []string `json:"evidence,omitempty"` +} + +// Coverage counts one bucket per availability over the planned total. +type Coverage struct { + Planned int `json:"planned"` + Available int `json:"available"` + Missing int `json:"missing"` + Errored int `json:"error"` + Skipped int `json:"skip"` + NotProduced int `json:"not_produced"` + Ratio *float64 `json:"ratio"` +} + +// MetricsEnvelope is the additive named-metrics payload. Legacy results omit it. +type MetricsEnvelope struct { + JudgeID string `json:"judge_id"` + Metrics []MetricResult `json:"metrics"` + Quality Quality `json:"metric_quality_state"` + Coverage Coverage `json:"coverage"` + Diagnostics []string `json:"diagnostics,omitempty"` +} + +// ExecutionMetadata preserves how far evaluation got and why it stopped. It is +// carried even when no scoring Result exists at all. +type ExecutionMetadata struct { + Stage Stage `json:"execution_stage"` + Completion Completion `json:"completion_state"` + Cause Cause `json:"termination_cause"` + DeadlineSource string `json:"deadline_source,omitempty"` + CancelSource string `json:"cancel_source,omitempty"` + Detail string `json:"detail,omitempty"` + Timeout *TimeoutMetadata `json:"timeout,omitempty"` +} + +// TimeoutMetadata keeps a timeout visible even after its result was accepted. +type TimeoutMetadata struct { + Occurred bool `json:"occurred"` + Source string `json:"source,omitempty"` + Recovered bool `json:"recovered,omitempty"` + Attempts int `json:"attempts,omitempty"` +} + +// SlotsFromConfig turns the configured metric registry into planned slots. The +// result is the coverage denominator: it is fixed before execution and never +// derived from what a judge returned. +func SlotsFromConfig(metrics []config.MetricConfig) []MetricSlot { + if len(metrics) == 0 { + return nil + } + out := make([]MetricSlot, 0, len(metrics)) + for _, m := range metrics { + out = append(out, MetricSlot{ + ID: m.ID, + Required: m.Required, + MinScore: m.MinScore, + EvidenceRequired: m.EvidenceRequired, + }) + } + return out +} + +// ValidScore reports a finite number in [0,1]. Zero is a valid score; NaN and +// infinities are never clamped into range. +func ValidScore(v *float64) bool { + return v != nil && !math.IsNaN(*v) && !math.IsInf(*v, 0) && *v >= 0 && *v <= 1 +} + +// DeriveQuality applies the four threshold rules in order: thresholds plus all +// thresholded values available gives MET/NOT_MET; thresholds plus any thresholded +// value unavailable gives UNKNOWN; no thresholds plus any valid score gives +// NOT_CONFIGURED; no thresholds and no valid scores gives NOT_ASSESSED. +func DeriveQuality(planned []MetricSlot, got []MetricResult) Quality { + byID := make(map[string]MetricResult, len(got)) + for _, m := range got { + byID[m.MetricID] = m + } + var thresholded []MetricSlot + var valid int + for _, s := range planned { + if s.MinScore != nil { + thresholded = append(thresholded, s) + } + if m, ok := byID[s.ID]; ok && usableScore(m) { + valid++ + } + } + if len(thresholded) > 0 { + allAvailable, met := true, true + for _, s := range thresholded { + m, ok := byID[s.ID] + if !ok || !usableScore(m) { + allAvailable = false + continue + } + if *m.Value < *s.MinScore { + met = false + } + } + if !allAvailable { + return QualityUnknown + } + if met { + return QualityMet + } + return QualityNotMet + } + if valid > 0 { + return QualityNotConfigured + } + return QualityNotAssessed +} + +func usableScore(m MetricResult) bool { + return m.Availability == Available && ValidScore(m.Value) +} + +// FoldGroup derives one group completion from member completions, keeping the +// most severe member state so truncation never hides an earlier failure: +// FAILED > INCOMPLETE > INTERRUPTED > COMPLETED. +func FoldGroup(members []Completion) (Completion, error) { + if len(members) == 0 { + return "", ErrEmptyGroup + } + rank := map[Completion]int{ + CompletionFailed: 3, CompletionIncomplete: 2, + CompletionInterrupted: 1, CompletionFinished: 0, + } + best := members[0] + for _, c := range members[1:] { + if rank[c] > rank[best] { + best = c + } + } + return best, nil +} + +// EmittedMetric is one producer-supplied metric entry before validation. +type EmittedMetric struct { + MetricID string + Availability string + Value *float64 + Reason string + Evidence []string +} + +// PayloadProblem is a whole-envelope defect: nothing in the payload can be +// trusted, so every planned slot becomes ERROR. +type PayloadProblem string + +const ( + ProblemNone PayloadProblem = "" + ProblemUnparseable PayloadProblem = "unparseable_payload" + ProblemTopLevelShape PayloadProblem = "top_level_shape" + ProblemDuplicateJSONKey PayloadProblem = "duplicate_json_key" + ProblemUnknownTopField PayloadProblem = "unknown_top_level_field" + ProblemSchemaVersion PayloadProblem = "schema_version" + ProblemUnboundResult PayloadProblem = "unbound_or_stale_result" + ProblemSizeLimit PayloadProblem = "size_limit" +) + +// MaterializeSlots builds exactly one result per planned slot. Unknown ids are +// output diagnostics and never new slots; a duplicated id invalidates its planned +// slot once; the coverage denominator always stays the planned count. +// +// prevented says evaluation never reached a final outcome (gate, cancellation, +// deadline, execution failure), so slots that were never reached become +// NOT_PRODUCED instead of being reported as producer errors. A score that was +// already obtained is never downgraded because evaluation stopped afterwards. +func MaterializeSlots(judgeID string, planned []MetricSlot, emitted []EmittedMetric, problem PayloadProblem, prevented bool) *MetricsEnvelope { + known := make(map[string]bool, len(planned)) + for _, s := range planned { + known[s.ID] = true + } + grouped := map[string][]EmittedMetric{} + diagnostics := []string{} + for _, e := range emitted { + if !known[e.MetricID] { + diagnostics = append(diagnostics, "unknown_metric_id: "+e.MetricID) + continue + } + grouped[e.MetricID] = append(grouped[e.MetricID], e) + } + + out := make([]MetricResult, 0, len(planned)) + for _, s := range planned { + entries := grouped[s.ID] + switch { + case problem != ProblemNone: + out = append(out, MetricResult{MetricID: s.ID, Availability: Errored, Reason: string(problem)}) + case len(entries) == 0: + out = append(out, emptySlot(s, prevented)) + case len(entries) > 1: + diagnostics = append(diagnostics, "duplicate_metric_id: "+s.ID) + out = append(out, MetricResult{MetricID: s.ID, Availability: Errored, Reason: "duplicate_metric_id"}) + default: + out = append(out, resolveSlot(s, entries[0], prevented)) + } + } + sort.Strings(diagnostics) + env := &MetricsEnvelope{JudgeID: judgeID, Metrics: out} + if len(diagnostics) > 0 { + env.Diagnostics = diagnostics + } + env.Quality = DeriveQuality(planned, out) + env.Coverage = summarizeCoverage(planned, out) + return env +} + +func emptySlot(s MetricSlot, prevented bool) MetricResult { + if prevented { + return MetricResult{MetricID: s.ID, Availability: NotProduced, Reason: "evaluation_prevented"} + } + if s.Required { + return MetricResult{MetricID: s.ID, Availability: Missing, Reason: "required_metric_not_emitted"} + } + return MetricResult{MetricID: s.ID, Availability: Missing, Reason: "not_emitted"} +} + +func resolveSlot(s MetricSlot, e EmittedMetric, prevented bool) MetricResult { + res := MetricResult{MetricID: s.ID, Evidence: e.Evidence, Reason: e.Reason} + switch strings.ToUpper(strings.TrimSpace(e.Availability)) { + case "SKIP", "MISSING": + // A decline or omission is only meaningful from an evaluation that + // reached the slot; a stopped evaluation leaves NOT_PRODUCED instead. + if prevented { + res.Availability, res.Reason = NotProduced, "evaluation_prevented" + return res + } + if strings.EqualFold(strings.TrimSpace(e.Availability), "SKIP") { + res.Availability = Declined + if res.Reason == "" { + res.Reason = "declined_by_judge" + } + return res + } + res.Availability = Missing + if res.Reason == "" { + res.Reason = "not_emitted" + } + return res + case "ERROR": + res.Availability = Errored + if res.Reason == "" { + res.Reason = "metric_error" + } + return res + case "AVAILABLE", "": + if !ValidScore(e.Value) { + res.Availability, res.Reason = Errored, "invalid_value" + return res + } + if s.EvidenceRequired && len(e.Evidence) == 0 { + res.Availability, res.Reason = Errored, "missing_evidence" + return res + } + res.Availability, res.Value = Available, e.Value + return res + default: + res.Availability, res.Reason = Errored, "unknown_availability_state" + return res + } +} + +func summarizeCoverage(planned []MetricSlot, got []MetricResult) Coverage { + c := Coverage{Planned: len(planned)} + for _, m := range got { + switch m.Availability { + case Available: + c.Available++ + case Missing: + c.Missing++ + case Errored: + c.Errored++ + case Declined: + c.Skipped++ + case NotProduced: + c.NotProduced++ + } + } + if c.Planned > 0 { + r := float64(c.Available) / float64(c.Planned) + c.Ratio = &r + } + return c +} diff --git a/internal/judge/metrics_test.go b/internal/judge/metrics_test.go new file mode 100644 index 00000000..7e89973d --- /dev/null +++ b/internal/judge/metrics_test.go @@ -0,0 +1,502 @@ +package judge + +import ( + "context" + "errors" + "fmt" + "math" + "reflect" + "testing" + + "github.com/alibaba/skill-up/internal/config" +) + +func f(v float64) *float64 { return &v } + +func twoSlots() []MetricSlot { + return []MetricSlot{{ID: "correctness", Required: true}, {ID: "completeness"}} +} + +func TestFoldGroup(t *testing.T) { + cases := []struct { + name string + members []Completion + want Completion + wantErr bool + }{ + {"completed plus pending cancelled", []Completion{CompletionFinished, CompletionInterrupted}, CompletionInterrupted, false}, + {"failed plus pending cancelled keeps failed", []Completion{CompletionFailed, CompletionInterrupted}, CompletionFailed, false}, + {"incomplete plus pending cancelled keeps incomplete", []Completion{CompletionIncomplete, CompletionInterrupted}, CompletionIncomplete, false}, + {"failed outranks incomplete", []Completion{CompletionIncomplete, CompletionFailed}, CompletionFailed, false}, + {"all completed", []Completion{CompletionFinished, CompletionFinished}, CompletionFinished, false}, + {"never started is still interrupted not a new state", []Completion{CompletionInterrupted, CompletionInterrupted}, CompletionInterrupted, false}, + {"single member", []Completion{CompletionIncomplete}, CompletionIncomplete, false}, + {"empty group is refused", []Completion{}, "", true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := FoldGroup(tc.members) + if tc.wantErr { + if !errors.Is(err, ErrEmptyGroup) { + t.Fatalf("want ErrEmptyGroup, got %v", err) + } + return + } + if err != nil { + t.Fatalf("unexpected error %v", err) + } + if got != tc.want { + t.Fatalf("fold = %s, want %s", got, tc.want) + } + }) + } +} + +func TestDeriveQuality(t *testing.T) { + cases := []struct { + name string + planned []MetricSlot + got []MetricResult + want Quality + }{ + { + "thresholds configured and all satisfied", + []MetricSlot{{ID: "a", MinScore: f(0.7)}, {ID: "b", MinScore: f(0.5)}}, + []MetricResult{{MetricID: "a", Availability: Available, Value: f(0.7)}, {MetricID: "b", Availability: Available, Value: f(0.9)}}, + QualityMet, + }, + { + "threshold boundary is inclusive", + []MetricSlot{{ID: "a", MinScore: f(0.75)}}, + []MetricResult{{MetricID: "a", Availability: Available, Value: f(0.75)}}, + QualityMet, + }, + { + "thresholds configured and one below", + []MetricSlot{{ID: "a", MinScore: f(0.7)}, {ID: "b", MinScore: f(0.7)}}, + []MetricResult{{MetricID: "a", Availability: Available, Value: f(0.9)}, {MetricID: "b", Availability: Available, Value: f(0.2)}}, + QualityNotMet, + }, + { + "thresholded value unavailable yields UNKNOWN", + []MetricSlot{{ID: "a", MinScore: f(0.7)}, {ID: "b", MinScore: f(0.7)}}, + []MetricResult{{MetricID: "a", Availability: Available, Value: f(0.9)}, {MetricID: "b", Availability: Declined, Reason: "declined_by_judge"}}, + QualityUnknown, + }, + { + "cancellation with thresholds configured yields UNKNOWN", + []MetricSlot{{ID: "a", MinScore: f(0.7)}}, + []MetricResult{{MetricID: "a", Availability: NotProduced, Reason: "evaluation_prevented"}}, + QualityUnknown, + }, + { + "gate short-circuit with thresholds configured yields UNKNOWN", + []MetricSlot{{ID: "a", MinScore: f(0.7)}, {ID: "b", MinScore: f(0.7)}}, + []MetricResult{{MetricID: "a", Availability: NotProduced}, {MetricID: "b", Availability: NotProduced}}, + QualityUnknown, + }, + { + "valid scores and no thresholds is NOT_CONFIGURED", + twoSlots(), + []MetricResult{{MetricID: "correctness", Availability: Available, Value: f(0.8)}, {MetricID: "completeness", Availability: Available, Value: f(0.6)}}, + QualityNotConfigured, + }, + { + "an explicit zero score is a valid score", + twoSlots(), + []MetricResult{{MetricID: "correctness", Availability: Available, Value: f(0)}}, + QualityNotConfigured, + }, + { + "no thresholds and no valid scores is NOT_ASSESSED", + twoSlots(), + []MetricResult{{MetricID: "correctness", Availability: Missing}, {MetricID: "completeness", Availability: Declined}}, + QualityNotAssessed, + }, + { + "cancelled slots without thresholds are NOT_ASSESSED", + twoSlots(), + []MetricResult{{MetricID: "correctness", Availability: NotProduced}, {MetricID: "completeness", Availability: NotProduced}}, + QualityNotAssessed, + }, + { + "an errored value is not a valid score", + twoSlots(), + []MetricResult{{MetricID: "correctness", Availability: Errored, Reason: "invalid_value"}}, + QualityNotAssessed, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := DeriveQuality(tc.planned, tc.got); got != tc.want { + t.Fatalf("quality = %s, want %s", got, tc.want) + } + }) + } +} + +func TestValidScore(t *testing.T) { + cases := []struct { + name string + val *float64 + want bool + }{ + {"nil", nil, false}, + {"zero is valid", f(0), true}, + {"one is valid", f(1), true}, + {"negative", f(-0.01), false}, + {"above one", f(1.4), false}, + {"NaN is never clamped", f(math.NaN()), false}, + {"positive infinity", f(math.Inf(1)), false}, + {"negative infinity", f(math.Inf(-1)), false}, + {"mid range", f(0.75), true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := ValidScore(tc.val); got != tc.want { + t.Fatalf("ValidScore = %v, want %v", got, tc.want) + } + }) + } +} + +func TestMaterializeSlotsAvailability(t *testing.T) { + cases := []struct { + name string + emitted []EmittedMetric + prevented bool + want map[string]Availability + wantDiags []string + wantPlanned int + }{ + { + name: "valid two-metric output", + emitted: []EmittedMetric{{MetricID: "correctness", Availability: "AVAILABLE", Value: f(0.8)}, {MetricID: "completeness", Availability: "AVAILABLE", Value: f(0.5)}}, + want: map[string]Availability{"correctness": Available, "completeness": Available}, + wantPlanned: 2, + }, + { + name: "optional omission is MISSING and required sibling survives", + emitted: []EmittedMetric{{MetricID: "correctness", Availability: "AVAILABLE", Value: f(0.9)}}, + want: map[string]Availability{"correctness": Available, "completeness": Missing}, + wantPlanned: 2, + }, + { + name: "unknown id is a diagnostic and never a new slot", + emitted: []EmittedMetric{{MetricID: "correctness", Availability: "AVAILABLE", Value: f(0.9)}, {MetricID: "invented", Availability: "AVAILABLE", Value: f(0.1)}}, + want: map[string]Availability{"correctness": Available, "completeness": Missing}, + wantDiags: []string{"unknown_metric_id: invented"}, + wantPlanned: 2, + }, + { + name: "duplicate id invalidates that planned slot once and keeps no other slot", + emitted: []EmittedMetric{{MetricID: "correctness", Availability: "AVAILABLE", Value: f(0.9)}, {MetricID: "correctness", Availability: "AVAILABLE", Value: f(0.2)}, {MetricID: "completeness", Availability: "AVAILABLE", Value: f(0.4)}}, + want: map[string]Availability{"correctness": Errored, "completeness": Available}, + wantDiags: []string{"duplicate_metric_id: correctness"}, + wantPlanned: 2, + }, + { + name: "out of range value poisons only its own slot", + emitted: []EmittedMetric{{MetricID: "correctness", Availability: "AVAILABLE", Value: f(1.4)}, {MetricID: "completeness", Availability: "AVAILABLE", Value: f(0.4)}}, + want: map[string]Availability{"correctness": Errored, "completeness": Available}, + wantPlanned: 2, + }, + { + name: "non-finite value poisons only its own slot", + emitted: []EmittedMetric{{MetricID: "correctness", Availability: "AVAILABLE", Value: f(math.NaN())}, {MetricID: "completeness", Availability: "AVAILABLE", Value: f(0.4)}}, + want: map[string]Availability{"correctness": Errored, "completeness": Available}, + wantPlanned: 2, + }, + { + name: "judge-declared SKIP keeps its own reason and is never zero", + emitted: []EmittedMetric{{MetricID: "correctness", Availability: "AVAILABLE", Value: f(0.8)}, {MetricID: "completeness", Availability: "SKIP", Reason: "insufficient evidence"}}, + want: map[string]Availability{"correctness": Available, "completeness": Declined}, + wantPlanned: 2, + }, + { + name: "cancelled before or during invocation produces NOT_PRODUCED, never SKIP", + prevented: true, + want: map[string]Availability{"correctness": NotProduced, "completeness": NotProduced}, + wantPlanned: 2, + }, + { + name: "a score obtained before the stop is not downgraded to NOT_PRODUCED", + prevented: true, + emitted: []EmittedMetric{{MetricID: "correctness", Availability: "AVAILABLE", Value: f(0.7)}}, + want: map[string]Availability{"correctness": Available, "completeness": NotProduced}, + wantPlanned: 2, + }, + { + name: "a SKIP or MISSING claim from a stopped evaluation is NOT_PRODUCED, not a decline", + prevented: true, + emitted: []EmittedMetric{{MetricID: "correctness", Availability: "SKIP", Reason: "ran out of time"}}, + want: map[string]Availability{"correctness": NotProduced, "completeness": NotProduced}, + wantPlanned: 2, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + env := MaterializeSlots("j1", twoSlots(), tc.emitted, ProblemNone, tc.prevented) + if len(env.Metrics) != tc.wantPlanned { + t.Fatalf("materialized %d slots, planned %d: denominator must not move", len(env.Metrics), tc.wantPlanned) + } + if env.Coverage.Planned != tc.wantPlanned { + t.Fatalf("coverage planned = %d, want %d", env.Coverage.Planned, tc.wantPlanned) + } + got := map[string]Availability{} + for _, m := range env.Metrics { + got[m.MetricID] = m.Availability + } + if !reflect.DeepEqual(got, tc.want) { + t.Fatalf("availability = %v, want %v", got, tc.want) + } + if !reflect.DeepEqual(env.Diagnostics, tc.wantDiags) && len(tc.wantDiags) > 0 { + t.Fatalf("diagnostics = %v, want %v", env.Diagnostics, tc.wantDiags) + } + for _, m := range env.Metrics { + if m.Availability != Available && m.Value != nil { + t.Fatalf("slot %s with availability %s carries a value %v; unusable states must stay valueless", m.MetricID, m.Availability, *m.Value) + } + } + }) + } +} + +func TestMaterializeSlotsPayloadProblems(t *testing.T) { + problems := []PayloadProblem{ProblemUnparseable, ProblemTopLevelShape, ProblemDuplicateJSONKey, + ProblemUnknownTopField, ProblemSchemaVersion, ProblemUnboundResult, ProblemSizeLimit} + for _, p := range problems { + t.Run(string(p), func(t *testing.T) { + // A well-formed pair of scores must still be discarded, because the + // envelope as a whole cannot be trusted. + emitted := []EmittedMetric{ + {MetricID: "correctness", Availability: "AVAILABLE", Value: f(0.8)}, + {MetricID: "completeness", Availability: "AVAILABLE", Value: f(0.6)}, + } + env := MaterializeSlots("j1", twoSlots(), emitted, p, false) + for _, m := range env.Metrics { + if m.Availability != Errored || m.Reason != string(p) || m.Value != nil { + t.Fatalf("slot %s = %+v, want ERROR/%s with no value", m.MetricID, m, p) + } + } + if env.Coverage.Planned != 2 || env.Coverage.Errored != 2 { + t.Fatalf("coverage = %+v, want planned 2 and errored 2", env.Coverage) + } + if env.Quality != QualityNotAssessed { + t.Fatalf("quality = %s, want NOT_ASSESSED", env.Quality) + } + }) + } +} + +func TestSlotLevelDefectKeepsSiblingsUsable(t *testing.T) { + planned := []MetricSlot{{ID: "correctness", Required: true}, {ID: "completeness"}, {ID: "instruction_following"}} + emitted := []EmittedMetric{ + {MetricID: "correctness", Availability: "AVAILABLE", Value: f(0.9)}, + {MetricID: "completeness", Availability: "AVAILABLE", Value: f(2.5)}, // invalid + {MetricID: "instruction_following", Availability: "AVAILABLE", Value: f(0.5)}, + } + env := MaterializeSlots("j1", planned, emitted, ProblemNone, false) + if env.Metrics[0].Availability != Available || env.Metrics[2].Availability != Available { + t.Fatalf("valid siblings must stay AVAILABLE: %+v", env.Metrics) + } + if env.Metrics[1].Availability != Errored || env.Metrics[1].Reason != "invalid_value" { + t.Fatalf("defective slot = %+v", env.Metrics[1]) + } + if env.Coverage.Available != 2 || env.Coverage.Errored != 1 || env.Coverage.Planned != 3 { + t.Fatalf("coverage = %+v, want 2 available of 3 planned", env.Coverage) + } + if *env.Coverage.Ratio != 2.0/3.0 { + t.Fatalf("ratio = %v, want 2/3", *env.Coverage.Ratio) + } + if env.Quality != QualityNotConfigured { + t.Fatalf("quality = %s, want NOT_CONFIGURED", env.Quality) + } +} + +func TestEvidenceRequirementIsPerSlot(t *testing.T) { + planned := []MetricSlot{{ID: "correctness", EvidenceRequired: true}, {ID: "completeness"}} + env := MaterializeSlots("j1", planned, []EmittedMetric{ + {MetricID: "correctness", Availability: "AVAILABLE", Value: f(0.8)}, + {MetricID: "completeness", Availability: "AVAILABLE", Value: f(0.8)}, + }, ProblemNone, false) + if env.Metrics[0].Availability != Errored || env.Metrics[0].Reason != "missing_evidence" { + t.Fatalf("evidence-less slot = %+v", env.Metrics[0]) + } + if env.Metrics[1].Availability != Available { + t.Fatalf("sibling without an evidence requirement must stay usable: %+v", env.Metrics[1]) + } +} + +func TestCoverageWithoutPlannedSlotsStaysNull(t *testing.T) { + env := MaterializeSlots("j1", nil, nil, ProblemNone, false) + if env.Coverage.Planned != 0 || env.Coverage.Ratio != nil { + t.Fatalf("coverage = %+v, want planned 0 and null ratio", env.Coverage) + } + if env.Quality != QualityNotAssessed { + t.Fatalf("quality = %s, want NOT_ASSESSED", env.Quality) + } +} + +func TestSlotsFromConfigKeepsThePlannedDenominator(t *testing.T) { + registry := []config.MetricConfig{ + {ID: "correctness", Required: true, EvidenceRequired: true}, + {ID: "completeness", MinScore: f(0.75)}, + } + slots := SlotsFromConfig(registry) + if len(slots) != 2 { + t.Fatalf("slots = %+v, want 2", slots) + } + if !slots[0].Required || !slots[0].EvidenceRequired || slots[0].MinScore != nil { + t.Fatalf("slot 0 = %+v", slots[0]) + } + if slots[1].MinScore == nil || *slots[1].MinScore != 0.75 { + t.Fatalf("slot 1 = %+v", slots[1]) + } + if SlotsFromConfig(nil) != nil { + t.Fatal("a judge without a registry must keep no slots") + } + // The registry, not a returned payload, sets the denominator. + env := MaterializeSlots("j1", slots, []EmittedMetric{{MetricID: "correctness", Availability: "AVAILABLE", Value: f(0.9)}}, ProblemNone, false) + if env.Coverage.Planned != 2 { + t.Fatalf("planned = %d, want the registry count of 2", env.Coverage.Planned) + } +} + +func TestClassifyStop(t *testing.T) { + cases := []struct { + name string + err error + deadlineSource string + signal bool + want Stop + }{ + {"no error means no cause", nil, "", false, Stop{Cause: CauseNone}}, + { + "deadline with a known knob names it", + context.DeadlineExceeded, "judge.timeout_seconds", true, + Stop{Cause: CauseDeadlineExpiry, DeadlineSource: "judge.timeout_seconds", Detail: "context deadline exceeded"}, + }, + { + "deadline without a known origin stays unknown", + context.DeadlineExceeded, "", false, + Stop{Cause: CauseDeadlineExpiry, DeadlineSource: CancelUnspecified, Detail: "context deadline exceeded"}, + }, + { + "cancellation observed as a process signal is attributed", + context.Canceled, "", true, + Stop{Cause: CauseCancellation, CancelSource: CancelOSSignal, Detail: "context canceled"}, + }, + { + "a bare cancellation is never claimed as user-initiated", + context.Canceled, "", false, + Stop{Cause: CauseCancellation, CancelSource: CancelUnspecified, Detail: "context canceled"}, + }, + { + "a wrapped cancellation is still a cancellation", + fmt.Errorf("judge session failed: %w", context.Canceled), "", false, + Stop{Cause: CauseCancellation, CancelSource: CancelUnspecified, Detail: "judge session failed: context canceled"}, + }, + { + "an unrelated error is an execution error", + errors.New("transport reset"), "", false, + Stop{Cause: CauseExecutionError, Detail: "transport reset"}, + }, + { + "a deadline wins over a signal when both fired", + context.DeadlineExceeded, "case.constraints.timeout_seconds", true, + Stop{Cause: CauseDeadlineExpiry, DeadlineSource: "case.constraints.timeout_seconds", Detail: "context deadline exceeded"}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := ClassifyStop(tc.err, tc.deadlineSource, tc.signal) + if got != tc.want { + t.Fatalf("ClassifyStop = %+v, want %+v", got, tc.want) + } + }) + } +} + +func TestCompareScoringDefinitions(t *testing.T) { + base := ScoringDefinition{JudgeID: "j1", MetricID: "correctness", RubricHash: "sha:rubric", ScaleHash: "sha:scale", + Provider: "openai", Model: "gpt-x", ModelRevision: "2026-05-01"} + cases := []struct { + name string + other ScoringDefinition + wantSame bool + wantReasons []string + wantAdvisory []string + }{ + { + name: "identical definition is comparable with no advisory", + other: base, + wantSame: true, + }, + { + name: "different provider and model with the same scoring definition still compares", + other: withIdentity(base, "anthropic", "claude-y", "2026-06-01"), + wantSame: true, + wantAdvisory: []string{"provider_differs", "model_alias_differs", "model_revision_differs"}, + }, + { + name: "unknown revision is advisory only", + other: withIdentity(base, "openai", "gpt-x", ""), + wantSame: true, + wantAdvisory: []string{"model_revision_unknown"}, + }, + { + name: "known but differing revision is advisory only", + other: withIdentity(base, "openai", "gpt-x", "2026-01-01"), + wantSame: true, + wantAdvisory: []string{"model_revision_differs"}, + }, + { + name: "rubric difference blocks", + other: withRubric(base, "sha:other"), + wantSame: false, + wantReasons: []string{"rubric_mismatch"}, + }, + { + name: "scoring scale difference blocks", + other: withScale(base, "sha:other-scale"), + wantSame: false, + wantReasons: []string{"scoring_scale_mismatch"}, + }, + { + name: "metric id difference blocks", + other: withMetric(base, "completeness"), + wantSame: false, + wantReasons: []string{"metric_id_mismatch"}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := CompareScoringDefinitions(base, tc.other) + if got.Comparable != tc.wantSame { + t.Fatalf("comparable = %v, want %v (reasons %v)", got.Comparable, tc.wantSame, got.Reasons) + } + if !reflect.DeepEqual(got.Reasons, tc.wantReasons) && len(tc.wantReasons) > 0 { + t.Fatalf("reasons = %v, want %v", got.Reasons, tc.wantReasons) + } + if len(tc.wantReasons) == 0 && len(got.Reasons) != 0 { + t.Fatalf("unexpected blocking reasons %v", got.Reasons) + } + if !reflect.DeepEqual(got.Advisories, tc.wantAdvisory) && len(tc.wantAdvisory) > 0 { + t.Fatalf("advisories = %v, want %v", got.Advisories, tc.wantAdvisory) + } + if len(tc.wantAdvisory) == 0 && len(got.Advisories) != 0 { + t.Fatalf("unexpected advisories %v", got.Advisories) + } + }) + } +} + +func withIdentity(d ScoringDefinition, provider, model, rev string) ScoringDefinition { + d.Provider, d.Model, d.ModelRevision = provider, model, rev + return d +} + +func withRubric(d ScoringDefinition, h string) ScoringDefinition { d.RubricHash = h; return d } +func withScale(d ScoringDefinition, h string) ScoringDefinition { d.ScaleHash = h; return d } +func withMetric(d ScoringDefinition, id string) ScoringDefinition { d.MetricID = id; return d } From 8a0d4b78358258e8fc45e1f98deb0becd79d45b3 Mon Sep 17 00:00:00 2001 From: Max_Guobaixin Date: Tue, 27 Oct 2026 18:35:51 +0800 Subject: [PATCH 02/12] style: gofmt the #317 test tables --- internal/config/metrics_validation_test.go | 18 ++++++++-------- internal/evaluator/metrics_wiring_test.go | 24 +++++++++++++--------- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/internal/config/metrics_validation_test.go b/internal/config/metrics_validation_test.go index 3ddf3a6e..3004a696 100644 --- a/internal/config/metrics_validation_test.go +++ b/internal/config/metrics_validation_test.go @@ -16,19 +16,19 @@ func TestValidateJudgeMetrics(t *testing.T) { }{ {name: "absent registry is valid legacy configuration", metrics: nil, wantOK: true}, { - name: "two well-formed slots", - metrics: []MetricConfig{{ID: "correctness", Required: true}, {ID: "completeness", MinScore: f(0.7)}}, - wantOK: true, + name: "two well-formed slots", + metrics: []MetricConfig{{ID: "correctness", Required: true}, {ID: "completeness", MinScore: f(0.7)}}, + wantOK: true, }, { - name: "a zero threshold is distinguishable from an absent one and stays valid", - metrics: []MetricConfig{{ID: "correctness", MinScore: f(0)}}, - wantOK: true, + name: "a zero threshold is distinguishable from an absent one and stays valid", + metrics: []MetricConfig{{ID: "correctness", MinScore: f(0)}}, + wantOK: true, }, { - name: "upper bound is inclusive", - metrics: []MetricConfig{{ID: "correctness", MinScore: f(1)}}, - wantOK: true, + name: "upper bound is inclusive", + metrics: []MetricConfig{{ID: "correctness", MinScore: f(1)}}, + wantOK: true, }, { name: "threshold above the score range is rejected", diff --git a/internal/evaluator/metrics_wiring_test.go b/internal/evaluator/metrics_wiring_test.go index 043680a7..afb14e4d 100644 --- a/internal/evaluator/metrics_wiring_test.go +++ b/internal/evaluator/metrics_wiring_test.go @@ -64,14 +64,18 @@ func TestMemberStopAttributesSources(t *testing.T) { want: judge.Stop{Cause: judge.CauseDeadlineExpiry, DeadlineSource: judge.CancelUnspecified, Detail: "context deadline exceeded"}, }, { - name: "cancellation without an observed signal stays unknown", - build: func(*testing.T) (context.Context, context.Context) { return canceledContext(false), canceledContext(false) }, - want: judge.Stop{Cause: judge.CauseCancellation, CancelSource: judge.CancelUnspecified, Detail: "context canceled"}, + name: "cancellation without an observed signal stays unknown", + build: func(*testing.T) (context.Context, context.Context) { + return canceledContext(false), canceledContext(false) + }, + want: judge.Stop{Cause: judge.CauseCancellation, CancelSource: judge.CancelUnspecified, Detail: "context canceled"}, }, { - name: "cancellation with an observed signal is attributed", - build: func(*testing.T) (context.Context, context.Context) { return canceledContext(true), canceledContext(true) }, - want: judge.Stop{Cause: judge.CauseCancellation, CancelSource: judge.CancelOSSignal, Detail: "context canceled"}, + name: "cancellation with an observed signal is attributed", + build: func(*testing.T) (context.Context, context.Context) { + return canceledContext(true), canceledContext(true) + }, + want: judge.Stop{Cause: judge.CauseCancellation, CancelSource: judge.CancelOSSignal, Detail: "context canceled"}, }, } for _, tc := range cases { @@ -374,10 +378,10 @@ func TestSignalObserverReportsOnlyObservedSignals(t *testing.T) { func TestGroupFoldOverMemberOutcomes(t *testing.T) { cases := []struct { - name string - states []judge.Completion - want judge.Completion - wantErr bool + name string + states []judge.Completion + want judge.Completion + wantErr bool }{ {"completed plus pending cancelled", []judge.Completion{judge.CompletionFinished, judge.CompletionInterrupted}, judge.CompletionInterrupted, false}, {"failed plus pending cancelled keeps FAILED", []judge.Completion{judge.CompletionFailed, judge.CompletionInterrupted}, judge.CompletionFailed, false}, From 132a6d02d941b9f5444f253ace76ce273a82e432 Mon Sep 17 00:00:00 2001 From: Max_Guobaixin Date: Tue, 27 Oct 2026 19:09:51 +0800 Subject: [PATCH 03/12] feat(judge): real named-metric input channel via the script judge The previous commit added the state machine but no producer: both grouped call sites materialized slots from a nil emission list, so no judge could ever report an AVAILABLE metric and DeriveQuality/FoldGroup had no production caller. This adds the missing slice-1 path. - script judge executes, collects its invocation-bound result file, validates it and materializes one slot per planned registry entry - whole-envelope defects (duplicate json keys, unknown top-level field, bad schema version, unbound or stale invocation, size limit) poison every slot; entry-level defects poison only their own slot and keep valid siblings - a timeout no longer discards a trustworthy result: collected scores stay AVAILABLE while the existing script-timeout verdict mapping is preserved, and slots never reached become NOT_PRODUCED rather than an invented zero - group completion and cause are derived with FoldGroup and persisted into evaluation.json beside the unchanged all_required verdict - integration tests spawn the real script judge through executeCase; the process-kill cancellation case is POSIX-only for the same process-group reason upstream already documents Removes the unconsumed definition-compatibility helper and the cancel sources that slice 1 cannot actually detect; those return with their consumers. --- internal/evaluator/evaluator.go | 54 ++ .../evaluator/metrics_integration_test.go | 284 +++++++++++ internal/evaluator/multi_judge.go | 30 ++ internal/judge/compat.go | 67 --- internal/judge/factory.go | 1 + internal/judge/metrics.go | 12 +- internal/judge/metrics_payload.go | 192 +++++++ internal/judge/metrics_test.go | 83 --- internal/judge/script.go | 91 +++- internal/judge/script_metrics_test.go | 473 ++++++++++++++++++ internal/report/grouped_evaluation.go | 10 +- internal/report/metrics_persistence_test.go | 114 +++++ internal/runner/runner.go | 7 +- 13 files changed, 1249 insertions(+), 169 deletions(-) create mode 100644 internal/evaluator/metrics_integration_test.go delete mode 100644 internal/judge/compat.go create mode 100644 internal/judge/metrics_payload.go create mode 100644 internal/judge/script_metrics_test.go create mode 100644 internal/report/metrics_persistence_test.go diff --git a/internal/evaluator/evaluator.go b/internal/evaluator/evaluator.go index ae4d918b..036eb821 100644 --- a/internal/evaluator/evaluator.go +++ b/internal/evaluator/evaluator.go @@ -6,6 +6,8 @@ package evaluator import ( "context" + "crypto/rand" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -110,6 +112,11 @@ type EvalResult struct { // AggregationStrategy identifies the multi-judge policy, including gate failures. AggregationStrategy string + // GroupCompletion folds the member completion states and GroupCause records why + // the plan ended. Both are derived metadata, never another verdict. (#317) + GroupCompletion judge.Completion + GroupCause judge.Cause + // JudgeSession is the separate agent session used by agent_judge. It is // preserved even when the judge fails to produce a valid grading result. JudgeSession *agent.SessionResult @@ -753,6 +760,7 @@ func (e *defaultEvaluator) runJudgePhaseWithSpan( cleanupScript := configureScriptJudge(ctx, j, judgeInput.Transcript, strictTimeout) defer cleanupScript() + e.configureMetricsChannel(ctx, j, configName, caseCfg, artifactRoot) grading, err := j.Evaluate(ctx, judgeInput) if err != nil { @@ -784,6 +792,52 @@ func (e *defaultEvaluator) runJudgePhaseWithSpan( return *result } +// configureMetricsChannel assigns one private, invocation-bound result path for a +// script judge that declares named metrics. The path is created fresh per +// invocation and carries a random id, so a leftover file from an earlier attempt +// can never be accepted as this run's result. Local (none) environments only in +// this slice: a sandboxed script cannot see a host path, and channel work for +// those runtimes is a separate bounded change. +func (e *defaultEvaluator) configureMetricsChannel( + ctx context.Context, + j judge.Judge, + configName string, + caseCfg *config.CaseConfig, + artifactRoot string, +) { + sj, ok := j.(*judge.ScriptJudge) + if !ok || len(sj.MetricSlots) == 0 { + return + } + if e.evalCfg.Environment.Type != "none" { + logging.DebugContextf(ctx, "Evaluator: named metrics are not collected for case %s in the %s environment", caseCfg.ID, e.evalCfg.Environment.Type) + return + } + if configName == "" || caseCfg.ID == "" || e.outputDir == "" { + return + } + dir := filepath.Join(e.outputDir, caseCfg.ID, configName, "outputs", artifactRoot) + if err := os.MkdirAll(dir, 0o755); err != nil { + logging.WarnContextf(ctx, "Evaluator: cannot create metrics dir for case %s: %v", caseCfg.ID, err) + return + } + invocation, err := metricsInvocationID() + if err != nil { + logging.WarnContextf(ctx, "Evaluator: cannot generate metrics invocation id for case %s: %v", caseCfg.ID, err) + return + } + sj.MetricsResultPath = filepath.Join(dir, "metrics-"+invocation+".json") + sj.MetricsInvocationID = invocation +} + +func metricsInvocationID() (string, error) { + buf := make([]byte, 8) + if _, err := rand.Read(buf); err != nil { + return "", err + } + return hex.EncodeToString(buf), nil +} + func configureScriptJudge(ctx context.Context, j judge.Judge, transcriptData transcript.Transcript, strictTimeout bool) func() { sj, ok := j.(*judge.ScriptJudge) if !ok { diff --git a/internal/evaluator/metrics_integration_test.go b/internal/evaluator/metrics_integration_test.go new file mode 100644 index 00000000..05f9118e --- /dev/null +++ b/internal/evaluator/metrics_integration_test.go @@ -0,0 +1,284 @@ +package evaluator + +import ( + "context" + "os" + "path/filepath" + goruntime "runtime" + "strings" + "testing" + "time" + + "github.com/alibaba/skill-up/internal/agent" + "github.com/alibaba/skill-up/internal/config" + "github.com/alibaba/skill-up/internal/judge" + "github.com/alibaba/skill-up/internal/runtime" +) + +// Integration tests for the named-metrics path through the real evaluator: a +// script judge is executed as a child process by executeCase, its +// invocation-bound result file is collected, and the resulting envelope reaches +// the grouped outcome and the derived group state. + +func metricsScript(t *testing.T, dir string, body string) string { + t.Helper() + name := "score.sh" + if goruntime.GOOS == windowsGOOS { + name = "score.ps1" + } + path := filepath.Join(dir, name) + if err := os.WriteFile(path, []byte(body), 0o755); err != nil { + t.Fatalf("write script: %v", err) + } + return path +} + +const ( + posixScore = `#!/bin/sh +cat > "$EVAL_METRICS_RESULT_PATH" < 0 { - out.Comparable = false - } - if a.Provider != b.Provider { - out.Advisories = append(out.Advisories, "provider_differs") - } - if a.Model != b.Model { - out.Advisories = append(out.Advisories, "model_alias_differs") - } - if a.ModelRevision == "" || b.ModelRevision == "" { - out.Advisories = append(out.Advisories, "model_revision_unknown") - } else if a.ModelRevision != b.ModelRevision { - out.Advisories = append(out.Advisories, "model_revision_differs") - } - return out -} diff --git a/internal/judge/factory.go b/internal/judge/factory.go index 5ed025cd..2aef0ea2 100644 --- a/internal/judge/factory.go +++ b/internal/judge/factory.go @@ -34,6 +34,7 @@ func NewJudge(cfg config.JudgeConfig, ag agent.Agent, rt runtime.Runtime) (Judge ScriptPath: cfg.ScriptPath, TimeoutSeconds: derefInt(cfg.TimeoutSeconds), Runtime: rt, + MetricSlots: SlotsFromConfig(cfg.Metrics), }, nil case "agent_judge": diff --git a/internal/judge/metrics.go b/internal/judge/metrics.go index aa36bc42..d67513ad 100644 --- a/internal/judge/metrics.go +++ b/internal/judge/metrics.go @@ -87,13 +87,12 @@ const ( QualityNotAssessed Quality = "NOT_ASSESSED" ) -// Cancellation and deadline sources are recorded only when actually observed. -// A bare context.Canceled does not establish user intent, so it stays unknown. +// Only two cancel sources are actually detectable today: a process interrupt observed by +// the CLI, and everything else, which is reported as unknown rather than guessed. +// A bare context.Canceled does not establish user intent. const ( - CancelOSSignal = "os_signal" - CancelParentContext = "parent_context" - CancelChildScope = "child_scope" - CancelUnspecified = "unknown" + CancelOSSignal = "os_signal" + CancelUnspecified = "unknown" ) // ErrEmptyGroup guards group folding against inventing a state. @@ -190,7 +189,6 @@ type TimeoutMetadata struct { Occurred bool `json:"occurred"` Source string `json:"source,omitempty"` Recovered bool `json:"recovered,omitempty"` - Attempts int `json:"attempts,omitempty"` } // SlotsFromConfig turns the configured metric registry into planned slots. The diff --git a/internal/judge/metrics_payload.go b/internal/judge/metrics_payload.go new file mode 100644 index 00000000..e623987e --- /dev/null +++ b/internal/judge/metrics_payload.go @@ -0,0 +1,192 @@ +package judge + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" +) + +// Structured named-metrics result payload for the script judge (#317, slice 1). +// The runner assigns one result path per invocation; stdout is never a score +// channel and there is no fallback parsing. + +// MetricsSchemaVersion is the only payload version accepted. +const MetricsSchemaVersion = 1 + +// MaxMetricsPayloadBytes bounds a single collected result file. +const MaxMetricsPayloadBytes = 1 << 20 + +var metricsEntryFields = map[string]bool{ + "metric_id": true, + "availability": true, + "value": true, + "reason": true, + "evidence": true, +} + +// ParseMetricsPayload validates a collected result file. A whole-envelope defect +// returns a PayloadProblem so every planned slot becomes ERROR; a defect confined +// to one entry is returned as that entry so only its own slot is poisoned. +func ParseMetricsPayload(raw []byte, invocationID string) ([]EmittedMetric, PayloadProblem) { + if len(raw) > MaxMetricsPayloadBytes { + return nil, ProblemSizeLimit + } + if problem := checkTopLevelKeys(raw); problem != ProblemNone { + return nil, problem + } + var envelope struct { + SchemaVersion int `json:"schema_version"` + InvocationID string `json:"invocation_id"` + Metrics []map[string]json.RawMessage `json:"metrics"` + } + reader := bytes.NewReader(raw) + decoder := json.NewDecoder(reader) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&envelope); err != nil { + var typeErr *json.UnmarshalTypeError + if errors.As(err, &typeErr) { + return nil, ProblemTopLevelShape + } + if bytes.Contains([]byte(err.Error()), []byte("unknown field")) { + return nil, ProblemUnknownTopField + } + return nil, ProblemUnparseable + } + if err := decoder.Decode(new(json.RawMessage)); !errors.Is(err, io.EOF) { + return nil, ProblemTopLevelShape // trailing content after one value + } + if envelope.SchemaVersion != MetricsSchemaVersion { + return nil, ProblemSchemaVersion + } + if invocationID != "" && envelope.InvocationID != invocationID { + return nil, ProblemUnboundResult + } + if envelope.Metrics == nil { + return nil, ProblemTopLevelShape + } + out := make([]EmittedMetric, 0, len(envelope.Metrics)) + for _, entry := range envelope.Metrics { + metric, problem := decodeMetricsEntry(entry) + if problem != ProblemNone { + return nil, problem + } + out = append(out, metric) + } + return out, ProblemNone +} + +// decodeMetricsEntry reads one entry. Ignoring the grammar or declaring a field +// of the wrong kind is a payload defect; a non-numeric or absent value stays a +// slot-level problem so only that slot becomes ERROR. +func decodeMetricsEntry(entry map[string]json.RawMessage) (EmittedMetric, PayloadProblem) { + for field := range entry { + if !metricsEntryFields[field] { + return EmittedMetric{}, ProblemUnknownTopField + } + } + var metric EmittedMetric + for field, target := range map[string]any{ + "metric_id": &metric.MetricID, + "availability": &metric.Availability, + "reason": &metric.Reason, + "evidence": &metric.Evidence, + } { + raw, ok := entry[field] + if !ok { + continue + } + if err := json.Unmarshal(raw, target); err != nil { + return EmittedMetric{}, ProblemTopLevelShape + } + } + if raw, ok := entry["value"]; ok && !bytes.Equal(bytes.TrimSpace(raw), []byte("null")) { + var value float64 + if err := json.Unmarshal(raw, &value); err != nil { + // Non-numeric value: claim AVAILABLE with no value so the slot is + // poisoned on its own instead of failing the whole payload. + metric.Availability = string(Available) + return metric, ProblemNone + } + metric.Value = &value + } + return metric, ProblemNone +} + +// checkTopLevelKeys reports duplicate JSON keys at any nesting level and a +// non-object top level. A duplicate makes binding ambiguous, so nothing in the +// payload can be trusted. +func checkTopLevelKeys(raw []byte) PayloadProblem { + decoder := json.NewDecoder(bytes.NewReader(raw)) + token, err := decoder.Token() + if err != nil { + return ProblemUnparseable + } + delim, ok := token.(json.Delim) + if !ok || delim != '{' { + return ProblemTopLevelShape + } + return scanObject(decoder) +} + +func scanObject(decoder *json.Decoder) PayloadProblem { + seen := map[string]bool{} + for decoder.More() { + keyToken, err := decoder.Token() + if err != nil { + return ProblemUnparseable + } + key, ok := keyToken.(string) + if !ok { + return ProblemTopLevelShape + } + if seen[key] { + return ProblemDuplicateJSONKey + } + seen[key] = true + valueToken, err := decoder.Token() + if err != nil { + return ProblemUnparseable + } + if problem := scanValue(decoder, valueToken); problem != ProblemNone { + return problem + } + } + if _, err := decoder.Token(); err != nil { // consume the closing brace + return ProblemUnparseable + } + return ProblemNone +} + +func scanValue(decoder *json.Decoder, token json.Token) PayloadProblem { + delim, isDelim := token.(json.Delim) + switch { + case !isDelim: + return ProblemNone + case delim == '{': + return scanObject(decoder) + case delim == '[': + for decoder.More() { + item, err := decoder.Token() + if err != nil { + return ProblemUnparseable + } + if problem := scanValue(decoder, item); problem != ProblemNone { + return problem + } + } + if _, err := decoder.Token(); err != nil { // consume the closing bracket + return ProblemUnparseable + } + return ProblemNone + default: + return ProblemTopLevelShape + } +} + +// MetricsPayloadTemplate documents the exact grammar a script must satisfy. +func MetricsPayloadTemplate(invocationID string) string { + return fmt.Sprintf(`{"schema_version":%d,"invocation_id":%q,"metrics":[{"metric_id":"...","availability":"AVAILABLE","value":0.0,"evidence":["..."]}]}`, + MetricsSchemaVersion, invocationID) +} diff --git a/internal/judge/metrics_test.go b/internal/judge/metrics_test.go index 7e89973d..0a56e8cf 100644 --- a/internal/judge/metrics_test.go +++ b/internal/judge/metrics_test.go @@ -417,86 +417,3 @@ func TestClassifyStop(t *testing.T) { }) } } - -func TestCompareScoringDefinitions(t *testing.T) { - base := ScoringDefinition{JudgeID: "j1", MetricID: "correctness", RubricHash: "sha:rubric", ScaleHash: "sha:scale", - Provider: "openai", Model: "gpt-x", ModelRevision: "2026-05-01"} - cases := []struct { - name string - other ScoringDefinition - wantSame bool - wantReasons []string - wantAdvisory []string - }{ - { - name: "identical definition is comparable with no advisory", - other: base, - wantSame: true, - }, - { - name: "different provider and model with the same scoring definition still compares", - other: withIdentity(base, "anthropic", "claude-y", "2026-06-01"), - wantSame: true, - wantAdvisory: []string{"provider_differs", "model_alias_differs", "model_revision_differs"}, - }, - { - name: "unknown revision is advisory only", - other: withIdentity(base, "openai", "gpt-x", ""), - wantSame: true, - wantAdvisory: []string{"model_revision_unknown"}, - }, - { - name: "known but differing revision is advisory only", - other: withIdentity(base, "openai", "gpt-x", "2026-01-01"), - wantSame: true, - wantAdvisory: []string{"model_revision_differs"}, - }, - { - name: "rubric difference blocks", - other: withRubric(base, "sha:other"), - wantSame: false, - wantReasons: []string{"rubric_mismatch"}, - }, - { - name: "scoring scale difference blocks", - other: withScale(base, "sha:other-scale"), - wantSame: false, - wantReasons: []string{"scoring_scale_mismatch"}, - }, - { - name: "metric id difference blocks", - other: withMetric(base, "completeness"), - wantSame: false, - wantReasons: []string{"metric_id_mismatch"}, - }, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - got := CompareScoringDefinitions(base, tc.other) - if got.Comparable != tc.wantSame { - t.Fatalf("comparable = %v, want %v (reasons %v)", got.Comparable, tc.wantSame, got.Reasons) - } - if !reflect.DeepEqual(got.Reasons, tc.wantReasons) && len(tc.wantReasons) > 0 { - t.Fatalf("reasons = %v, want %v", got.Reasons, tc.wantReasons) - } - if len(tc.wantReasons) == 0 && len(got.Reasons) != 0 { - t.Fatalf("unexpected blocking reasons %v", got.Reasons) - } - if !reflect.DeepEqual(got.Advisories, tc.wantAdvisory) && len(tc.wantAdvisory) > 0 { - t.Fatalf("advisories = %v, want %v", got.Advisories, tc.wantAdvisory) - } - if len(tc.wantAdvisory) == 0 && len(got.Advisories) != 0 { - t.Fatalf("unexpected advisories %v", got.Advisories) - } - }) - } -} - -func withIdentity(d ScoringDefinition, provider, model, rev string) ScoringDefinition { - d.Provider, d.Model, d.ModelRevision = provider, model, rev - return d -} - -func withRubric(d ScoringDefinition, h string) ScoringDefinition { d.RubricHash = h; return d } -func withScale(d ScoringDefinition, h string) ScoringDefinition { d.ScaleHash = h; return d } -func withMetric(d ScoringDefinition, id string) ScoringDefinition { d.MetricID = id; return d } diff --git a/internal/judge/script.go b/internal/judge/script.go index 92f58493..02d656cd 100644 --- a/internal/judge/script.go +++ b/internal/judge/script.go @@ -2,8 +2,10 @@ package judge import ( "context" + "errors" "fmt" "log" + "os" "strconv" "strings" "time" @@ -37,6 +39,18 @@ type ScriptJudge struct { // Runtime executes the script in the same environment as the evaluated agent. // When nil, ScriptJudge creates a none runtime so local execution uses the same path. Runtime evalruntime.Runtime + + // MetricSlots is the planned named-metric registry for this judge. Empty means + // the judge produces no metrics envelope at all. + MetricSlots []MetricSlot + + // MetricsResultPath is the runner-assigned host path for this invocation's + // structured result. Set only when the script runs against the local runtime. + MetricsResultPath string + + // MetricsInvocationID binds the collected file to this exact invocation, so a + // leftover file from an earlier attempt is never accepted. + MetricsInvocationID string } // Evaluate implements the Judge interface by executing the configured script. @@ -114,24 +128,38 @@ func (j *ScriptJudge) evaluateInRuntime(ctx context.Context, rt evalruntime.Runt if remoteTranscript != "" { transcriptEnv = plan.envPath(remoteTranscript) } + env := map[string]string{ + "EVAL_TRANSCRIPT_PATH": transcriptEnv, + "EVAL_FINAL_MESSAGE": in.FinalMessage, + "EVAL_EXIT_CODE": strconv.Itoa(in.ExitCode), + } + if len(j.MetricSlots) > 0 && j.MetricsResultPath != "" { + env["EVAL_METRICS_RESULT_PATH"] = j.MetricsResultPath + env["EVAL_METRICS_INVOCATION_ID"] = j.MetricsInvocationID + env["EVAL_METRICS_TEMPLATE"] = MetricsPayloadTemplate(j.MetricsInvocationID) + } result, err := rt.Exec(ctx, command, evalruntime.ExecOptions{ Cwd: cwd, TimeoutSec: int(timeout.Seconds()), - Env: map[string]string{ - "EVAL_TRANSCRIPT_PATH": transcriptEnv, - "EVAL_FINAL_MESSAGE": in.FinalMessage, - "EVAL_EXIT_CODE": strconv.Itoa(in.ExitCode), - }, + Env: env, }) evidence := strings.TrimSpace(result.Stdout) debugInfo := strings.TrimSpace(result.Stderr) if err != nil { if ctx.Err() == context.DeadlineExceeded { + // A timeout does not invalidate a trustworthy result: if this + // invocation's own schema-valid file was written before the deadline, + // the scores are collected while the existing verdict mapping and the + // timeout record are preserved. if j.TimeoutIsError { timeoutErr := fmt.Errorf("script timed out after %s: %w", timeout, ctx.Err()) - return NewErrorResult(timeoutErr, in.TurnsExecuted, in.TurnsTotal), timeoutErr + res := NewErrorResult(timeoutErr, in.TurnsExecuted, in.TurnsTotal) + j.attachMetrics(res, in, true) + return res, timeoutErr } - return j.buildResult(false, "script timed out after "+timeout.String(), debugInfo, in), nil + res := j.buildResult(false, "script timed out after "+timeout.String(), debugInfo, in) + j.attachMetrics(res, in, true) + return res, nil } if result.ExitCode == 0 { return nil, fmt.Errorf("script execution failed: %w", err) @@ -142,13 +170,18 @@ func (j *ScriptJudge) evaluateInRuntime(ctx context.Context, rt evalruntime.Runt if detail == "" { detail = fmt.Sprintf("script exited with code %d", result.ExitCode) } - return j.buildResult(false, detail, debugInfo, in), nil + res := j.buildResult(false, detail, debugInfo, in) + // A nonzero exit never collects scores as a valid result (row P5): the file + // stays a diagnostic artifact only. + return res, nil } if evidence == "" { evidence = "script passed (exit code 0)" } - return j.buildResult(true, evidence, debugInfo, in), nil + res := j.buildResult(true, evidence, debugInfo, in) + j.attachMetrics(res, in, false) + return res, nil } // buildResult creates a judge Result from script execution outcome. @@ -165,3 +198,43 @@ func (j *ScriptJudge) buildResult(passed bool, evidence, debugInfo string, in In return NewResult([]AssertionResult{assertion}, in.TurnsExecuted, in.TurnsTotal) } + +// attachMetrics collects this invocation's structured metric result file and +// materializes one slot per planned registry entry onto the judge result. +// +// The rules it enforces: the file must be a regular file at exactly the assigned +// path, bound to this invocation id, within the size limit, and schema-valid. A +// whole-envelope defect poisons every slot; a single bad entry poisons only its +// own slot and independently valid siblings stay AVAILABLE. Absence of a file +// after a completed run leaves slots MISSING rather than zero. +func (j *ScriptJudge) attachMetrics(res *Result, in Input, prevented bool) { + if res == nil || len(j.MetricSlots) == 0 { + return + } + if j.MetricsResultPath == "" { + res.Metrics = MaterializeSlots("script", j.MetricSlots, nil, ProblemUnboundResult, prevented) + return + } + info, err := os.Stat(j.MetricsResultPath) + if errors.Is(err, os.ErrNotExist) && prevented { + // Nothing was ever produced because evaluation was interrupted: the slots + // are NOT_PRODUCED, not a rejected payload. + res.Metrics = MaterializeSlots("script", j.MetricSlots, nil, ProblemNone, true) + return + } + if err != nil || info.IsDir() { + problem := ProblemUnboundResult + if err != nil && !os.IsNotExist(err) { + problem = ProblemUnparseable + } + res.Metrics = MaterializeSlots("script", j.MetricSlots, nil, problem, false) + return + } + raw, err := os.ReadFile(j.MetricsResultPath) + if err != nil { + res.Metrics = MaterializeSlots("script", j.MetricSlots, nil, ProblemUnparseable, prevented) + return + } + emitted, payloadProblem := ParseMetricsPayload(raw, j.MetricsInvocationID) + res.Metrics = MaterializeSlots("script", j.MetricSlots, emitted, payloadProblem, false) +} diff --git a/internal/judge/script_metrics_test.go b/internal/judge/script_metrics_test.go new file mode 100644 index 00000000..ad47b817 --- /dev/null +++ b/internal/judge/script_metrics_test.go @@ -0,0 +1,473 @@ +package judge + +import ( + "context" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/alibaba/skill-up/internal/config" + "github.com/alibaba/skill-up/internal/platform" +) + +// These tests drive the real ScriptJudge.Evaluate path: a script is written to +// disk, executed as a child process, and the invocation-bound metrics file it +// produces is collected, validated and materialized. Nothing here hand-builds an +// envelope. + +type metricsScriptCase struct { + name string + posix string + windows string + // registry configures the planned slots for this judge. + registry []config.MetricConfig + // expectMetrics is keyed by metric id. + expectAvailability map[string]Availability + expectValues map[string]float64 + expectQuality Quality + expectPlanned int + expectNilEnvelope bool + expectReasons map[string]string +} + +func (tc metricsScriptCase) scriptName() string { + if runtime.GOOS == platform.GOOSWindows { + return "metrics.ps1" + } + return "metrics.sh" +} + +func (tc metricsScriptCase) body() string { + if runtime.GOOS == platform.GOOSWindows { + return tc.windows + } + return tc.posix +} + +// writeMetricsScript puts the producer script in dir and returns its path. +func writeMetricsScript(t *testing.T, dir string, tc metricsScriptCase) string { + t.Helper() + path := filepath.Join(dir, tc.scriptName()) + if err := os.WriteFile(path, []byte(tc.body()), 0o755); err != nil { + t.Fatalf("write script: %v", err) + } + return path +} + +func runMetricsScriptCase(t *testing.T, tc metricsScriptCase) *Result { + t.Helper() + dir := t.TempDir() + script := writeMetricsScript(t, dir, tc) + resultPath := filepath.Join(dir, "metrics-result.json") + j := &ScriptJudge{ + ScriptPath: script, + MetricSlots: SlotsFromConfig(tc.registry), + MetricsResultPath: resultPath, + MetricsInvocationID: "inv-42", + } + res, err := j.Evaluate(context.Background(), Input{WorkspacePath: dir, FinalMessage: "output", ExitCode: 0}) + if err != nil { + t.Fatalf("Evaluate returned %v", err) + } + return res +} + +const ( + posixWriteTwoMetrics = `#!/bin/sh +cat > "$EVAL_METRICS_RESULT_PATH" < "$EVAL_METRICS_RESULT_PATH" < "$EVAL_METRICS_RESULT_PATH" < "$EVAL_METRICS_RESULT_PATH" < "$EVAL_METRICS_RESULT_PATH" <&2 +exit 3 +` + windowsValidFileThenFail = "$body = '{\"schema_version\":1,\"invocation_id\":\"' + $env:EVAL_METRICS_INVOCATION_ID + '\",\"metrics\":[{\"metric_id\":\"correctness\",\"availability\":\"AVAILABLE\",\"value\":0.9,\"evidence\":[\"ok\"]}]}'\n" + + "Set-Content -LiteralPath $env:EVAL_METRICS_RESULT_PATH -Value $body\nexit 3\n" + + posixDeclinesOne = `#!/bin/sh +cat > "$EVAL_METRICS_RESULT_PATH" <= 0.5, so the decision is evaluable. + expectAvailability: map[string]Availability{ + "correctness": Available, "completeness": Declined, + }, + expectValues: map[string]float64{"correctness": 0.8}, + expectQuality: QualityMet, + expectPlanned: 2, + }, + { + name: "a threshold on the declined metric makes the decision unevaluable", + posix: posixDeclinesOne, + windows: windowsDeclinesOne, + registry: []config.MetricConfig{{ID: "correctness", Required: true}, {ID: "completeness", MinScore: &minScore}}, + expectAvailability: map[string]Availability{ + "correctness": Available, "completeness": Declined, + }, + expectQuality: QualityUnknown, + expectPlanned: 2, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + res := runMetricsScriptCase(t, tc) + if tc.expectNilEnvelope { + if res.Metrics != nil { + t.Fatalf("expected no envelope after a nonzero exit, got %+v", res.Metrics) + } + return + } + if res.Metrics == nil { + t.Fatal("expected a metrics envelope from the collected file") + } + env := res.Metrics + if env.Coverage.Planned != tc.expectPlanned { + t.Fatalf("planned = %d, want %d", env.Coverage.Planned, tc.expectPlanned) + } + if len(env.Metrics) != tc.expectPlanned { + t.Fatalf("materialized %d slots, want %d", len(env.Metrics), tc.expectPlanned) + } + if env.Quality != tc.expectQuality { + t.Fatalf("quality = %s, want %s", env.Quality, tc.expectQuality) + } + for _, metric := range env.Metrics { + want, ok := tc.expectAvailability[metric.MetricID] + if !ok { + t.Fatalf("unexpected slot %q", metric.MetricID) + } + if metric.Availability != want { + t.Fatalf("%s availability = %s, want %s (reason %q)", metric.MetricID, metric.Availability, want, metric.Reason) + } + if want != Available && metric.Value != nil { + t.Fatalf("%s carries value %v while unavailable: no fabricated zero allowed", metric.MetricID, *metric.Value) + } + if reason, ok := tc.expectReasons[metric.MetricID]; ok && !strings.Contains(metric.Reason, reason) { + t.Fatalf("%s reason = %q, want substring %q", metric.MetricID, metric.Reason, reason) + } + } + for id, wantValue := range tc.expectValues { + for _, metric := range env.Metrics { + if metric.MetricID == id { + if metric.Value == nil || *metric.Value != wantValue { + t.Fatalf("%s value = %v, want %v", id, metric.Value, wantValue) + } + } + } + } + // The verdict mapping itself must not change because metrics exist. + if res.Status != StatusPass && !tc.nameIsFailing() { + t.Fatalf("status = %s, expected the legacy PASS for a zero-exit script", res.Status) + } + }) + } +} + +func (tc metricsScriptCase) nameIsFailing() bool { + return strings.Contains(tc.name, "nonzero exit") +} + +func TestScriptJudgeWithoutRegistryKeepsLegacyShape(t *testing.T) { + dir := t.TempDir() + name, body := "check.sh", "#!/bin/sh\necho ok\nexit 0\n" + if runtime.GOOS == platform.GOOSWindows { + name, body = "check.ps1", "Write-Output ok\nexit 0\n" + } + script := writeMetricsScript(t, dir, metricsScriptCase{posix: body, windows: body}) + _ = name + j := &ScriptJudge{ScriptPath: script} + res, err := j.Evaluate(context.Background(), Input{WorkspacePath: dir}) + if err != nil { + t.Fatalf("Evaluate: %v", err) + } + if res.Metrics != nil { + t.Fatalf("a judge with no registry gained an envelope: %+v", res.Metrics) + } + if res.Status != StatusPass { + t.Fatalf("status = %s, want legacy PASS", res.Status) + } +} + +func TestParseMetricsPayloadRejections(t *testing.T) { + cases := []struct { + name string + raw string + invocation string + wantProblem PayloadProblem + wantEmissions int + }{ + {"valid empty metrics", `{"schema_version":1,"invocation_id":"i","metrics":[]}`, "i", ProblemNone, 0}, + {"top level not an object", `[]`, "i", ProblemTopLevelShape, 0}, + {"garbage", `not json`, "i", ProblemUnparseable, 0}, + {"duplicate nested key", `{"schema_version":1,"invocation_id":"i","metrics":[{"metric_id":"a","metric_id":"a"}]}`, "i", ProblemDuplicateJSONKey, 0}, + {"unknown top level field", `{"schema_version":1,"invocation_id":"i","metrics":[],"extra":1}`, "i", ProblemUnknownTopField, 0}, + {"wrong schema version", `{"schema_version":2,"invocation_id":"i","metrics":[]}`, "i", ProblemSchemaVersion, 0}, + {"mismatched invocation", `{"schema_version":1,"invocation_id":"other","metrics":[]}`, "i", ProblemUnboundResult, 0}, + {"missing metrics array", `{"schema_version":1,"invocation_id":"i"}`, "i", ProblemTopLevelShape, 0}, + {"trailing second value", `{"schema_version":1,"invocation_id":"i","metrics":[]} {}`, "i", ProblemTopLevelShape, 0}, + {"unknown entry field", `{"schema_version":1,"invocation_id":"i","metrics":[{"metric_id":"a","bogus":1}]}`, "i", ProblemUnknownTopField, 0}, + {"non numeric value stays a slot problem", `{"schema_version":1,"invocation_id":"i","metrics":[{"metric_id":"a","availability":"AVAILABLE","value":"0.5"}]}`, "i", ProblemNone, 1}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + emitted, problem := ParseMetricsPayload([]byte(tc.raw), tc.invocation) + if problem != tc.wantProblem { + t.Fatalf("problem = %q, want %q", problem, tc.wantProblem) + } + if len(emitted) != tc.wantEmissions { + t.Fatalf("emissions = %d, want %d", len(emitted), tc.wantEmissions) + } + }) + } +} + +// Timeout does not invalidate a trustworthy result (#6437995472 §2). +// +// These drive the real DeadlineExceeded branch of ScriptJudge.Evaluate with an +// already-expired parent context, which is deterministic on every platform. The +// result file stands in for the child having written it before the kill: it is +// bound to this exact invocation id, so the collector must accept it while the +// legacy verdict mapping stays unchanged. A full process-kill variant needs POSIX +// process-group semantics and is the same case upstream keeps POSIX-only. +func TestScriptJudgeTimeoutCollectsValidInvocationBoundResult(t *testing.T) { + for _, tc := range []struct { + name string + timeoutError bool + wantStatus Status + wantErr bool + }{ + {"grouped mapping keeps ERROR", true, StatusError, true}, + {"single-judge mapping keeps FAIL", false, StatusFail, false}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + script := writeMetricsScript(t, dir, metricsScriptCase{ + posix: "#!/bin/sh\nsleep 5\nexit 0\n", + windows: "Start-Sleep -Seconds 5\nexit 0\n", + }) + resultPath := filepath.Join(dir, "metrics-result.json") + payload := `{"schema_version":1,"invocation_id":"inv-42","metrics":[{"metric_id":"correctness","availability":"AVAILABLE","value":0.8,"evidence":["written before the deadline"]},{"metric_id":"completeness","availability":"AVAILABLE","value":0,"evidence":["explicit zero"]}]}` + if err := os.WriteFile(resultPath, []byte(payload), 0o600); err != nil { + t.Fatal(err) + } + expired, cancelExpired := context.WithTimeout(context.Background(), 0) + t.Cleanup(cancelExpired) + j := &ScriptJudge{ + ScriptPath: script, + TimeoutIsError: tc.timeoutError, + MetricSlots: SlotsFromConfig(twoMetricRegistry()), + MetricsResultPath: resultPath, + MetricsInvocationID: "inv-42", + } + res, err := j.Evaluate(expired, Input{WorkspacePath: dir}) + if tc.wantErr && err == nil { + t.Fatal("expected the timeout error to still be returned") + } + if !tc.wantErr && err != nil { + t.Fatalf("unexpected error %v", err) + } + if res == nil { + t.Fatal("expected a result envelope even on timeout") + } + if res.Status != tc.wantStatus { + t.Fatalf("status = %s, want %s (verdict mapping must not change)", res.Status, tc.wantStatus) + } + if res.Metrics == nil { + t.Fatal("a timeout discarded a collectible valid result") + } + if res.Metrics.Coverage.Available != 2 || res.Metrics.Quality != QualityNotConfigured { + t.Fatalf("envelope = %+v, want both scores retained", res.Metrics) + } + for _, metric := range res.Metrics.Metrics { + if metric.Availability != Available || metric.Value == nil { + t.Fatalf("slot = %+v, an accepted score must stay AVAILABLE", metric) + } + } + }) + } +} + +func TestScriptJudgeTimeoutWithoutCollectedFileLeavesSlotsNotProduced(t *testing.T) { + dir := t.TempDir() + script := writeMetricsScript(t, dir, metricsScriptCase{ + posix: "#!/bin/sh\nsleep 5\nexit 0\n", + windows: "Start-Sleep -Seconds 5\nexit 0\n", + }) + expired, cancelExpired := context.WithTimeout(context.Background(), 0) + t.Cleanup(cancelExpired) + j := &ScriptJudge{ + ScriptPath: script, + MetricSlots: SlotsFromConfig(twoMetricRegistry()), + MetricsResultPath: filepath.Join(dir, "never-written.json"), + MetricsInvocationID: "inv-42", + } + res, err := j.Evaluate(expired, Input{WorkspacePath: dir}) + if res == nil { + t.Fatalf("expected a result, got nil (err=%v)", err) + } + if res.Metrics == nil { + t.Fatal("expected an envelope describing the unproduced slots") + } + for _, metric := range res.Metrics.Metrics { + if metric.Availability != NotProduced || metric.Value != nil { + t.Fatalf("slot = %+v, want NOT_PRODUCED with no value after an interrupted run", metric) + } + } + if res.Metrics.Quality != QualityNotAssessed { + t.Fatalf("quality = %s, want NOT_ASSESSED", res.Metrics.Quality) + } +} diff --git a/internal/report/grouped_evaluation.go b/internal/report/grouped_evaluation.go index c6369030..f9d36689 100644 --- a/internal/report/grouped_evaluation.go +++ b/internal/report/grouped_evaluation.go @@ -11,7 +11,13 @@ type GroupedEvaluation struct { } // GroupAggregation records the case-level decision without flattening criteria. +// +// Completion and Cause are derived group metadata (#317): completion folds over the +// member completions and never replaces Status, and Cause records why the plan ended +// separately from how far it got. type GroupAggregation struct { - Strategy string `json:"strategy"` - Status judge.Status `json:"status"` + Strategy string `json:"strategy"` + Status judge.Status `json:"status"` + Completion judge.Completion `json:"completion_state,omitempty"` + Cause judge.Cause `json:"termination_cause,omitempty"` } diff --git a/internal/report/metrics_persistence_test.go b/internal/report/metrics_persistence_test.go new file mode 100644 index 00000000..c38d5258 --- /dev/null +++ b/internal/report/metrics_persistence_test.go @@ -0,0 +1,114 @@ +package report + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/alibaba/skill-up/internal/judge" +) + +// The named-metrics envelope has to survive the real artifact writer, not just +// struct marshalling: evaluation.json is what downstream tooling reads. + +func plannedSlots() []judge.MetricSlot { + return []judge.MetricSlot{{ID: "correctness", Required: true}, {ID: "completeness"}} +} + +func collectedEnvelope(t *testing.T, judgeID string) *judge.MetricsEnvelope { + t.Helper() + slots := plannedSlots() + payload := `{"schema_version":1,"invocation_id":"inv-1","metrics":[{"metric_id":"correctness","availability":"AVAILABLE","value":0,"evidence":["explicit zero"]},{"metric_id":"completeness","availability":"AVAILABLE","value":0.6,"evidence":["partial"]}]}` + emitted, problem := judge.ParseMetricsPayload([]byte(payload), "inv-1") + if problem != judge.ProblemNone { + t.Fatalf("payload rejected: %s", problem) + } + env := judge.MaterializeSlots(judgeID, slots, emitted, judge.ProblemNone, false) + if env.Coverage.Available != 2 { + t.Fatalf("envelope = %+v", env) + } + return env +} + +func TestGroupedEvaluationPersistsMetricsAndExecution(t *testing.T) { + dir := t.TempDir() + ws, err := NewIterationWorkspace(dir, "test-skill", 1) + if err != nil { + t.Fatalf("workspace: %v", err) + } + env := collectedEnvelope(t, "a") + evaluation := &GroupedEvaluation{ + Version: 1, + JudgeResults: []judge.Outcome{ + { + ID: "a", Type: "script", Status: judge.StatusPass, + Metrics: env, + Execution: &judge.ExecutionMetadata{Stage: judge.StageFinished, Completion: judge.CompletionFinished, Cause: judge.CauseNone}, + }, + { + ID: "b", Type: "script", Status: judge.StatusSkip, SkipReason: "cancelled before invocation", + Metrics: judge.MaterializeSlots("b", plannedSlots(), nil, judge.ProblemNone, true), + Execution: &judge.ExecutionMetadata{ + Stage: judge.StageNotInvoked, Completion: judge.CompletionInterrupted, + Cause: judge.CauseCancellation, CancelSource: judge.CancelOSSignal, + }, + }, + }, + Aggregation: GroupAggregation{ + Strategy: "all_required", Status: judge.StatusError, + Completion: judge.CompletionInterrupted, Cause: judge.CauseCancellation, + }, + } + if err := ws.EnsureDirs([]string{"case-1"}); err != nil { + t.Fatalf("ensure dirs: %v", err) + } + if err := ws.WriteEvaluation("case-1", "with_skill", evaluation); err != nil { + t.Fatalf("write: %v", err) + } + raw, err := os.ReadFile(filepath.Join(ws.ConfigDir("case-1", "with_skill"), "evaluation.json")) + if err != nil { + t.Fatalf("read artifact: %v", err) + } + var back GroupedEvaluation + if err := json.Unmarshal(raw, &back); err != nil { + t.Fatalf("read back: %v", err) + } + if back.Aggregation.Completion != judge.CompletionInterrupted || back.Aggregation.Cause != judge.CauseCancellation { + t.Fatalf("group state lost: %+v", back.Aggregation) + } + first, second := back.JudgeResults[0], back.JudgeResults[1] + if first.Metrics == nil || first.Metrics.Coverage.Planned != 2 || first.Metrics.Coverage.Available != 2 { + t.Fatalf("member a envelope = %+v", first.Metrics) + } + if first.Metrics.Quality != judge.QualityNotConfigured { + t.Fatalf("member a quality = %s, want NOT_CONFIGURED", first.Metrics.Quality) + } + // The explicit zero must survive as zero, not as null and not as absent. + var zeroValue *float64 + for _, metric := range first.Metrics.Metrics { + if metric.MetricID == "correctness" { + zeroValue = metric.Value + } + } + if zeroValue == nil || *zeroValue != 0 { + t.Fatalf("explicit zero not persisted: %v", zeroValue) + } + if second.Execution == nil || second.Execution.Stage != judge.StageNotInvoked || + second.Execution.Completion != judge.CompletionInterrupted || + second.Execution.CancelSource != judge.CancelOSSignal { + t.Fatalf("member b execution = %+v", second.Execution) + } + if second.Metrics == nil || second.Metrics.Coverage.NotProduced != 2 { + t.Fatalf("member b metrics = %+v", second.Metrics) + } + for _, metric := range second.Metrics.Metrics { + if metric.Availability != judge.NotProduced || metric.Value != nil { + t.Fatalf("persisted slot = %+v", metric) + } + } + // Legacy compatibility fields are untouched. + if first.Status != judge.StatusPass || second.Status != judge.StatusSkip || second.SkipReason == "" { + t.Fatalf("compat fields changed: %+v %+v", first, second) + } +} diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 815410dd..c5e85189 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -596,7 +596,12 @@ func (r *Runner) writeCaseArtifacts(ws *report.IterationWorkspace, res *evaluato Version: 1, Gates: res.ExpectResult, JudgeResults: res.JudgeResults, - Aggregation: report.GroupAggregation{Strategy: strategy, Status: res.Status}, + Aggregation: report.GroupAggregation{ + Strategy: strategy, + Status: res.Status, + Completion: res.GroupCompletion, + Cause: res.GroupCause, + }, } if err := ws.WriteEvaluation(res.CaseID, cfgName, grouped); err != nil { return err From c19b401ddcdd5fa08dcfb72ba5f8a223260f5584 Mon Sep 17 00:00:00 2001 From: Max_Guobaixin Date: Wed, 28 Oct 2026 00:42:23 +0800 Subject: [PATCH 04/12] fix(judge): harden the metrics collection channel and acceptance rule Found while reviewing the real data flow rather than the helper functions: - accepted only means the member reached its own final outcome: no stop, or a deadline that still left collected scores. A cancellation arriving mid-run is never acceptance; previously a killed member kept a COMPLETED-looking verdict and its untouched slots read MISSING instead of NOT_PRODUCED - the collector Lstat-s the assigned path and requires a regular, non-symlink file, and checks the size before reading instead of after - an unassigned channel (non-local runtime) reports metrics_channel_unsupported rather than pretending a stale file was found - the real child-process timeout path is now covered on Windows too: the script writes its bound result, outlives the 1s deadline and is killed; long sleeps were shortened because 30s orphans destabilised concurrent package builds - README documents the payload grammar, injected variables, persistence targets and an explicit not-implemented list for this slice --- .../evaluator/metrics_integration_test.go | 90 +++++++++++++++++-- internal/evaluator/metrics_wiring_test.go | 2 +- internal/evaluator/multi_judge.go | 48 +++++++--- internal/judge/README.md | 60 +++++++++++++ internal/judge/metrics.go | 3 + internal/judge/script.go | 34 +++++-- internal/judge/script_metrics_test.go | 77 ++++++++++++++++ internal/judge/script_metrics_timeout_test.go | 87 ++++++++++++++++++ internal/report/grouped_evaluation.go | 8 +- mt_341.txt | 1 + 10 files changed, 380 insertions(+), 30 deletions(-) create mode 100644 internal/judge/script_metrics_timeout_test.go create mode 100644 mt_341.txt diff --git a/internal/evaluator/metrics_integration_test.go b/internal/evaluator/metrics_integration_test.go index 05f9118e..b46ea3db 100644 --- a/internal/evaluator/metrics_integration_test.go +++ b/internal/evaluator/metrics_integration_test.go @@ -45,10 +45,32 @@ exit 0 "Set-Content -LiteralPath $env:EVAL_METRICS_RESULT_PATH -Value $body\nWrite-Output scored\nexit 0\n" ) +// sleepPosix and sleepWindows outlive the judge timeout so a later cancellation +// or deadline lands while the member is still running. +const ( + sleepPosix = "#!/bin/sh\nsleep 5\nexit 0\n" + sleepWindows = "Start-Sleep -Seconds 5\nexit 0\n" +) + func scoreRegistry() []config.MetricConfig { return []config.MetricConfig{{ID: "correctness", Required: true}, {ID: "completeness"}} } +// newScratchRuntime builds a none runtime on a path the test removes best-effort, +// so a lingering Windows grandchild cannot block harness cleanup. +func newScratchRuntime(t *testing.T, dir string) runtime.Runtime { + t.Helper() + rt, err := runtime.NewRuntime(runtime.Config{Type: "none", WorkspaceDir: dir}) + if err != nil { + t.Fatalf("new runtime: %v", err) + } + if err := rt.Create(context.Background()); err != nil { + t.Fatalf("create runtime: %v", err) + } + t.Cleanup(func() { _ = rt.Close() }) + return rt +} + func newScriptRuntime(t *testing.T) runtime.Runtime { t.Helper() rt, err := runtime.NewRuntime(runtime.Config{Type: "none", WorkspaceDir: t.TempDir()}) @@ -152,11 +174,19 @@ func TestExecuteCaseMultiJudgeDerivesGroupStateFromMembers(t *testing.T) { // accepted result must keep its verdict and its scores. func TestExecuteCaseCancellationKeepsAcceptedResult(t *testing.T) { if goruntime.GOOS == windowsGOOS { - t.Skip("POSIX process-group kill semantics; the sleeping child holds the temp dir on Windows") + // The accepted-then-cancelled ordering depends on the first member finishing + // before the timer fires, which is not deterministic under Windows process + // startup costs; the deterministic interruption case is covered by + // TestExecuteCaseCancellationInterruptsRunningMember. + t.Skip("timing-sensitive accepted-retention ordering; POSIX only") + } + scratch, err := os.MkdirTemp("", "skill-up-cancel-") + if err != nil { + t.Fatal(err) } - fast := metricsScript(t, t.TempDir(), pick(posixScore, powershellScore)) - slowDir := t.TempDir() - slow := metricsScript(t, slowDir, "#!/bin/sh\nsleep 5\nexit 0\n") + t.Cleanup(func() { _ = os.RemoveAll(scratch) }) + fast := metricsScript(t, scratch, pick(posixScore, powershellScore)) + slow := metricsScript(t, scratch, pick(sleepPosix, sleepWindows)) five := 30 members := []config.JudgeConfig{ {ID: "fast", Type: "script", ScriptPath: fast, TimeoutSeconds: &five, Metrics: scoreRegistry()}, @@ -167,7 +197,7 @@ func TestExecuteCaseCancellationKeepsAcceptedResult(t *testing.T) { Agent: ag, OutputDir: t.TempDir(), EvalCfg: &config.EvalConfig{Environment: config.Environment{Type: "none"}, Judges: &members}, }) - rt := newScriptRuntime(t) + rt := newScratchRuntime(t, scratch) ctx, cancel := context.WithCancel(context.Background()) go func() { time.Sleep(1500 * time.Millisecond) @@ -282,3 +312,53 @@ func pick(posix, windows string) string { } return posix } + +// TestExecuteCaseCancellationInterruptsRunningMember is deterministic: the case +// context is already cancelled, so the member in flight never reaches a final +// outcome. Its planned slots must read NOT_PRODUCED (never reached) rather than +// MISSING (omitted by a completed judge), while the legacy SKIP/ERROR verdict +// mapping and the member assertion denominator stay unchanged. +func TestExecuteCaseCancellationInterruptsRunningMember(t *testing.T) { + scratch, err := os.MkdirTemp("", "skill-up-interrupt-") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(scratch) }) + script := metricsScript(t, scratch, pick(posixScore, powershellScore)) + limit := 30 + members := []config.JudgeConfig{ + {ID: "only", Type: "script", ScriptPath: script, TimeoutSeconds: &limit, Metrics: scoreRegistry()}, + } + ag := &mockAgent{name: "test", output: "done"} + e := newTestEvaluator(EvalOptions{ + Agent: ag, OutputDir: t.TempDir(), + EvalCfg: &config.EvalConfig{Environment: config.Environment{Type: "none"}, Judges: &members}, + }) + rt := newScratchRuntime(t, scratch) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + result := e.executeCase(ctx, &config.CaseConfig{ID: "interrupt", Input: config.Input{Prompt: "hello"}}, "with_skill", rt, nil) + + if len(result.JudgeResults) != 1 { + t.Fatalf("outcomes = %d", len(result.JudgeResults)) + } + outcome := result.JudgeResults[0] + if outcome.Execution == nil || outcome.Execution.Completion != judge.CompletionInterrupted || + outcome.Execution.Cause != judge.CauseCancellation { + t.Fatalf("execution = %+v", outcome.Execution) + } + if outcome.Metrics == nil { + t.Fatal("expected an envelope for the planned slots") + } + for _, metric := range outcome.Metrics.Metrics { + if metric.Availability != judge.NotProduced || metric.Value != nil { + t.Fatalf("slot = %+v, want NOT_PRODUCED with no value", metric) + } + } + if outcome.Metrics.Quality != judge.QualityNotAssessed { + t.Fatalf("quality = %s, want NOT_ASSESSED", outcome.Metrics.Quality) + } + if result.GroupCompletion != judge.CompletionInterrupted || result.GroupCause != judge.CauseCancellation { + t.Fatalf("group = %s/%s", result.GroupCompletion, result.GroupCause) + } +} diff --git a/internal/evaluator/metrics_wiring_test.go b/internal/evaluator/metrics_wiring_test.go index afb14e4d..032de1b7 100644 --- a/internal/evaluator/metrics_wiring_test.go +++ b/internal/evaluator/metrics_wiring_test.go @@ -275,7 +275,7 @@ func TestMetadataSurvivesNilResult(t *testing.T) { graded := EvalResult{Status: judge.StatusError, Grading: nil, Error: context.Canceled} outcome := JudgeOutcome{ ID: member.ID, Type: member.Type, Status: graded.Status, Result: graded.Grading, - Metrics: gradedMetrics(member, graded, stop), + Metrics: gradedMetrics(member, graded, false), Execution: memberExecution(graded, stop, false), } if outcome.Result != nil { diff --git a/internal/evaluator/multi_judge.go b/internal/evaluator/multi_judge.go index 926d50e4..b4ad3156 100644 --- a/internal/evaluator/multi_judge.go +++ b/internal/evaluator/multi_judge.go @@ -67,13 +67,22 @@ func (e *defaultEvaluator) runMultipleJudges( if graded.Status == judge.StatusError && graded.Error != nil { causes = append(causes, graded.Error) } - // An outcome that reached a final validated state before the stop is - // accepted and retained: a later cancellation or deadline never rewrites - // its verdict, and its scores are not downgraded because evaluation then - // stopped. Only members without such an outcome take the interrupt verdict. - accepted := graded.Grading != nil && graded.Status != judge.StatusError stop := memberStop(ctx, memberCtx, memberTimeout) - outcome.Metrics = gradedMetrics(member, graded, stop) + // A member is accepted only when it reached its own final outcome: no stop + // at all, or a deadline that still left usable collected scores. A + // cancellation that arrived while the member was running never counts as + // acceptance, so its slots stay NOT_PRODUCED instead of looking like an + // omitted metric, and its verdict is not laundered into a completed run. + accepted := false + if graded.Grading != nil && graded.Status != judge.StatusError { + switch stop.Cause { + case judge.CauseNone: + accepted = true + case judge.CauseDeadlineExpiry: + accepted = envelopeHasScores(graded.Grading.Metrics) + } + } + outcome.Metrics = gradedMetrics(member, graded, accepted) outcome.Execution = memberExecution(graded, stop, accepted) if memberErr := recordMemberContextError(memberCtx, &outcome, accepted); memberErr != nil { causes = append(causes, memberErr) @@ -158,10 +167,10 @@ func preventedMetrics(judgeID string, registry []config.MetricConfig) *judge.Met return judge.MaterializeSlots(judgeID, slots, nil, judge.ProblemNone, true) } -// gradedMetrics builds the envelope from what a judge produced. A stop that -// happened after a usable score never discards that score; slots that were never -// reached stay NOT_PRODUCED rather than becoming zero. -func gradedMetrics(member config.JudgeConfig, graded EvalResult, stop judge.Stop) *judge.MetricsEnvelope { +// gradedMetrics builds the envelope from what a judge produced. A score that was +// already collected survives a later stop; slots that were never reached stay +// NOT_PRODUCED rather than becoming zero or looking omitted. +func gradedMetrics(member config.JudgeConfig, graded EvalResult, accepted bool) *judge.MetricsEnvelope { slots := judge.SlotsFromConfig(member.Metrics) if len(slots) == 0 { return nil @@ -169,7 +178,10 @@ func gradedMetrics(member config.JudgeConfig, graded EvalResult, stop judge.Stop if graded.Grading != nil && graded.Grading.Metrics != nil { return graded.Grading.Metrics } - prevented := stop.Cause != judge.CauseNone && graded.Grading == nil + // Anything the execution layer stopped before a final outcome leaves its slots + // NOT_PRODUCED: the metric was never reached, which is not the same as a judge + // omitting or declining a metric in a completed result. + prevented := !accepted return judge.MaterializeSlots(member.ID, slots, nil, judge.ProblemNone, prevented) } @@ -304,6 +316,20 @@ func causeForReason(reason string) judge.Cause { // than a plan terminator, so it does not become the group cause. Members without // execution metadata (legacy judges with no metric registry) contribute nothing, // and an empty result leaves both values unset so the artifact omits them. +// envelopeHasScores reports whether a collected envelope carries at least one +// usable score, which is what lets a deadline-terminated member stay accepted. +func envelopeHasScores(env *judge.MetricsEnvelope) bool { + if env == nil { + return false + } + for _, metric := range env.Metrics { + if metric.Availability == judge.Available { + return true + } + } + return false +} + func foldGroupState(outcomes []JudgeOutcome) (judge.Completion, judge.Cause) { completions := make([]judge.Completion, 0, len(outcomes)) cause := judge.CauseNone diff --git a/internal/judge/README.md b/internal/judge/README.md index 32e32845..71bb5b55 100644 --- a/internal/judge/README.md +++ b/internal/judge/README.md @@ -334,3 +334,63 @@ Rules that the implementation enforces: - Compatibility is untouched: `PASS/FAIL/SKIP/ERROR`, the #309 assertion and member denominators with zero-credit skipped members, the existing script-timeout verdict mapping, and the JSONL event schema all behave as before. + +### Script judge structured metrics result (slice 1 input channel) + +A script judge with a `metrics:` registry is given one private, invocation-bound +result path per attempt. The runner injects: + +| Variable | Meaning | +|---|---| +| `EVAL_METRICS_RESULT_PATH` | the only path the collector reads; written by the script, atomically if possible | +| `EVAL_METRICS_INVOCATION_ID` | random per invocation; must be echoed back inside the payload | +| `EVAL_METRICS_TEMPLATE` | a minimal valid payload the script can copy | + +Accepted grammar (`schema_version` must be `1`): + +```json +{ + "schema_version": 1, + "invocation_id": "", + "metrics": [ + {"metric_id": "correctness", "availability": "AVAILABLE", "value": 0.0, "evidence": ["..."]}, + {"metric_id": "completeness", "availability": "SKIP", "reason": "insufficient evidence"} + ] +} +``` + +`availability` is one of `AVAILABLE`, `MISSING`, `SKIP`, `ERROR`. `AVAILABLE` +requires a finite number in `[0,1]` — an explicit `0` is a real score. Unknown +entry fields, unknown top-level fields, duplicate JSON keys at any level, a wrong +schema version, a mismatched invocation id, a non-object top level, trailing +content and an oversized file are all rejected. + +Rejection semantics: + +- Whole-envelope defect ⇒ every planned slot `ERROR` with that reason; nothing is usable. +- One bad entry ⇒ only that slot `ERROR`; independently valid siblings stay `AVAILABLE`. +- The path is `Lstat`-ed and must be a regular, non-symlink file, and its size is + checked before it is read. A directory, symlink or device is never collected. +- A file that never appeared because evaluation was interrupted ⇒ `NOT_PRODUCED`. +- A file that never appeared after the script exited `0` ⇒ rejected payload, not a + fabricated zero. +- A nonzero exit never collects scores even if a valid file exists; the file stays + an on-disk diagnostic. + +Persistence: the envelope rides on `judge.Result.Metrics` and on each grouped +`judge.Outcome` (`metrics` plus `execution`), so it lands in `report.json` +(single judge) and in `evaluation.json` (`judge_results[]`, plus +`aggregation.completion_state` / `aggregation.termination_cause`). `grading.json` +keeps the Anthropic shape untouched. + +Not implemented in this slice: + +- `agent_judge` numeric metric output (its strict result contract is separate). +- Collection for sandboxed runtimes: only the local `none` environment assigns a + path, so a script judge with metrics elsewhere reports `metrics_channel_unsupported` + rather than pretending a stale file was found. +- Report/HTML/Markdown rendering of metrics, benchmark statistics, per-metric + comparisons, and any addition to the JSONL evaluation-event stream (which needs + its own schema and consumer review). +- Cancel-source attribution beyond an observed process interrupt: anything not + provably signalled is reported as `unknown`, never inferred from error text. diff --git a/internal/judge/metrics.go b/internal/judge/metrics.go index d67513ad..270fe951 100644 --- a/internal/judge/metrics.go +++ b/internal/judge/metrics.go @@ -307,6 +307,9 @@ const ( ProblemSchemaVersion PayloadProblem = "schema_version" ProblemUnboundResult PayloadProblem = "unbound_or_stale_result" ProblemSizeLimit PayloadProblem = "size_limit" + // ProblemChannelUnsupported is not a payload defect: it says the collection channel + // itself was unavailable for this run, so no file could ever have been produced. + ProblemChannelUnsupported PayloadProblem = "metrics_channel_unsupported" ) // MaterializeSlots builds exactly one result per planned slot. Unknown ids are diff --git a/internal/judge/script.go b/internal/judge/script.go index 02d656cd..66110124 100644 --- a/internal/judge/script.go +++ b/internal/judge/script.go @@ -202,32 +202,48 @@ func (j *ScriptJudge) buildResult(passed bool, evidence, debugInfo string, in In // attachMetrics collects this invocation's structured metric result file and // materializes one slot per planned registry entry onto the judge result. // -// The rules it enforces: the file must be a regular file at exactly the assigned -// path, bound to this invocation id, within the size limit, and schema-valid. A +// The rules it enforces: the path must resolve to a regular, non-symlink file of +// an acceptable size, bound to this invocation id, and schema-valid. A // whole-envelope defect poisons every slot; a single bad entry poisons only its -// own slot and independently valid siblings stay AVAILABLE. Absence of a file -// after a completed run leaves slots MISSING rather than zero. +// own slot and independently valid siblings stay AVAILABLE. A file that never +// appeared while evaluation was interrupted yields NOT_PRODUCED, and one that +// never appeared after a completed run yields a rejected payload rather than a +// fabricated zero. func (j *ScriptJudge) attachMetrics(res *Result, in Input, prevented bool) { if res == nil || len(j.MetricSlots) == 0 { return } if j.MetricsResultPath == "" { - res.Metrics = MaterializeSlots("script", j.MetricSlots, nil, ProblemUnboundResult, prevented) + // The channel is not available for this judge (a non-local runtime, or a + // judge built without one). Report that honestly rather than implying a + // stale file was found. + res.Metrics = MaterializeSlots("script", j.MetricSlots, nil, ProblemChannelUnsupported, prevented) return } - info, err := os.Stat(j.MetricsResultPath) + info, err := os.Lstat(j.MetricsResultPath) if errors.Is(err, os.ErrNotExist) && prevented { // Nothing was ever produced because evaluation was interrupted: the slots // are NOT_PRODUCED, not a rejected payload. res.Metrics = MaterializeSlots("script", j.MetricSlots, nil, ProblemNone, true) return } - if err != nil || info.IsDir() { + if err != nil { problem := ProblemUnboundResult - if err != nil && !os.IsNotExist(err) { + if !os.IsNotExist(err) { problem = ProblemUnparseable } - res.Metrics = MaterializeSlots("script", j.MetricSlots, nil, problem, false) + res.Metrics = MaterializeSlots("script", j.MetricSlots, nil, problem, prevented) + return + } + if !info.Mode().IsRegular() { + // A symlink, directory or device node is never a collected result. + res.Metrics = MaterializeSlots("script", j.MetricSlots, nil, ProblemUnboundResult, prevented) + return + } + if info.Size() > MaxMetricsPayloadBytes { + // Bound the read before it happens: an oversized file must not be loaded + // into memory just to be rejected afterwards. + res.Metrics = MaterializeSlots("script", j.MetricSlots, nil, ProblemSizeLimit, prevented) return } raw, err := os.ReadFile(j.MetricsResultPath) diff --git a/internal/judge/script_metrics_test.go b/internal/judge/script_metrics_test.go index ad47b817..2b563c86 100644 --- a/internal/judge/script_metrics_test.go +++ b/internal/judge/script_metrics_test.go @@ -471,3 +471,80 @@ func TestScriptJudgeTimeoutWithoutCollectedFileLeavesSlotsNotProduced(t *testing t.Fatalf("quality = %s, want NOT_ASSESSED", res.Metrics.Quality) } } + +func TestScriptJudgeRejectsOversizedAndNonRegularResults(t *testing.T) { + t.Run("oversized payload is rejected before it is read", func(t *testing.T) { + dir := t.TempDir() + script := writeMetricsScript(t, dir, metricsScriptCase{ + posix: "#!/bin/sh\nexit 0\n", windows: "exit 0\n", + }) + resultPath := filepath.Join(dir, "metrics-result.json") + // A padding string plus valid JSON shape: over the collector limit. + padding := strings.Repeat("x", MaxMetricsPayloadBytes+16) + payload := `{"schema_version":1,"invocation_id":"inv-42","metrics":[{"metric_id":"correctness","availability":"AVAILABLE","value":0.5,"evidence":["` + padding + `"]}]}` + if err := os.WriteFile(resultPath, []byte(payload), 0o600); err != nil { + t.Fatal(err) + } + j := &ScriptJudge{ + ScriptPath: script, MetricSlots: SlotsFromConfig(twoMetricRegistry()), + MetricsResultPath: resultPath, MetricsInvocationID: "inv-42", + } + res, err := j.Evaluate(context.Background(), Input{WorkspacePath: dir}) + if err != nil { + t.Fatalf("Evaluate: %v", err) + } + for _, metric := range res.Metrics.Metrics { + if metric.Availability != Errored || metric.Reason != string(ProblemSizeLimit) { + t.Fatalf("slot = %+v, want ERROR/%s", metric, ProblemSizeLimit) + } + } + }) + + t.Run("a non-regular path is never collected", func(t *testing.T) { + dir := t.TempDir() + script := writeMetricsScript(t, dir, metricsScriptCase{ + posix: "#!/bin/sh\nexit 0\n", windows: "exit 0\n", + }) + resultPath := filepath.Join(dir, "metrics-result.json") + if err := os.Mkdir(resultPath, 0o755); err != nil { + t.Fatal(err) + } + j := &ScriptJudge{ + ScriptPath: script, MetricSlots: SlotsFromConfig(twoMetricRegistry()), + MetricsResultPath: resultPath, MetricsInvocationID: "inv-42", + } + res, err := j.Evaluate(context.Background(), Input{WorkspacePath: dir}) + if err != nil { + t.Fatalf("Evaluate: %v", err) + } + for _, metric := range res.Metrics.Metrics { + if metric.Availability != Errored || metric.Reason != string(ProblemUnboundResult) { + t.Fatalf("slot = %+v, want ERROR/%s", metric, ProblemUnboundResult) + } + } + }) + + t.Run("an unavailable channel is reported as such, not as a stale file", func(t *testing.T) { + dir := t.TempDir() + script := writeMetricsScript(t, dir, metricsScriptCase{ + posix: "#!/bin/sh\necho ok\nexit 0\n", windows: "Write-Output ok\nexit 0\n", + }) + j := &ScriptJudge{ + ScriptPath: script, MetricSlots: SlotsFromConfig(twoMetricRegistry()), + // MetricsResultPath deliberately empty: a non-local runtime never gets one. + MetricsInvocationID: "inv-42", + } + res, err := j.Evaluate(context.Background(), Input{WorkspacePath: dir}) + if err != nil { + t.Fatalf("Evaluate: %v", err) + } + if res.Status != StatusPass { + t.Fatalf("legacy verdict changed: %s", res.Status) + } + for _, metric := range res.Metrics.Metrics { + if metric.Availability != Errored || metric.Reason != string(ProblemChannelUnsupported) { + t.Fatalf("slot = %+v, want ERROR/%s", metric, ProblemChannelUnsupported) + } + } + }) +} diff --git a/internal/judge/script_metrics_timeout_test.go b/internal/judge/script_metrics_timeout_test.go new file mode 100644 index 00000000..ee196485 --- /dev/null +++ b/internal/judge/script_metrics_timeout_test.go @@ -0,0 +1,87 @@ +package judge + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" +) + +// TestScriptJudgeRealSubprocessTimeoutCollectsResult runs a real child process +// that writes its invocation-bound result and then outlives the judge timeout, so +// the collector sees a file the child actually produced before being killed. +// +// Scratch paths deliberately live outside t.TempDir and the child's working +// directory is the shared temp root: a lingering Windows grandchild otherwise +// holds the directory the harness wants to delete, which is exactly why +// TestScriptJudge_Timeout stays POSIX-only upstream. Removal is best-effort. + +const realTimeoutPayload = `{"schema_version":1,"invocation_id":"inv-42","metrics":[{"metric_id":"correctness","availability":"AVAILABLE","value":0.25,"evidence":["written before the kill"]},{"metric_id":"completeness","availability":"AVAILABLE","value":0,"evidence":["explicit zero"]}]}` + +const realTimeoutPosix = "#!/bin/sh\n" + + "cat > \"$EVAL_METRICS_RESULT_PATH\" < Date: Wed, 28 Oct 2026 00:42:32 +0800 Subject: [PATCH 05/12] chore: drop a scratch merge-check output committed by mistake --- mt_341.txt | 1 - 1 file changed, 1 deletion(-) delete mode 100644 mt_341.txt diff --git a/mt_341.txt b/mt_341.txt deleted file mode 100644 index c0088bc9..00000000 --- a/mt_341.txt +++ /dev/null @@ -1 +0,0 @@ -0cc91588eafba6dda93e481b2189baeb8f62b422 From 6c697acb9a76f15263c55d1de62d677d5306030f Mon Sep 17 00:00:00 2001 From: Baixin Guo <141702260+GBX-Max1220@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:27:40 +0800 Subject: [PATCH 06/12] fix(judge): address named metrics review findings --- internal/config/metrics_validation_test.go | 14 +- internal/config/validator.go | 8 +- internal/evaluator/cancellation.go | 6 +- internal/evaluator/evaluator.go | 16 + internal/evaluator/metrics_finalize.go | 131 ++++++++ .../evaluator/metrics_integration_test.go | 294 ++++++++++++++---- internal/evaluator/metrics_wiring_test.go | 137 ++++---- internal/evaluator/multi_judge.go | 216 ++++++++----- internal/judge/README.md | 26 +- internal/judge/factory.go | 1 + internal/judge/judge.go | 4 + internal/judge/metrics.go | 112 +++++-- internal/judge/metrics_payload.go | 3 +- internal/judge/metrics_test.go | 122 ++++---- internal/judge/script.go | 62 ++-- internal/judge/script_metrics_test.go | 180 ++++++----- internal/judge/script_metrics_timeout_test.go | 6 +- internal/report/grouped_evaluation.go | 10 +- internal/report/metrics_persistence_test.go | 22 +- internal/report/reporter.go | 46 +-- .../runner/metrics_single_persistence_test.go | 50 +++ internal/runner/runner.go | 6 +- 22 files changed, 1031 insertions(+), 441 deletions(-) create mode 100644 internal/evaluator/metrics_finalize.go create mode 100644 internal/runner/metrics_single_persistence_test.go diff --git a/internal/config/metrics_validation_test.go b/internal/config/metrics_validation_test.go index 3004a696..091caf66 100644 --- a/internal/config/metrics_validation_test.go +++ b/internal/config/metrics_validation_test.go @@ -5,8 +5,6 @@ import ( "testing" ) -func f(v float64) *float64 { return &v } - func TestValidateJudgeMetrics(t *testing.T) { cases := []struct { name string @@ -17,27 +15,27 @@ func TestValidateJudgeMetrics(t *testing.T) { {name: "absent registry is valid legacy configuration", metrics: nil, wantOK: true}, { name: "two well-formed slots", - metrics: []MetricConfig{{ID: "correctness", Required: true}, {ID: "completeness", MinScore: f(0.7)}}, + metrics: []MetricConfig{{ID: "correctness", Required: true}, {ID: "completeness", MinScore: new(0.7)}}, wantOK: true, }, { name: "a zero threshold is distinguishable from an absent one and stays valid", - metrics: []MetricConfig{{ID: "correctness", MinScore: f(0)}}, + metrics: []MetricConfig{{ID: "correctness", MinScore: new(0.0)}}, wantOK: true, }, { name: "upper bound is inclusive", - metrics: []MetricConfig{{ID: "correctness", MinScore: f(1)}}, + metrics: []MetricConfig{{ID: "correctness", MinScore: new(1.0)}}, wantOK: true, }, { name: "threshold above the score range is rejected", - metrics: []MetricConfig{{ID: "correctness", MinScore: f(1.1)}}, + metrics: []MetricConfig{{ID: "correctness", MinScore: new(1.1)}}, wantErrs: []string{"min_score must be between 0.0 and 1.0"}, }, { name: "negative threshold is rejected", - metrics: []MetricConfig{{ID: "correctness", MinScore: f(-0.5)}}, + metrics: []MetricConfig{{ID: "correctness", MinScore: new(-0.5)}}, wantErrs: []string{"min_score must be between 0.0 and 1.0"}, }, { @@ -82,7 +80,7 @@ func TestMetricRegistryKeepsAnAbsentThresholdDistinctFromZero(t *testing.T) { // derivation reports as NOT_CONFIGURED rather than UNKNOWN; a zero threshold is // a real requirement. The two must not collapse during configuration loading. absent := MetricConfig{ID: "correctness"} - zero := MetricConfig{ID: "completeness", MinScore: f(0)} + zero := MetricConfig{ID: "completeness", MinScore: new(0.0)} if absent.MinScore != nil { t.Fatalf("unset threshold must stay nil, got %v", *absent.MinScore) } diff --git a/internal/config/validator.go b/internal/config/validator.go index 1fd310db..586f3973 100644 --- a/internal/config/validator.go +++ b/internal/config/validator.go @@ -643,7 +643,7 @@ func validateJudgeList(members *[]JudgeConfig) []string { // validateJudgeMetrics checks the planned named-metric registry. Ids are stable // configuration identifiers, must be unique, and a configured threshold has to sit // inside the declared [0,1] score range. -func validateJudgeMetrics(path string, metrics []MetricConfig) []string { +func validateJudgeMetrics(configPath string, metrics []MetricConfig) []string { if len(metrics) == 0 { return nil } @@ -651,14 +651,14 @@ func validateJudgeMetrics(path string, metrics []MetricConfig) []string { seen := make(map[string]bool, len(metrics)) for i, metric := range metrics { if !judgeIDPattern.MatchString(metric.ID) { - errs = append(errs, fmt.Sprintf("%s.metrics[%d].id must be a safe lowercase identifier", path, i)) + errs = append(errs, fmt.Sprintf("%s.metrics[%d].id must be a safe lowercase identifier", configPath, i)) } if seen[metric.ID] { - errs = append(errs, fmt.Sprintf("%s.metrics[%d].id %q is duplicated", path, i, metric.ID)) + errs = append(errs, fmt.Sprintf("%s.metrics[%d].id %q is duplicated", configPath, i, metric.ID)) } seen[metric.ID] = true if metric.MinScore != nil && (*metric.MinScore < 0.0 || *metric.MinScore > 1.0) { - errs = append(errs, fmt.Sprintf("%s.metrics[%d].min_score must be between 0.0 and 1.0", path, i)) + errs = append(errs, fmt.Sprintf("%s.metrics[%d].min_score must be between 0.0 and 1.0", configPath, i)) } } return errs diff --git a/internal/evaluator/cancellation.go b/internal/evaluator/cancellation.go index a1ed10b1..fecba785 100644 --- a/internal/evaluator/cancellation.go +++ b/internal/evaluator/cancellation.go @@ -10,8 +10,10 @@ import ( // unattributable cancellation stays "unknown" instead of being guessed as // user-initiated. -type signalObserverKey struct{} -type deadlineSourceKey struct{} +type ( + signalObserverKey struct{} + deadlineSourceKey struct{} +) // SignalObserver records whether this process actually observed an interrupt. // The flag is set by the signal handler path, never inferred afterwards. diff --git a/internal/evaluator/evaluator.go b/internal/evaluator/evaluator.go index 036eb821..5fb20203 100644 --- a/internal/evaluator/evaluator.go +++ b/internal/evaluator/evaluator.go @@ -117,6 +117,11 @@ type EvalResult struct { GroupCompletion judge.Completion GroupCause judge.Cause + // Execution and Metrics preserve the named-metric contract on single-judge + // paths, including outcomes that have no grading result. + Execution *judge.ExecutionMetadata `json:"execution,omitempty"` + Metrics *judge.MetricsEnvelope `json:"metrics,omitempty"` + // JudgeSession is the separate agent session used by agent_judge. It is // preserved even when the judge fails to produce a valid grading result. JudgeSession *agent.SessionResult @@ -130,6 +135,10 @@ type EvalResult struct { // Error holds any execution error. Error error + // judgeInvoked records whether this case reached Judge.Evaluate. It is used + // only while deriving execution metadata for configured metric slots. + judgeStage judge.Stage + // Configuration is "with_skill" or "without_skill". Configuration string } @@ -458,6 +467,7 @@ func (e *defaultEvaluator) executeCaseOnce(ctx context.Context, caseCfg *config. result.Status = judge.StatusError result.Error = fmt.Errorf("agent %s preflight failed: %w", runAgent.Name(), err) result.Configuration = configName + e.recordSkippedJudgesForCase(&result, caseCfg, "agent_execution_failed") return result } } else { @@ -467,6 +477,7 @@ func (e *defaultEvaluator) executeCaseOnce(ctx context.Context, caseCfg *config. result.Status = judge.StatusError result.Error = err result.Configuration = configName + e.recordSkippedJudgesForCase(&result, caseCfg, "agent_execution_failed") return result } defer func() { _ = rt.Close() }() @@ -632,6 +643,7 @@ func (e *defaultEvaluator) evaluateCaseSession( if plan.Multi { result.Grading, _ = judge.DefaultOutcomeAggregator().Aggregate(result.JudgeResults, result.Turns, turnsTotal) result.Grading.Status = judge.StatusFail + result.GroupCompletion, result.GroupCause = foldGroupState(result.JudgeResults) } return *result } @@ -646,6 +658,9 @@ func (e *defaultEvaluator) evaluateCaseSession( finalResult = e.runMultipleJudges(ctx, rt, caseCfg, configName, plan, turnsTotal, runAgent, judgeInput, result) } else { finalResult = e.runJudgePhase(ctx, rt, caseCfg, configName, judgeCfg, turnsTotal, runAgent, judgeInput, result, "judge/run", false) + if len(plan.Judges) > 0 { + finalizeJudgeMetrics(ctx, &finalResult, plan.Judges[0]) + } } if !plan.Multi && len(expectAssertions) > 0 && finalResult.Grading != nil { finalResult.Grading.AssertionResults = append(expectAssertions, finalResult.Grading.AssertionResults...) @@ -762,6 +777,7 @@ func (e *defaultEvaluator) runJudgePhaseWithSpan( defer cleanupScript() e.configureMetricsChannel(ctx, j, configName, caseCfg, artifactRoot) + result.judgeStage = judge.StageStarted grading, err := j.Evaluate(ctx, judgeInput) if err != nil { result.Grading = grading diff --git a/internal/evaluator/metrics_finalize.go b/internal/evaluator/metrics_finalize.go new file mode 100644 index 00000000..ce69341c --- /dev/null +++ b/internal/evaluator/metrics_finalize.go @@ -0,0 +1,131 @@ +package evaluator + +import ( + "context" + "errors" + + "github.com/alibaba/skill-up/internal/config" + "github.com/alibaba/skill-up/internal/judge" +) + +func (e *defaultEvaluator) recordSkippedJudgesForCase(result *EvalResult, caseCfg *config.CaseConfig, reason string) { + plan, err := config.ResolveJudgePlan(e.evalCfg, caseCfg) + if err == nil { + recordSkippedJudges(result, plan, reason) + } +} + +// finalizeJudgeMetrics applies the opt-in named-metric contract and preserves +// slots and execution facts outside grading for early returns and nil results. +func finalizeJudgeMetrics(ctx context.Context, result *EvalResult, cfg config.JudgeConfig) { + planned := judge.SlotsFromConfig(cfg.Metrics) + if len(planned) == 0 { + return + } + stage := result.judgeStage + if stage == "" { + stage = judge.StageNotInvoked + } + env := caseMetricsEnvelope(ctx, result, cfg, planned, stage) + result.Metrics = env + if result.Grading != nil { + result.Grading.Metrics = env + } + + production := judge.MetricProduced + if ctx.Err() != nil || (result.Grading != nil && result.Grading.Execution != nil && result.Grading.Execution.Completion == judge.CompletionInterrupted) { + production = judge.MetricPrevented + } + contractError := metricContractError(env, planned, production) + switch { + case contractError != "": + applyMetricError(result, contractError) + case env.Quality == judge.QualityNotMet: + applyMetricFailure(result) + } + result.Execution = deriveSingleExecution(ctx, result, env, planned, contractError, stage) +} + +func caseMetricsEnvelope(ctx context.Context, result *EvalResult, cfg config.JudgeConfig, planned []judge.MetricSlot, stage judge.Stage) *judge.MetricsEnvelope { + if result.Grading != nil && result.Grading.Metrics != nil { + return result.Grading.Metrics + } + production := judge.MetricProduced + if stage != judge.StageStarted || ctx.Err() != nil { + production = judge.MetricPrevented + } + problem := judge.ProblemNone + if cfg.Type != "script" && stage == judge.StageStarted { + problem = judge.ProblemChannelUnsupported + } + return judge.MaterializeSlots(cfg.ID, planned, nil, problem, production) +} + +func metricContractError(env *judge.MetricsEnvelope, planned []judge.MetricSlot, production judge.MetricProduction) string { + byID := make(map[string]judge.MetricResult, len(env.Metrics)) + for _, metric := range env.Metrics { + byID[metric.MetricID] = metric + if metric.Availability == judge.Errored { + return "named metric " + metric.MetricID + " is invalid: " + metric.Reason + } + } + if production == judge.MetricPrevented { + return "" + } + for _, slot := range planned { + metric, ok := byID[slot.ID] + if (!ok || metric.Availability != judge.Available) && (slot.Required || slot.MinScore != nil) { + return "required named metric unavailable: " + slot.ID + } + } + return "" +} + +func applyMetricError(result *EvalResult, message string) { + result.Status = judge.StatusError + result.Error = errors.New(message) + if result.Grading != nil { + result.Grading.Status = judge.StatusError + result.Grading.ErrorReason = &message + appendMetricFailure(result.Grading, message) + } +} + +func applyMetricFailure(result *EvalResult) { + result.Status = judge.StatusFail + if result.Grading != nil { + result.Grading.Status = judge.StatusFail + appendMetricFailure(result.Grading, "one or more configured score thresholds were not met") + } +} + +func deriveSingleExecution(ctx context.Context, result *EvalResult, env *judge.MetricsEnvelope, planned []judge.MetricSlot, contractError string, stage judge.Stage) *judge.ExecutionMetadata { + switch { + case ctx.Err() != nil: + stop := judge.ClassifyStop(ctx.Err(), judge.StopObservation{DeadlineSource: DeadlineSource(ctx), SignalObserved: SignalObserved(ctx)}) + execution := interruptedExecution(stage, stop) + if stop.Cause == judge.CauseDeadlineExpiry { + execution.Timeout = &judge.TimeoutMetadata{Occurred: true, Source: stop.DeadlineSource} + } + return execution + case result.Grading != nil && result.Grading.Execution != nil: + return result.Grading.Execution + case contractError != "": + return &judge.ExecutionMetadata{Stage: judge.StageFinished, Completion: completionForContract(env, planned), Cause: judge.CauseContractError} + case result.Error != nil: + return &judge.ExecutionMetadata{Stage: stage, Completion: judge.CompletionFailed, Cause: judge.CauseExecutionError, Detail: result.Error.Error()} + default: + return &judge.ExecutionMetadata{Stage: judge.StageFinished, Completion: completionForContract(env, planned), Cause: judge.CauseNone} + } +} + +func appendMetricFailure(result *judge.Result, evidence string) { + result.AssertionResults = append(result.AssertionResults, judge.AssertionResult{ + Text: "named metric contract", Passed: false, Evidence: evidence, + }) + result.Summary.Total++ + result.Summary.Failed++ + if result.Summary.Total > 0 { + result.Summary.PassRate = float64(result.Summary.Passed) / float64(result.Summary.Total) + } +} diff --git a/internal/evaluator/metrics_integration_test.go b/internal/evaluator/metrics_integration_test.go index b46ea3db..0f123271 100644 --- a/internal/evaluator/metrics_integration_test.go +++ b/internal/evaluator/metrics_integration_test.go @@ -27,7 +27,7 @@ func metricsScript(t *testing.T, dir string, body string) string { name = "score.ps1" } path := filepath.Join(dir, name) - if err := os.WriteFile(path, []byte(body), 0o755); err != nil { + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { t.Fatalf("write script: %v", err) } return path @@ -45,11 +45,11 @@ exit 0 "Set-Content -LiteralPath $env:EVAL_METRICS_RESULT_PATH -Value $body\nWrite-Output scored\nexit 0\n" ) -// sleepPosix and sleepWindows outlive the judge timeout so a later cancellation +// longWaitPosix and longWaitWindows outlive the judge timeout so a later cancellation // or deadline lands while the member is still running. const ( - sleepPosix = "#!/bin/sh\nsleep 5\nexit 0\n" - sleepWindows = "Start-Sleep -Seconds 5\nexit 0\n" + longWaitPosix = "#!/bin/sh\nsleep 5\nexit 0\n" + longWaitWindows = "Start-Sleep -Seconds 5\nexit 0\n" ) func scoreRegistry() []config.MetricConfig { @@ -100,22 +100,47 @@ func TestExecuteCaseSingleScriptJudgePersistsMetrics(t *testing.T) { }) rt := newScriptRuntime(t) result := e.executeCase(context.Background(), &config.CaseConfig{ID: "m1", Input: config.Input{Prompt: "hello"}}, "with_skill", rt, nil) + assertSingleScriptMetricPersistence(t, result) +} +func assertSingleScriptMetricPersistence(t *testing.T, result EvalResult) { + t.Helper() if result.Status != judge.StatusPass { t.Fatalf("status = %s, want PASS (%v)", result.Status, result.Error) } - if result.Grading == nil || result.Grading.Metrics == nil { + env := singleScriptMetricEnvelope(t, result) + assertMetricEnvelopeIdentityAndCoverage(t, env, result.Metrics) + assertMetricValues(t, env.Metrics) +} + +func singleScriptMetricEnvelope(t *testing.T, result EvalResult) *judge.MetricsEnvelope { + t.Helper() + if result.Grading == nil { + t.Fatal("grading result is nil") + } + if result.Grading.Metrics == nil { t.Fatalf("expected a collected envelope, got %+v", result.Grading) } - env := result.Grading.Metrics + return result.Grading.Metrics +} + +func assertMetricEnvelopeIdentityAndCoverage(t *testing.T, env, caseEnv *judge.MetricsEnvelope) { + t.Helper() + if env.JudgeID != "scorer" || caseEnv == nil || caseEnv.JudgeID != "scorer" { + t.Fatalf("configured judge identity not retained: grading=%+v case=%+v", env, caseEnv) + } if env.Coverage.Planned != 2 || env.Coverage.Available != 2 { t.Fatalf("coverage = %+v", env.Coverage) } if env.Quality != judge.QualityNotConfigured { t.Fatalf("quality = %s, want NOT_CONFIGURED (valid scores, no threshold decision)", env.Quality) } +} + +func assertMetricValues(t *testing.T, metrics []judge.MetricResult) { + t.Helper() var zero, partial bool - for _, metric := range env.Metrics { + for _, metric := range metrics { if metric.Value == nil { t.Fatalf("slot %s has no value", metric.MetricID) } @@ -127,7 +152,137 @@ func TestExecuteCaseSingleScriptJudgePersistsMetrics(t *testing.T) { } } if !zero || !partial { - t.Fatalf("collected values wrong: %+v", env.Metrics) + t.Fatalf("collected values wrong: %+v", metrics) + } +} + +func TestExecuteCaseSingleScriptMetricContractControlsVerdict(t *testing.T) { + for _, tc := range []struct { + name string + payload string + registry []config.MetricConfig + want judge.Status + quality judge.Quality + complete judge.Completion + }{ + { + name: "threshold miss fails", + payload: `{"schema_version":1,"invocation_id":"__INVOCATION__","metrics":[{"metric_id":"correctness","availability":"AVAILABLE","value":0.4}]}`, + registry: []config.MetricConfig{{ID: "correctness", Required: true, MinScore: new(0.7)}}, + want: judge.StatusFail, quality: judge.QualityNotMet, complete: judge.CompletionFinished, + }, + { + name: "required missing is error", + payload: `{"schema_version":1,"invocation_id":"__INVOCATION__","metrics":[{"metric_id":"optional","availability":"AVAILABLE","value":0.8}]}`, + registry: []config.MetricConfig{{ID: "correctness", Required: true}, {ID: "optional"}}, + want: judge.StatusError, quality: judge.QualityNotConfigured, complete: judge.CompletionIncomplete, + }, + { + name: "malformed payload is error", + payload: `not-json`, + registry: []config.MetricConfig{{ID: "correctness", Required: true}}, + want: judge.StatusError, quality: judge.QualityNotAssessed, complete: judge.CompletionFailed, + }, + { + name: "optional-only empty output remains completed", + payload: `{"schema_version":1,"invocation_id":"__INVOCATION__","metrics":[]}`, + registry: []config.MetricConfig{{ID: "optional"}}, + want: judge.StatusPass, quality: judge.QualityNotAssessed, complete: judge.CompletionFinished, + }, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + script := metricsPayloadScript(t, dir, tc.payload) + limit := 30 + member := config.JudgeConfig{ID: "scorer", Type: "script", ScriptPath: script, TimeoutSeconds: &limit, Metrics: tc.registry} + e := newTestEvaluator(EvalOptions{ + Agent: &mockAgent{name: "test", output: "done"}, OutputDir: t.TempDir(), + EvalCfg: &config.EvalConfig{Environment: config.Environment{Type: "none"}, Judge: member}, + }) + result := e.executeCase(context.Background(), &config.CaseConfig{ID: "metric-policy", Input: config.Input{Prompt: "hello"}}, "with_skill", newScriptRuntime(t), nil) + if result.Status != tc.want { + t.Fatalf("status=%s error=%v, want %s", result.Status, result.Error, tc.want) + } + if result.Metrics == nil || result.Metrics.Quality != tc.quality { + t.Fatalf("metrics=%+v, want quality %s", result.Metrics, tc.quality) + } + if result.Execution == nil || result.Execution.Completion != tc.complete { + t.Fatalf("execution=%+v, want completion %s", result.Execution, tc.complete) + } + }) + } +} + +func metricsPayloadScript(t *testing.T, dir, payload string) string { + t.Helper() + if goruntime.GOOS == windowsGOOS { + body := "$body = '" + payload + "' -replace '__INVOCATION__', $env:EVAL_METRICS_INVOCATION_ID\nSet-Content -LiteralPath $env:EVAL_METRICS_RESULT_PATH -Value $body\nexit 0\n" + return metricsScript(t, dir, body) + } + body := "#!/bin/sh\ncat > \"$EVAL_METRICS_RESULT_PATH\" < 0 - cancel := func() {} - if memberTimeout { - memberCtx, cancel = context.WithTimeout(ctx, time.Duration(*member.TimeoutSeconds)*time.Second) - } - memberResult := &EvalResult{SessionResult: result.SessionResult, CaseID: result.CaseID, CaseName: result.CaseName} - graded := e.runJudgePhase(memberCtx, rt, caseCfg, configName, member, turnsTotal, runAgent, input, memberResult, outcome.Artifacts, true) - outcome.Status, outcome.Result = graded.Status, graded.Grading - outcome.Session, outcome.Skills = graded.JudgeSession, graded.JudgeSkills - if graded.Grading != nil { - outcome.Recovery = graded.Grading.Recovery - } - outcome.Error = judgeOutcomeError(graded) - if graded.Status == judge.StatusError && graded.Error != nil { - causes = append(causes, graded.Error) - } - stop := memberStop(ctx, memberCtx, memberTimeout) - // A member is accepted only when it reached its own final outcome: no stop - // at all, or a deadline that still left usable collected scores. A - // cancellation that arrived while the member was running never counts as - // acceptance, so its slots stay NOT_PRODUCED instead of looking like an - // omitted metric, and its verdict is not laundered into a completed run. - accepted := false - if graded.Grading != nil && graded.Status != judge.StatusError { - switch stop.Cause { - case judge.CauseNone: - accepted = true - case judge.CauseDeadlineExpiry: - accepted = envelopeHasScores(graded.Grading.Metrics) - } - } - outcome.Metrics = gradedMetrics(member, graded, accepted) - outcome.Execution = memberExecution(graded, stop, accepted) - if memberErr := recordMemberContextError(memberCtx, &outcome, accepted); memberErr != nil { - causes = append(causes, memberErr) - } - cancel() - outcome.DurationMs = time.Since(started).Milliseconds() + outcome, memberCauses := e.runJudgeMember(ctx, rt, caseCfg, configName, member, turnsTotal, runAgent, input, result) + causes = append(causes, memberCauses...) result.JudgeResults = append(result.JudgeResults, outcome) } result.Grading, result.Status = judge.DefaultOutcomeAggregator().Aggregate(result.JudgeResults, result.Turns, turnsTotal) @@ -100,14 +73,65 @@ func (e *defaultEvaluator) runMultipleJudges( return *result } -func recordMemberContextError(ctx context.Context, outcome *JudgeOutcome, accepted bool) error { +func (e *defaultEvaluator) runJudgeMember(ctx context.Context, rt runtime.Runtime, caseCfg *config.CaseConfig, configName string, member config.JudgeConfig, turnsTotal int, runAgent agent.Agent, input judge.Input, result *EvalResult) (JudgeOutcome, []error) { + started := time.Now() + outcome := JudgeOutcome{ID: member.ID, Type: member.Type, Artifacts: "judge/" + member.ID + "/run"} + e.prepareOutputDir(ctx, configName, caseCfg.ID, outcome.Artifacts) + memberCtx := ctx + memberDeadlineSource := "" + cancel := func() {} + if member.TimeoutSeconds != nil && *member.TimeoutSeconds > 0 { + memberCtx, cancel = context.WithTimeout(ctx, time.Duration(*member.TimeoutSeconds)*time.Second) + memberDeadlineSource = judgeTimeoutSource + } + defer cancel() + memberResult := &EvalResult{SessionResult: result.SessionResult, CaseID: result.CaseID, CaseName: result.CaseName} + graded := e.runJudgePhase(memberCtx, rt, caseCfg, configName, member, turnsTotal, runAgent, input, memberResult, outcome.Artifacts, true) + finalizeJudgeMetrics(memberCtx, &graded, member) + outcome.Status, outcome.Result = graded.Status, graded.Grading + outcome.Session, outcome.Skills = graded.JudgeSession, graded.JudgeSkills + if graded.Grading != nil { + outcome.Recovery = graded.Grading.Recovery + } + outcome.Error = judgeOutcomeError(graded) + causes := []error{} + if graded.Status == judge.StatusError && graded.Error != nil { + causes = append(causes, graded.Error) + } + stop := memberStop(ctx, memberCtx, memberDeadlineSource) + decision := memberDecisionFor(graded, stop) + outcome.Metrics = gradedMetrics(member, graded, decision) + outcome.Execution = memberExecution(graded, stop, decision, judge.SlotsFromConfig(member.Metrics)) + if memberErr := recordMemberContextError(memberCtx, &outcome, decision); memberErr != nil { + causes = append(causes, memberErr) + } + outcome.DurationMs = time.Since(started).Milliseconds() + return outcome, causes +} + +func memberDecisionFor(graded EvalResult, stop judge.Stop) MemberDecision { + if graded.Grading == nil || graded.Status == judge.StatusError { + return MemberNotAccepted + } + switch stop.Cause { + case judge.CauseNone: + return MemberAccepted + case judge.CauseDeadlineExpiry: + if envelopeHasScores(graded.Grading.Metrics) { + return MemberAccepted + } + } + return MemberNotAccepted +} + +func recordMemberContextError(ctx context.Context, outcome *JudgeOutcome, decision MemberDecision) error { err := ctx.Err() if err == nil { return nil } // An accepted outcome keeps its own verdict; the stop is still recorded so the // timeout or cancellation stays visible in the execution metadata. - if !accepted && outcome.Status != judge.StatusError { + if decision != MemberAccepted && outcome.Status != judge.StatusError { outcome.Status = judge.StatusError outcome.Error = err.Error() } @@ -117,15 +141,19 @@ func recordMemberContextError(ctx context.Context, outcome *JudgeOutcome, accept // memberStop resolves why a member stopped. A member-level deadline is only // attributed to judge.timeout_seconds when the parent context had not itself // expired, so a case deadline firing through the member is never mislabeled. -func memberStop(parentCtx, memberCtx context.Context, memberTimeout bool) judge.Stop { +func memberStop(parentCtx, memberCtx context.Context, memberDeadlineSource string) judge.Stop { if memberCtx.Err() == nil { return judge.Stop{Cause: judge.CauseNone} } - source := DeadlineSource(parentCtx) - if memberTimeout && source == "" && parentCtx.Err() == nil && errors.Is(memberCtx.Err(), context.DeadlineExceeded) { - source = "judge.timeout_seconds" + var source string + if errors.Is(memberCtx.Err(), context.DeadlineExceeded) { + if parentCtx.Err() != nil { + source = DeadlineSource(parentCtx) + } else { + source = memberDeadlineSource + } } - return judge.ClassifyStop(memberCtx.Err(), source, SignalObserved(parentCtx)) + return judge.ClassifyStop(memberCtx.Err(), judge.StopObservation{DeadlineSource: source, SignalObserved: SignalObserved(parentCtx)}) } // stopDetail renders a human-readable stop note. The machine-readable cause and @@ -164,13 +192,13 @@ func preventedMetrics(judgeID string, registry []config.MetricConfig) *judge.Met if len(slots) == 0 { return nil } - return judge.MaterializeSlots(judgeID, slots, nil, judge.ProblemNone, true) + return judge.MaterializeSlots(judgeID, slots, nil, judge.ProblemNone, judge.MetricPrevented) } // gradedMetrics builds the envelope from what a judge produced. A score that was // already collected survives a later stop; slots that were never reached stay // NOT_PRODUCED rather than becoming zero or looking omitted. -func gradedMetrics(member config.JudgeConfig, graded EvalResult, accepted bool) *judge.MetricsEnvelope { +func gradedMetrics(member config.JudgeConfig, graded EvalResult, decision MemberDecision) *judge.MetricsEnvelope { slots := judge.SlotsFromConfig(member.Metrics) if len(slots) == 0 { return nil @@ -181,31 +209,25 @@ func gradedMetrics(member config.JudgeConfig, graded EvalResult, accepted bool) // Anything the execution layer stopped before a final outcome leaves its slots // NOT_PRODUCED: the metric was never reached, which is not the same as a judge // omitting or declining a metric in a completed result. - prevented := !accepted - return judge.MaterializeSlots(member.ID, slots, nil, judge.ProblemNone, prevented) + production := judge.MetricPrevented + if decision == MemberAccepted { + production = judge.MetricProduced + } + return judge.MaterializeSlots(member.ID, slots, nil, judge.ProblemNone, production) } // memberExecution describes one member's execution facts. Accepted results report // their own completion; a stopped member without a final outcome is INTERRUPTED // whether or not it had started, with the stage recording which of the two it was. -func memberExecution(graded EvalResult, stop judge.Stop, accepted bool) *judge.ExecutionMetadata { - if accepted { - completion := judge.CompletionFinished - if graded.Grading.Metrics != nil { - completion = completionForContract(graded.Grading.Metrics) - } - meta := &judge.ExecutionMetadata{ - Stage: judge.StageFinished, - Completion: completion, - Cause: judge.CauseNone, - } - if stop.Cause != judge.CauseNone { - meta.Cause, meta.DeadlineSource, meta.CancelSource, meta.Detail = stop.Cause, stop.DeadlineSource, stop.CancelSource, stop.Detail - if stop.Cause == judge.CauseDeadlineExpiry { - meta.Timeout = &judge.TimeoutMetadata{Occurred: true, Source: stop.DeadlineSource, Recovered: graded.Grading.Recovery != nil} - } - } - return meta +func memberExecution(graded EvalResult, stop judge.Stop, decision MemberDecision, planned []judge.MetricSlot) *judge.ExecutionMetadata { + if decision != MemberAccepted && graded.Execution != nil && graded.Execution.Stage == judge.StageFinished { + // A completed scoring pass can still violate the metric contract. Preserve + // its FAILED/INCOMPLETE completion rather than rewriting it as an execution + // interruption merely because the compatibility verdict is ERROR. + return graded.Execution + } + if decision == MemberAccepted { + return acceptedMemberExecution(graded, stop, planned) } stage := judge.StageStarted if graded.Grading == nil && graded.Status == judge.StatusSkip { @@ -224,24 +246,48 @@ func memberExecution(graded EvalResult, stop judge.Stop, accepted bool) *judge.E return execution } +func acceptedMemberExecution(graded EvalResult, stop judge.Stop, planned []judge.MetricSlot) *judge.ExecutionMetadata { + completion := judge.CompletionFinished + env := graded.Metrics + if graded.Grading != nil && graded.Grading.Metrics != nil { + env = graded.Grading.Metrics + } + if env != nil { + completion = completionForContract(env, planned) + } + meta := &judge.ExecutionMetadata{Stage: judge.StageFinished, Completion: completion, Cause: judge.CauseNone} + if stop.Cause == judge.CauseNone { + return meta + } + meta.Cause, meta.DeadlineSource, meta.CancelSource, meta.Detail = stop.Cause, stop.DeadlineSource, stop.CancelSource, stop.Detail + if stop.Cause == judge.CauseDeadlineExpiry { + meta.Timeout = &judge.TimeoutMetadata{Occurred: true, Source: stop.DeadlineSource, Recovered: graded.Grading.Recovery != nil} + } + return meta +} + // completionForContract maps the materialized slots onto the completion states: // an errored slot failed the contract, an unfulfilled required slot or an // unevaluable threshold leaves the outcome incomplete. -func completionForContract(env *judge.MetricsEnvelope) judge.Completion { - requiredMissing := false +func completionForContract(env *judge.MetricsEnvelope, planned []judge.MetricSlot) judge.Completion { + if env == nil { + return judge.CompletionFinished + } + byID := make(map[string]judge.MetricResult, len(env.Metrics)) for _, metric := range env.Metrics { + byID[metric.MetricID] = metric if metric.Availability == judge.Errored { return judge.CompletionFailed } } - for _, metric := range env.Metrics { - if metric.Availability != judge.Available { - requiredMissing = true + for _, slot := range planned { + metric, ok := byID[slot.ID] + if !ok || metric.Availability != judge.Available { + if slot.Required || slot.MinScore != nil { + return judge.CompletionIncomplete + } } } - if requiredMissing { - return judge.CompletionIncomplete - } return judge.CompletionFinished } @@ -286,10 +332,17 @@ func summarizeJudgeErrors(outcomes []JudgeOutcome) string { // reason strings are unchanged for compatibility; the machine-readable cause is // the new source of truth. func recordSkippedJudges(result *EvalResult, plan config.JudgePlan, reason string) { + cause := causeForReason(reason) if !plan.Multi { + if len(plan.Judges) > 0 { + member := plan.Judges[0] + result.Metrics = preventedMetrics(member.ID, member.Metrics) + if result.Metrics != nil { + result.Execution = interruptedExecution(judge.StageNotInvoked, judge.Stop{Cause: cause, Detail: reason}) + } + } return } - cause := causeForReason(reason) for _, member := range plan.Judges { result.JudgeResults = append(result.JudgeResults, JudgeOutcome{ ID: member.ID, Type: member.Type, Status: judge.StatusSkip, SkipReason: reason, @@ -297,6 +350,7 @@ func recordSkippedJudges(result *EvalResult, plan config.JudgePlan, reason strin Metrics: preventedMetrics(member.ID, member.Metrics), }) } + result.GroupCompletion, result.GroupCause = foldGroupState(result.JudgeResults) } func causeForReason(reason string) judge.Cause { diff --git a/internal/judge/README.md b/internal/judge/README.md index 71bb5b55..30a664cc 100644 --- a/internal/judge/README.md +++ b/internal/judge/README.md @@ -317,6 +317,11 @@ Rules that the implementation enforces: give `MET`/`NOT_MET`; thresholds with any thresholded value missing give `UNKNOWN`; valid scores with no thresholds give `NOT_CONFIGURED`; no valid scores give `NOT_ASSESSED`. +- For a judge that opts into a metric registry, a threshold miss changes its + verdict to `FAIL`; an errored slot or an unavailable required/thresholded slot + changes it to `ERROR`. Optional unavailable slots without thresholds do not + change the verdict and do not make completion `INCOMPLETE`. With no registry, + legacy verdict behavior is unchanged. - A timeout does not invalidate a trustworthy score. An accepted score stays `AVAILABLE`, and the timeout, its source and any recovery stay recorded. A score is never fabricated as zero, and an accepted result is never rewritten @@ -329,11 +334,11 @@ Rules that the implementation enforces: invalidates its planned slot once. The coverage denominator is the registry count and never moves. - Execution metadata and planned slots live on `Outcome` and the case result, not - only inside `judge.Result`, so a gate short-circuit or a cancellation with no - judge result still reports them. -- Compatibility is untouched: `PASS/FAIL/SKIP/ERROR`, the #309 assertion and member - denominators with zero-credit skipped members, the existing script-timeout verdict - mapping, and the JSONL event schema all behave as before. + only inside `judge.Result`, so single-judge early returns, gate short-circuits, + and cancellations with no judge result still report them. +- Legacy verdicts, the #309 assertion and member denominators with zero-credit + skipped members, the existing script-timeout mapping, and the JSONL event schema + remain unchanged when metrics are not configured. The metric registry is opt-in. ### Script judge structured metrics result (slice 1 input channel) @@ -377,11 +382,12 @@ Rejection semantics: - A nonzero exit never collects scores even if a valid file exists; the file stays an on-disk diagnostic. -Persistence: the envelope rides on `judge.Result.Metrics` and on each grouped -`judge.Outcome` (`metrics` plus `execution`), so it lands in `report.json` -(single judge) and in `evaluation.json` (`judge_results[]`, plus -`aggregation.completion_state` / `aggregation.termination_cause`). `grading.json` -keeps the Anthropic shape untouched. +Persistence: the envelope rides on `judge.Result.Metrics`, the single case result, +and each grouped `judge.Outcome` (`metrics` plus `execution`). It lands in +`report.json` and in `evaluation.json`; single-judge evaluation records use the +top-level `metrics` and `execution` fields, while grouped records use +`judge_results[]` plus `aggregation.completion_state` / `termination_cause`. +`grading.json` keeps the Anthropic shape untouched. Not implemented in this slice: diff --git a/internal/judge/factory.go b/internal/judge/factory.go index 2aef0ea2..f894598e 100644 --- a/internal/judge/factory.go +++ b/internal/judge/factory.go @@ -35,6 +35,7 @@ func NewJudge(cfg config.JudgeConfig, ag agent.Agent, rt runtime.Runtime) (Judge TimeoutSeconds: derefInt(cfg.TimeoutSeconds), Runtime: rt, MetricSlots: SlotsFromConfig(cfg.Metrics), + MetricJudgeID: cfg.ID, }, nil case "agent_judge": diff --git a/internal/judge/judge.go b/internal/judge/judge.go index c4f80854..ade52f70 100644 --- a/internal/judge/judge.go +++ b/internal/judge/judge.go @@ -153,6 +153,10 @@ type Result struct { // judges without a configured metrics registry leave this nil, and no legacy // field changes meaning. Metrics *MetricsEnvelope `json:"metrics,omitempty"` + + // Execution is evaluator-facing metadata copied to the case record rather + // than nested in the legacy grading payload. + Execution *ExecutionMetadata `json:"-"` } // RecoveryMetadata describes a recovered judge result without exposing the diff --git a/internal/judge/metrics.go b/internal/judge/metrics.go index 270fe951..646a523c 100644 --- a/internal/judge/metrics.go +++ b/internal/judge/metrics.go @@ -44,14 +44,22 @@ const ( type Cause string const ( - CauseNone Cause = "none" - CauseCancellation Cause = "cancellation" + // CauseNone means no termination cause was recorded. + CauseNone Cause = "none" + // CauseCancellation means the evaluation context was cancelled. + CauseCancellation Cause = "cancellation" + // CauseDeadlineExpiry means an evaluation deadline expired. CauseDeadlineExpiry Cause = "deadline_expiry" - CauseExpectGate Cause = "expect_gate" - CauseProcessExit Cause = "process_exit" + // CauseExpectGate means an expect pre-check prevented judge invocation. + CauseExpectGate Cause = "expect_gate" + // CauseProcessExit means a script exited unsuccessfully. + CauseProcessExit Cause = "process_exit" + // CauseExecutionError means evaluation failed during execution. CauseExecutionError Cause = "execution_error" - CauseContractError Cause = "contract_error" - CauseJudgeDeclined Cause = "judge_declined" + // CauseContractError means an input or output violated the evaluation contract. + CauseContractError Cause = "contract_error" + // CauseJudgeDeclined means a judge completed but declined a required metric. + CauseJudgeDeclined Cause = "judge_declined" ) // Stage is the program-recorded execution stage. INTERRUPTED members that @@ -60,19 +68,27 @@ const ( type Stage string const ( + // StageNotInvoked means the judge was never called. StageNotInvoked Stage = "NOT_INVOKED" - StageStarted Stage = "STARTED" - StageFinished Stage = "COMPLETED" + // StageStarted means the judge was called but did not finish normally. + StageStarted Stage = "STARTED" + // StageFinished means the judge returned a final result. + StageFinished Stage = "COMPLETED" ) // Availability describes one planned metric slot. type Availability string const ( - Available Availability = "AVAILABLE" - Missing Availability = "MISSING" - Errored Availability = "ERROR" - Declined Availability = "SKIP" + // Available means a valid score was produced. + Available Availability = "AVAILABLE" + // Missing means the judge completed without producing this slot. + Missing Availability = "MISSING" + // Errored means this slot or its payload was invalid. + Errored Availability = "ERROR" + // Declined means the judge explicitly skipped this slot. + Declined Availability = "SKIP" + // NotProduced means evaluation stopped before this slot was produced. NotProduced Availability = "NOT_PRODUCED" ) @@ -80,11 +96,16 @@ const ( type Quality string const ( - QualityMet Quality = "MET" - QualityNotMet Quality = "NOT_MET" - QualityUnknown Quality = "UNKNOWN" + // QualityMet means every configured threshold was met. + QualityMet Quality = "MET" + // QualityNotMet means at least one configured threshold was missed. + QualityNotMet Quality = "NOT_MET" + // QualityUnknown means a configured threshold could not be evaluated. + QualityUnknown Quality = "UNKNOWN" + // QualityNotConfigured means scores exist but no thresholds were configured. QualityNotConfigured Quality = "NOT_CONFIGURED" - QualityNotAssessed Quality = "NOT_ASSESSED" + // QualityNotAssessed means no usable scores exist. + QualityNotAssessed Quality = "NOT_ASSESSED" ) // Only two cancel sources are actually detectable today: a process interrupt observed by @@ -108,24 +129,30 @@ type Stop struct { Detail string } +// StopObservation contains provenance observed at the stop boundary. +type StopObservation struct { + DeadlineSource string + SignalObserved bool +} + // ClassifyStop turns an observed context error into a cause plus its source. // deadlineSource names the configuration knob when a deadline was configured and // empty means the deadline's origin is unknown; signalObserved is true only when // the process actually saw an interrupt, because a bare context.Canceled does not // establish user-initiated cancellation. -func ClassifyStop(ctxErr error, deadlineSource string, signalObserved bool) Stop { +func ClassifyStop(ctxErr error, observation StopObservation) Stop { switch { case ctxErr == nil: return Stop{Cause: CauseNone} case errors.Is(ctxErr, context.DeadlineExceeded): - source := deadlineSource + source := observation.DeadlineSource if source == "" { source = CancelUnspecified } return Stop{Cause: CauseDeadlineExpiry, DeadlineSource: source, Detail: ctxErr.Error()} case errors.Is(ctxErr, context.Canceled): source := CancelUnspecified - if signalObserved { + if observation.SignalObserved { source = CancelOSSignal } return Stop{Cause: CauseCancellation, CancelSource: source, Detail: ctxErr.Error()} @@ -299,19 +326,38 @@ type EmittedMetric struct { type PayloadProblem string const ( - ProblemNone PayloadProblem = "" - ProblemUnparseable PayloadProblem = "unparseable_payload" - ProblemTopLevelShape PayloadProblem = "top_level_shape" + // ProblemNone means the payload passed whole-envelope validation. + ProblemNone PayloadProblem = "" + // ProblemUnparseable means the JSON payload could not be decoded. + ProblemUnparseable PayloadProblem = "unparseable_payload" + // ProblemTopLevelShape means the payload root had the wrong JSON type. + ProblemTopLevelShape PayloadProblem = "top_level_shape" + // ProblemDuplicateJSONKey means a JSON object repeats a key. ProblemDuplicateJSONKey PayloadProblem = "duplicate_json_key" - ProblemUnknownTopField PayloadProblem = "unknown_top_level_field" - ProblemSchemaVersion PayloadProblem = "schema_version" - ProblemUnboundResult PayloadProblem = "unbound_or_stale_result" - ProblemSizeLimit PayloadProblem = "size_limit" + // ProblemUnknownTopField means the envelope contains an undeclared field. + ProblemUnknownTopField PayloadProblem = "unknown_top_level_field" + // ProblemSchemaVersion means the payload schema version is unsupported. + ProblemSchemaVersion PayloadProblem = "schema_version" + // ProblemUnboundResult means the result is not bound to this invocation. + ProblemUnboundResult PayloadProblem = "unbound_or_stale_result" + // ProblemSizeLimit means the payload exceeds the collection size limit. + ProblemSizeLimit PayloadProblem = "size_limit" // ProblemChannelUnsupported is not a payload defect: it says the collection channel // itself was unavailable for this run, so no file could ever have been produced. ProblemChannelUnsupported PayloadProblem = "metrics_channel_unsupported" ) +// MetricProduction records whether a judge reached a final outcome before its +// metric slots were materialized. +type MetricProduction uint8 + +const ( + // MetricProduced means evaluation reached a final outcome. + MetricProduced MetricProduction = iota + // MetricPrevented means a gate, stop, or execution failure prevented output. + MetricPrevented +) + // MaterializeSlots builds exactly one result per planned slot. Unknown ids are // output diagnostics and never new slots; a duplicated id invalidates its planned // slot once; the coverage denominator always stays the planned count. @@ -320,7 +366,7 @@ const ( // deadline, execution failure), so slots that were never reached become // NOT_PRODUCED instead of being reported as producer errors. A score that was // already obtained is never downgraded because evaluation stopped afterwards. -func MaterializeSlots(judgeID string, planned []MetricSlot, emitted []EmittedMetric, problem PayloadProblem, prevented bool) *MetricsEnvelope { +func MaterializeSlots(judgeID string, planned []MetricSlot, emitted []EmittedMetric, problem PayloadProblem, production MetricProduction) *MetricsEnvelope { known := make(map[string]bool, len(planned)) for _, s := range planned { known[s.ID] = true @@ -342,12 +388,12 @@ func MaterializeSlots(judgeID string, planned []MetricSlot, emitted []EmittedMet case problem != ProblemNone: out = append(out, MetricResult{MetricID: s.ID, Availability: Errored, Reason: string(problem)}) case len(entries) == 0: - out = append(out, emptySlot(s, prevented)) + out = append(out, emptySlot(s, production)) case len(entries) > 1: diagnostics = append(diagnostics, "duplicate_metric_id: "+s.ID) out = append(out, MetricResult{MetricID: s.ID, Availability: Errored, Reason: "duplicate_metric_id"}) default: - out = append(out, resolveSlot(s, entries[0], prevented)) + out = append(out, resolveSlot(s, entries[0], production)) } } sort.Strings(diagnostics) @@ -360,8 +406,8 @@ func MaterializeSlots(judgeID string, planned []MetricSlot, emitted []EmittedMet return env } -func emptySlot(s MetricSlot, prevented bool) MetricResult { - if prevented { +func emptySlot(s MetricSlot, production MetricProduction) MetricResult { + if production == MetricPrevented { return MetricResult{MetricID: s.ID, Availability: NotProduced, Reason: "evaluation_prevented"} } if s.Required { @@ -370,13 +416,13 @@ func emptySlot(s MetricSlot, prevented bool) MetricResult { return MetricResult{MetricID: s.ID, Availability: Missing, Reason: "not_emitted"} } -func resolveSlot(s MetricSlot, e EmittedMetric, prevented bool) MetricResult { +func resolveSlot(s MetricSlot, e EmittedMetric, production MetricProduction) MetricResult { res := MetricResult{MetricID: s.ID, Evidence: e.Evidence, Reason: e.Reason} switch strings.ToUpper(strings.TrimSpace(e.Availability)) { case "SKIP", "MISSING": // A decline or omission is only meaningful from an evaluation that // reached the slot; a stopped evaluation leaves NOT_PRODUCED instead. - if prevented { + if production == MetricPrevented { res.Availability, res.Reason = NotProduced, "evaluation_prevented" return res } diff --git a/internal/judge/metrics_payload.go b/internal/judge/metrics_payload.go index e623987e..46d56746 100644 --- a/internal/judge/metrics_payload.go +++ b/internal/judge/metrics_payload.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io" + "strings" ) // Structured named-metrics result payload for the script judge (#317, slice 1). @@ -49,7 +50,7 @@ func ParseMetricsPayload(raw []byte, invocationID string) ([]EmittedMetric, Payl if errors.As(err, &typeErr) { return nil, ProblemTopLevelShape } - if bytes.Contains([]byte(err.Error()), []byte("unknown field")) { + if strings.Contains(err.Error(), "unknown field") { return nil, ProblemUnknownTopField } return nil, ProblemUnparseable diff --git a/internal/judge/metrics_test.go b/internal/judge/metrics_test.go index 0a56e8cf..72bf67f0 100644 --- a/internal/judge/metrics_test.go +++ b/internal/judge/metrics_test.go @@ -11,8 +11,6 @@ import ( "github.com/alibaba/skill-up/internal/config" ) -func f(v float64) *float64 { return &v } - func twoSlots() []MetricSlot { return []MetricSlot{{ID: "correctness", Required: true}, {ID: "completeness"}} } @@ -61,50 +59,50 @@ func TestDeriveQuality(t *testing.T) { }{ { "thresholds configured and all satisfied", - []MetricSlot{{ID: "a", MinScore: f(0.7)}, {ID: "b", MinScore: f(0.5)}}, - []MetricResult{{MetricID: "a", Availability: Available, Value: f(0.7)}, {MetricID: "b", Availability: Available, Value: f(0.9)}}, + []MetricSlot{{ID: "a", MinScore: new(0.7)}, {ID: "b", MinScore: new(0.5)}}, + []MetricResult{{MetricID: "a", Availability: Available, Value: new(0.7)}, {MetricID: "b", Availability: Available, Value: new(0.9)}}, QualityMet, }, { "threshold boundary is inclusive", - []MetricSlot{{ID: "a", MinScore: f(0.75)}}, - []MetricResult{{MetricID: "a", Availability: Available, Value: f(0.75)}}, + []MetricSlot{{ID: "a", MinScore: new(0.75)}}, + []MetricResult{{MetricID: "a", Availability: Available, Value: new(0.75)}}, QualityMet, }, { "thresholds configured and one below", - []MetricSlot{{ID: "a", MinScore: f(0.7)}, {ID: "b", MinScore: f(0.7)}}, - []MetricResult{{MetricID: "a", Availability: Available, Value: f(0.9)}, {MetricID: "b", Availability: Available, Value: f(0.2)}}, + []MetricSlot{{ID: "a", MinScore: new(0.7)}, {ID: "b", MinScore: new(0.7)}}, + []MetricResult{{MetricID: "a", Availability: Available, Value: new(0.9)}, {MetricID: "b", Availability: Available, Value: new(0.2)}}, QualityNotMet, }, { "thresholded value unavailable yields UNKNOWN", - []MetricSlot{{ID: "a", MinScore: f(0.7)}, {ID: "b", MinScore: f(0.7)}}, - []MetricResult{{MetricID: "a", Availability: Available, Value: f(0.9)}, {MetricID: "b", Availability: Declined, Reason: "declined_by_judge"}}, + []MetricSlot{{ID: "a", MinScore: new(0.7)}, {ID: "b", MinScore: new(0.7)}}, + []MetricResult{{MetricID: "a", Availability: Available, Value: new(0.9)}, {MetricID: "b", Availability: Declined, Reason: "declined_by_judge"}}, QualityUnknown, }, { "cancellation with thresholds configured yields UNKNOWN", - []MetricSlot{{ID: "a", MinScore: f(0.7)}}, + []MetricSlot{{ID: "a", MinScore: new(0.7)}}, []MetricResult{{MetricID: "a", Availability: NotProduced, Reason: "evaluation_prevented"}}, QualityUnknown, }, { "gate short-circuit with thresholds configured yields UNKNOWN", - []MetricSlot{{ID: "a", MinScore: f(0.7)}, {ID: "b", MinScore: f(0.7)}}, + []MetricSlot{{ID: "a", MinScore: new(0.7)}, {ID: "b", MinScore: new(0.7)}}, []MetricResult{{MetricID: "a", Availability: NotProduced}, {MetricID: "b", Availability: NotProduced}}, QualityUnknown, }, { "valid scores and no thresholds is NOT_CONFIGURED", twoSlots(), - []MetricResult{{MetricID: "correctness", Availability: Available, Value: f(0.8)}, {MetricID: "completeness", Availability: Available, Value: f(0.6)}}, + []MetricResult{{MetricID: "correctness", Availability: Available, Value: new(0.8)}, {MetricID: "completeness", Availability: Available, Value: new(0.6)}}, QualityNotConfigured, }, { "an explicit zero score is a valid score", twoSlots(), - []MetricResult{{MetricID: "correctness", Availability: Available, Value: f(0)}}, + []MetricResult{{MetricID: "correctness", Availability: Available, Value: new(0.0)}}, QualityNotConfigured, }, { @@ -142,14 +140,14 @@ func TestValidScore(t *testing.T) { want bool }{ {"nil", nil, false}, - {"zero is valid", f(0), true}, - {"one is valid", f(1), true}, - {"negative", f(-0.01), false}, - {"above one", f(1.4), false}, - {"NaN is never clamped", f(math.NaN()), false}, - {"positive infinity", f(math.Inf(1)), false}, - {"negative infinity", f(math.Inf(-1)), false}, - {"mid range", f(0.75), true}, + {"zero is valid", new(0.0), true}, + {"one is valid", new(1.0), true}, + {"negative", new(-0.01), false}, + {"above one", new(1.4), false}, + {"NaN is never clamped", new(math.NaN()), false}, + {"positive infinity", new(math.Inf(1)), false}, + {"negative infinity", new(math.Inf(-1)), false}, + {"mid range", new(0.75), true}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -160,80 +158,86 @@ func TestValidScore(t *testing.T) { } } -func TestMaterializeSlotsAvailability(t *testing.T) { - cases := []struct { - name string - emitted []EmittedMetric - prevented bool - want map[string]Availability - wantDiags []string - wantPlanned int - }{ +type materializeCase struct { + name string + emitted []EmittedMetric + prevented MetricProduction + want map[string]Availability + wantDiags []string + wantPlanned int +} + +func materializeCases() []materializeCase { + return []materializeCase{ { name: "valid two-metric output", - emitted: []EmittedMetric{{MetricID: "correctness", Availability: "AVAILABLE", Value: f(0.8)}, {MetricID: "completeness", Availability: "AVAILABLE", Value: f(0.5)}}, + emitted: []EmittedMetric{{MetricID: "correctness", Availability: "AVAILABLE", Value: new(0.8)}, {MetricID: "completeness", Availability: "AVAILABLE", Value: new(0.5)}}, want: map[string]Availability{"correctness": Available, "completeness": Available}, wantPlanned: 2, }, { name: "optional omission is MISSING and required sibling survives", - emitted: []EmittedMetric{{MetricID: "correctness", Availability: "AVAILABLE", Value: f(0.9)}}, + emitted: []EmittedMetric{{MetricID: "correctness", Availability: "AVAILABLE", Value: new(0.9)}}, want: map[string]Availability{"correctness": Available, "completeness": Missing}, wantPlanned: 2, }, { name: "unknown id is a diagnostic and never a new slot", - emitted: []EmittedMetric{{MetricID: "correctness", Availability: "AVAILABLE", Value: f(0.9)}, {MetricID: "invented", Availability: "AVAILABLE", Value: f(0.1)}}, + emitted: []EmittedMetric{{MetricID: "correctness", Availability: "AVAILABLE", Value: new(0.9)}, {MetricID: "invented", Availability: "AVAILABLE", Value: new(0.1)}}, want: map[string]Availability{"correctness": Available, "completeness": Missing}, wantDiags: []string{"unknown_metric_id: invented"}, wantPlanned: 2, }, { name: "duplicate id invalidates that planned slot once and keeps no other slot", - emitted: []EmittedMetric{{MetricID: "correctness", Availability: "AVAILABLE", Value: f(0.9)}, {MetricID: "correctness", Availability: "AVAILABLE", Value: f(0.2)}, {MetricID: "completeness", Availability: "AVAILABLE", Value: f(0.4)}}, + emitted: []EmittedMetric{{MetricID: "correctness", Availability: "AVAILABLE", Value: new(0.9)}, {MetricID: "correctness", Availability: "AVAILABLE", Value: new(0.2)}, {MetricID: "completeness", Availability: "AVAILABLE", Value: new(0.4)}}, want: map[string]Availability{"correctness": Errored, "completeness": Available}, wantDiags: []string{"duplicate_metric_id: correctness"}, wantPlanned: 2, }, { name: "out of range value poisons only its own slot", - emitted: []EmittedMetric{{MetricID: "correctness", Availability: "AVAILABLE", Value: f(1.4)}, {MetricID: "completeness", Availability: "AVAILABLE", Value: f(0.4)}}, + emitted: []EmittedMetric{{MetricID: "correctness", Availability: "AVAILABLE", Value: new(1.4)}, {MetricID: "completeness", Availability: "AVAILABLE", Value: new(0.4)}}, want: map[string]Availability{"correctness": Errored, "completeness": Available}, wantPlanned: 2, }, { name: "non-finite value poisons only its own slot", - emitted: []EmittedMetric{{MetricID: "correctness", Availability: "AVAILABLE", Value: f(math.NaN())}, {MetricID: "completeness", Availability: "AVAILABLE", Value: f(0.4)}}, + emitted: []EmittedMetric{{MetricID: "correctness", Availability: "AVAILABLE", Value: new(math.NaN())}, {MetricID: "completeness", Availability: "AVAILABLE", Value: new(0.4)}}, want: map[string]Availability{"correctness": Errored, "completeness": Available}, wantPlanned: 2, }, { name: "judge-declared SKIP keeps its own reason and is never zero", - emitted: []EmittedMetric{{MetricID: "correctness", Availability: "AVAILABLE", Value: f(0.8)}, {MetricID: "completeness", Availability: "SKIP", Reason: "insufficient evidence"}}, + emitted: []EmittedMetric{{MetricID: "correctness", Availability: "AVAILABLE", Value: new(0.8)}, {MetricID: "completeness", Availability: "SKIP", Reason: "insufficient evidence"}}, want: map[string]Availability{"correctness": Available, "completeness": Declined}, wantPlanned: 2, }, { name: "cancelled before or during invocation produces NOT_PRODUCED, never SKIP", - prevented: true, + prevented: MetricPrevented, want: map[string]Availability{"correctness": NotProduced, "completeness": NotProduced}, wantPlanned: 2, }, { name: "a score obtained before the stop is not downgraded to NOT_PRODUCED", - prevented: true, - emitted: []EmittedMetric{{MetricID: "correctness", Availability: "AVAILABLE", Value: f(0.7)}}, + prevented: MetricPrevented, + emitted: []EmittedMetric{{MetricID: "correctness", Availability: "AVAILABLE", Value: new(0.7)}}, want: map[string]Availability{"correctness": Available, "completeness": NotProduced}, wantPlanned: 2, }, { name: "a SKIP or MISSING claim from a stopped evaluation is NOT_PRODUCED, not a decline", - prevented: true, + prevented: MetricPrevented, emitted: []EmittedMetric{{MetricID: "correctness", Availability: "SKIP", Reason: "ran out of time"}}, want: map[string]Availability{"correctness": NotProduced, "completeness": NotProduced}, wantPlanned: 2, }, } +} + +func TestMaterializeSlotsAvailability(t *testing.T) { + cases := materializeCases() for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { env := MaterializeSlots("j1", twoSlots(), tc.emitted, ProblemNone, tc.prevented) @@ -263,17 +267,19 @@ func TestMaterializeSlotsAvailability(t *testing.T) { } func TestMaterializeSlotsPayloadProblems(t *testing.T) { - problems := []PayloadProblem{ProblemUnparseable, ProblemTopLevelShape, ProblemDuplicateJSONKey, - ProblemUnknownTopField, ProblemSchemaVersion, ProblemUnboundResult, ProblemSizeLimit} + problems := []PayloadProblem{ + ProblemUnparseable, ProblemTopLevelShape, ProblemDuplicateJSONKey, + ProblemUnknownTopField, ProblemSchemaVersion, ProblemUnboundResult, ProblemSizeLimit, + } for _, p := range problems { t.Run(string(p), func(t *testing.T) { // A well-formed pair of scores must still be discarded, because the // envelope as a whole cannot be trusted. emitted := []EmittedMetric{ - {MetricID: "correctness", Availability: "AVAILABLE", Value: f(0.8)}, - {MetricID: "completeness", Availability: "AVAILABLE", Value: f(0.6)}, + {MetricID: "correctness", Availability: "AVAILABLE", Value: new(0.8)}, + {MetricID: "completeness", Availability: "AVAILABLE", Value: new(0.6)}, } - env := MaterializeSlots("j1", twoSlots(), emitted, p, false) + env := MaterializeSlots("j1", twoSlots(), emitted, p, MetricProduced) for _, m := range env.Metrics { if m.Availability != Errored || m.Reason != string(p) || m.Value != nil { t.Fatalf("slot %s = %+v, want ERROR/%s with no value", m.MetricID, m, p) @@ -292,11 +298,11 @@ func TestMaterializeSlotsPayloadProblems(t *testing.T) { func TestSlotLevelDefectKeepsSiblingsUsable(t *testing.T) { planned := []MetricSlot{{ID: "correctness", Required: true}, {ID: "completeness"}, {ID: "instruction_following"}} emitted := []EmittedMetric{ - {MetricID: "correctness", Availability: "AVAILABLE", Value: f(0.9)}, - {MetricID: "completeness", Availability: "AVAILABLE", Value: f(2.5)}, // invalid - {MetricID: "instruction_following", Availability: "AVAILABLE", Value: f(0.5)}, + {MetricID: "correctness", Availability: "AVAILABLE", Value: new(0.9)}, + {MetricID: "completeness", Availability: "AVAILABLE", Value: new(2.5)}, // invalid + {MetricID: "instruction_following", Availability: "AVAILABLE", Value: new(0.5)}, } - env := MaterializeSlots("j1", planned, emitted, ProblemNone, false) + env := MaterializeSlots("j1", planned, emitted, ProblemNone, MetricProduced) if env.Metrics[0].Availability != Available || env.Metrics[2].Availability != Available { t.Fatalf("valid siblings must stay AVAILABLE: %+v", env.Metrics) } @@ -317,9 +323,9 @@ func TestSlotLevelDefectKeepsSiblingsUsable(t *testing.T) { func TestEvidenceRequirementIsPerSlot(t *testing.T) { planned := []MetricSlot{{ID: "correctness", EvidenceRequired: true}, {ID: "completeness"}} env := MaterializeSlots("j1", planned, []EmittedMetric{ - {MetricID: "correctness", Availability: "AVAILABLE", Value: f(0.8)}, - {MetricID: "completeness", Availability: "AVAILABLE", Value: f(0.8)}, - }, ProblemNone, false) + {MetricID: "correctness", Availability: "AVAILABLE", Value: new(0.8)}, + {MetricID: "completeness", Availability: "AVAILABLE", Value: new(0.8)}, + }, ProblemNone, MetricProduced) if env.Metrics[0].Availability != Errored || env.Metrics[0].Reason != "missing_evidence" { t.Fatalf("evidence-less slot = %+v", env.Metrics[0]) } @@ -329,7 +335,7 @@ func TestEvidenceRequirementIsPerSlot(t *testing.T) { } func TestCoverageWithoutPlannedSlotsStaysNull(t *testing.T) { - env := MaterializeSlots("j1", nil, nil, ProblemNone, false) + env := MaterializeSlots("j1", nil, nil, ProblemNone, MetricProduced) if env.Coverage.Planned != 0 || env.Coverage.Ratio != nil { t.Fatalf("coverage = %+v, want planned 0 and null ratio", env.Coverage) } @@ -341,7 +347,7 @@ func TestCoverageWithoutPlannedSlotsStaysNull(t *testing.T) { func TestSlotsFromConfigKeepsThePlannedDenominator(t *testing.T) { registry := []config.MetricConfig{ {ID: "correctness", Required: true, EvidenceRequired: true}, - {ID: "completeness", MinScore: f(0.75)}, + {ID: "completeness", MinScore: new(0.75)}, } slots := SlotsFromConfig(registry) if len(slots) != 2 { @@ -357,7 +363,7 @@ func TestSlotsFromConfigKeepsThePlannedDenominator(t *testing.T) { t.Fatal("a judge without a registry must keep no slots") } // The registry, not a returned payload, sets the denominator. - env := MaterializeSlots("j1", slots, []EmittedMetric{{MetricID: "correctness", Availability: "AVAILABLE", Value: f(0.9)}}, ProblemNone, false) + env := MaterializeSlots("j1", slots, []EmittedMetric{{MetricID: "correctness", Availability: "AVAILABLE", Value: new(0.9)}}, ProblemNone, MetricProduced) if env.Coverage.Planned != 2 { t.Fatalf("planned = %d, want the registry count of 2", env.Coverage.Planned) } @@ -410,7 +416,7 @@ func TestClassifyStop(t *testing.T) { } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - got := ClassifyStop(tc.err, tc.deadlineSource, tc.signal) + got := ClassifyStop(tc.err, StopObservation{DeadlineSource: tc.deadlineSource, SignalObserved: tc.signal}) if got != tc.want { t.Fatalf("ClassifyStop = %+v, want %+v", got, tc.want) } diff --git a/internal/judge/script.go b/internal/judge/script.go index 66110124..186eb214 100644 --- a/internal/judge/script.go +++ b/internal/judge/script.go @@ -43,6 +43,9 @@ type ScriptJudge struct { // MetricSlots is the planned named-metric registry for this judge. Empty means // the judge produces no metrics envelope at all. MetricSlots []MetricSlot + // MetricJudgeID is the stable configured judge identifier recorded in the + // metrics envelope. Empty preserves the standalone "script" identifier. + MetricJudgeID string // MetricsResultPath is the runner-assigned host path for this invocation's // structured result. Set only when the script runs against the local runtime. @@ -147,19 +150,7 @@ func (j *ScriptJudge) evaluateInRuntime(ctx context.Context, rt evalruntime.Runt debugInfo := strings.TrimSpace(result.Stderr) if err != nil { if ctx.Err() == context.DeadlineExceeded { - // A timeout does not invalidate a trustworthy result: if this - // invocation's own schema-valid file was written before the deadline, - // the scores are collected while the existing verdict mapping and the - // timeout record are preserved. - if j.TimeoutIsError { - timeoutErr := fmt.Errorf("script timed out after %s: %w", timeout, ctx.Err()) - res := NewErrorResult(timeoutErr, in.TurnsExecuted, in.TurnsTotal) - j.attachMetrics(res, in, true) - return res, timeoutErr - } - res := j.buildResult(false, "script timed out after "+timeout.String(), debugInfo, in) - j.attachMetrics(res, in, true) - return res, nil + return j.timeoutResult(ctx, timeout, debugInfo, in) } if result.ExitCode == 0 { return nil, fmt.Errorf("script execution failed: %w", err) @@ -180,7 +171,28 @@ func (j *ScriptJudge) evaluateInRuntime(ctx context.Context, rt evalruntime.Runt evidence = "script passed (exit code 0)" } res := j.buildResult(true, evidence, debugInfo, in) - j.attachMetrics(res, in, false) + j.attachMetrics(res, MetricProduced) + return res, nil +} + +func (j *ScriptJudge) timeoutResult(ctx context.Context, timeout time.Duration, debugInfo string, in Input) (*Result, error) { + source := "judge.timeout_seconds" + execution := &ExecutionMetadata{ + Stage: StageStarted, Completion: CompletionInterrupted, + Cause: CauseDeadlineExpiry, DeadlineSource: source, + Timeout: &TimeoutMetadata{Occurred: true, Source: source}, + } + // Keep a trustworthy result written before the deadline. + if j.TimeoutIsError { + timeoutErr := fmt.Errorf("script timed out after %s: %w", timeout, ctx.Err()) + res := NewErrorResult(timeoutErr, in.TurnsExecuted, in.TurnsTotal) + res.Execution = execution + j.attachMetrics(res, MetricPrevented) + return res, timeoutErr + } + res := j.buildResult(false, "script timed out after "+timeout.String(), debugInfo, in) + res.Execution = execution + j.attachMetrics(res, MetricPrevented) return res, nil } @@ -209,22 +221,26 @@ func (j *ScriptJudge) buildResult(passed bool, evidence, debugInfo string, in In // appeared while evaluation was interrupted yields NOT_PRODUCED, and one that // never appeared after a completed run yields a rejected payload rather than a // fabricated zero. -func (j *ScriptJudge) attachMetrics(res *Result, in Input, prevented bool) { +func (j *ScriptJudge) attachMetrics(res *Result, production MetricProduction) { if res == nil || len(j.MetricSlots) == 0 { return } + judgeID := j.MetricJudgeID + if judgeID == "" { + judgeID = "script" + } if j.MetricsResultPath == "" { // The channel is not available for this judge (a non-local runtime, or a // judge built without one). Report that honestly rather than implying a // stale file was found. - res.Metrics = MaterializeSlots("script", j.MetricSlots, nil, ProblemChannelUnsupported, prevented) + res.Metrics = MaterializeSlots(judgeID, j.MetricSlots, nil, ProblemChannelUnsupported, production) return } info, err := os.Lstat(j.MetricsResultPath) - if errors.Is(err, os.ErrNotExist) && prevented { + if errors.Is(err, os.ErrNotExist) && production == MetricPrevented { // Nothing was ever produced because evaluation was interrupted: the slots // are NOT_PRODUCED, not a rejected payload. - res.Metrics = MaterializeSlots("script", j.MetricSlots, nil, ProblemNone, true) + res.Metrics = MaterializeSlots(judgeID, j.MetricSlots, nil, ProblemNone, MetricPrevented) return } if err != nil { @@ -232,25 +248,25 @@ func (j *ScriptJudge) attachMetrics(res *Result, in Input, prevented bool) { if !os.IsNotExist(err) { problem = ProblemUnparseable } - res.Metrics = MaterializeSlots("script", j.MetricSlots, nil, problem, prevented) + res.Metrics = MaterializeSlots(judgeID, j.MetricSlots, nil, problem, production) return } if !info.Mode().IsRegular() { // A symlink, directory or device node is never a collected result. - res.Metrics = MaterializeSlots("script", j.MetricSlots, nil, ProblemUnboundResult, prevented) + res.Metrics = MaterializeSlots(judgeID, j.MetricSlots, nil, ProblemUnboundResult, production) return } if info.Size() > MaxMetricsPayloadBytes { // Bound the read before it happens: an oversized file must not be loaded // into memory just to be rejected afterwards. - res.Metrics = MaterializeSlots("script", j.MetricSlots, nil, ProblemSizeLimit, prevented) + res.Metrics = MaterializeSlots(judgeID, j.MetricSlots, nil, ProblemSizeLimit, production) return } raw, err := os.ReadFile(j.MetricsResultPath) if err != nil { - res.Metrics = MaterializeSlots("script", j.MetricSlots, nil, ProblemUnparseable, prevented) + res.Metrics = MaterializeSlots(judgeID, j.MetricSlots, nil, ProblemUnparseable, production) return } emitted, payloadProblem := ParseMetricsPayload(raw, j.MetricsInvocationID) - res.Metrics = MaterializeSlots("script", j.MetricSlots, emitted, payloadProblem, false) + res.Metrics = MaterializeSlots(judgeID, j.MetricSlots, emitted, payloadProblem, MetricProduced) } diff --git a/internal/judge/script_metrics_test.go b/internal/judge/script_metrics_test.go index 2b563c86..ca42a9f7 100644 --- a/internal/judge/script_metrics_test.go +++ b/internal/judge/script_metrics_test.go @@ -50,7 +50,7 @@ func (tc metricsScriptCase) body() string { func writeMetricsScript(t *testing.T, dir string, tc metricsScriptCase) string { t.Helper() path := filepath.Join(dir, tc.scriptName()) - if err := os.WriteFile(path, []byte(tc.body()), 0o755); err != nil { + if err := os.WriteFile(path, []byte(tc.body()), 0o600); err != nil { t.Fatalf("write script: %v", err) } return path @@ -145,9 +145,12 @@ func twoMetricRegistry() []config.MetricConfig { return []config.MetricConfig{{ID: "correctness", Required: true}, {ID: "completeness"}} } -func TestScriptJudgeCollectsNamedMetrics(t *testing.T) { - minScore := 0.5 - cases := []metricsScriptCase{ +func metricsScriptCases() []metricsScriptCase { + return append(metricsScriptBasicCases(), metricsScriptAdditionalCases()...) +} + +func metricsScriptBasicCases() []metricsScriptCase { + return []metricsScriptCase{ { name: "two metrics including an explicit zero", posix: posixWriteTwoMetrics, @@ -201,6 +204,12 @@ func TestScriptJudgeCollectsNamedMetrics(t *testing.T) { "correctness": "unbound_or_stale_result", "completeness": "unbound_or_stale_result", }, }, + } +} + +func metricsScriptAdditionalCases() []metricsScriptCase { + minScore := 0.5 + return []metricsScriptCase{ { // Contract §10: the runner collects the exact assigned file after a // successful exit and rejects a missing one, so a zero-exit script that @@ -263,57 +272,69 @@ func TestScriptJudgeCollectsNamedMetrics(t *testing.T) { expectPlanned: 2, }, } +} + +func TestScriptJudgeCollectsNamedMetrics(t *testing.T) { + cases := metricsScriptCases() for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - res := runMetricsScriptCase(t, tc) - if tc.expectNilEnvelope { - if res.Metrics != nil { - t.Fatalf("expected no envelope after a nonzero exit, got %+v", res.Metrics) - } - return - } - if res.Metrics == nil { - t.Fatal("expected a metrics envelope from the collected file") - } - env := res.Metrics - if env.Coverage.Planned != tc.expectPlanned { - t.Fatalf("planned = %d, want %d", env.Coverage.Planned, tc.expectPlanned) - } - if len(env.Metrics) != tc.expectPlanned { - t.Fatalf("materialized %d slots, want %d", len(env.Metrics), tc.expectPlanned) - } - if env.Quality != tc.expectQuality { - t.Fatalf("quality = %s, want %s", env.Quality, tc.expectQuality) - } - for _, metric := range env.Metrics { - want, ok := tc.expectAvailability[metric.MetricID] - if !ok { - t.Fatalf("unexpected slot %q", metric.MetricID) - } - if metric.Availability != want { - t.Fatalf("%s availability = %s, want %s (reason %q)", metric.MetricID, metric.Availability, want, metric.Reason) - } - if want != Available && metric.Value != nil { - t.Fatalf("%s carries value %v while unavailable: no fabricated zero allowed", metric.MetricID, *metric.Value) - } - if reason, ok := tc.expectReasons[metric.MetricID]; ok && !strings.Contains(metric.Reason, reason) { - t.Fatalf("%s reason = %q, want substring %q", metric.MetricID, metric.Reason, reason) - } - } - for id, wantValue := range tc.expectValues { - for _, metric := range env.Metrics { - if metric.MetricID == id { - if metric.Value == nil || *metric.Value != wantValue { - t.Fatalf("%s value = %v, want %v", id, metric.Value, wantValue) - } - } + t.Run(tc.name, func(t *testing.T) { checkMetricsScriptCase(t, tc) }) + } +} + +func checkMetricsScriptCase(t *testing.T, tc metricsScriptCase) { + t.Helper() + res := runMetricsScriptCase(t, tc) + if tc.expectNilEnvelope { + if res.Metrics != nil { + t.Fatalf("expected no envelope after a nonzero exit, got %+v", res.Metrics) + } + return + } + if res.Metrics == nil { + t.Fatal("expected a metrics envelope from the collected file") + } + env := res.Metrics + if env.Coverage.Planned != tc.expectPlanned { + t.Fatalf("planned = %d, want %d", env.Coverage.Planned, tc.expectPlanned) + } + if len(env.Metrics) != tc.expectPlanned { + t.Fatalf("materialized %d slots, want %d", len(env.Metrics), tc.expectPlanned) + } + if env.Quality != tc.expectQuality { + t.Fatalf("quality = %s, want %s", env.Quality, tc.expectQuality) + } + checkMetricValues(t, env, tc) + // The verdict mapping itself must not change because metrics exist. + if res.Status != StatusPass && !tc.nameIsFailing() { + t.Fatalf("status = %s, expected the legacy PASS for a zero-exit script", res.Status) + } +} + +func checkMetricValues(t *testing.T, env *MetricsEnvelope, tc metricsScriptCase) { + t.Helper() + for _, metric := range env.Metrics { + want, ok := tc.expectAvailability[metric.MetricID] + if !ok { + t.Fatalf("unexpected slot %q", metric.MetricID) + } + if metric.Availability != want { + t.Fatalf("%s availability = %s, want %s (reason %q)", metric.MetricID, metric.Availability, want, metric.Reason) + } + if want != Available && metric.Value != nil { + t.Fatalf("%s carries value %v while unavailable: no fabricated zero allowed", metric.MetricID, *metric.Value) + } + if reason, ok := tc.expectReasons[metric.MetricID]; ok && !strings.Contains(metric.Reason, reason) { + t.Fatalf("%s reason = %q, want substring %q", metric.MetricID, metric.Reason, reason) + } + } + for id, wantValue := range tc.expectValues { + for _, metric := range env.Metrics { + if metric.MetricID == id { + if metric.Value == nil || *metric.Value != wantValue { + t.Fatalf("%s value = %v, want %v", id, metric.Value, wantValue) } } - // The verdict mapping itself must not change because metrics exist. - if res.Status != StatusPass && !tc.nameIsFailing() { - t.Fatalf("status = %s, expected the legacy PASS for a zero-exit script", res.Status) - } - }) + } } } @@ -472,33 +493,36 @@ func TestScriptJudgeTimeoutWithoutCollectedFileLeavesSlotsNotProduced(t *testing } } -func TestScriptJudgeRejectsOversizedAndNonRegularResults(t *testing.T) { - t.Run("oversized payload is rejected before it is read", func(t *testing.T) { - dir := t.TempDir() - script := writeMetricsScript(t, dir, metricsScriptCase{ - posix: "#!/bin/sh\nexit 0\n", windows: "exit 0\n", - }) - resultPath := filepath.Join(dir, "metrics-result.json") - // A padding string plus valid JSON shape: over the collector limit. - padding := strings.Repeat("x", MaxMetricsPayloadBytes+16) - payload := `{"schema_version":1,"invocation_id":"inv-42","metrics":[{"metric_id":"correctness","availability":"AVAILABLE","value":0.5,"evidence":["` + padding + `"]}]}` - if err := os.WriteFile(resultPath, []byte(payload), 0o600); err != nil { - t.Fatal(err) - } - j := &ScriptJudge{ - ScriptPath: script, MetricSlots: SlotsFromConfig(twoMetricRegistry()), - MetricsResultPath: resultPath, MetricsInvocationID: "inv-42", - } - res, err := j.Evaluate(context.Background(), Input{WorkspacePath: dir}) - if err != nil { - t.Fatalf("Evaluate: %v", err) - } - for _, metric := range res.Metrics.Metrics { - if metric.Availability != Errored || metric.Reason != string(ProblemSizeLimit) { - t.Fatalf("slot = %+v, want ERROR/%s", metric, ProblemSizeLimit) - } - } +func testOversizedMetricsPayload(t *testing.T) { + t.Helper() + dir := t.TempDir() + script := writeMetricsScript(t, dir, metricsScriptCase{ + posix: "#!/bin/sh\nexit 0\n", windows: "exit 0\n", }) + resultPath := filepath.Join(dir, "metrics-result.json") + // A padding string plus valid JSON shape: over the collector limit. + padding := strings.Repeat("x", MaxMetricsPayloadBytes+16) + payload := `{"schema_version":1,"invocation_id":"inv-42","metrics":[{"metric_id":"correctness","availability":"AVAILABLE","value":0.5,"evidence":["` + padding + `"]}]}` + if err := os.WriteFile(resultPath, []byte(payload), 0o600); err != nil { + t.Fatal(err) + } + j := &ScriptJudge{ + ScriptPath: script, MetricSlots: SlotsFromConfig(twoMetricRegistry()), + MetricsResultPath: resultPath, MetricsInvocationID: "inv-42", + } + res, err := j.Evaluate(context.Background(), Input{WorkspacePath: dir}) + if err != nil { + t.Fatalf("Evaluate: %v", err) + } + for _, metric := range res.Metrics.Metrics { + if metric.Availability != Errored || metric.Reason != string(ProblemSizeLimit) { + t.Fatalf("slot = %+v, want ERROR/%s", metric, ProblemSizeLimit) + } + } +} + +func TestScriptJudgeRejectsOversizedAndNonRegularResults(t *testing.T) { + t.Run("oversized payload is rejected before it is read", testOversizedMetricsPayload) t.Run("a non-regular path is never collected", func(t *testing.T) { dir := t.TempDir() diff --git a/internal/judge/script_metrics_timeout_test.go b/internal/judge/script_metrics_timeout_test.go index ee196485..db006e92 100644 --- a/internal/judge/script_metrics_timeout_test.go +++ b/internal/judge/script_metrics_timeout_test.go @@ -41,11 +41,7 @@ func TestScriptJudgeRealSubprocessTimeoutCollectsResult(t *testing.T) { } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - scratch, err := os.MkdirTemp("", "skill-up-metrics-timeout-") - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = os.RemoveAll(scratch) }) + scratch := t.TempDir() script := writeMetricsScript(t, scratch, metricsScriptCase{ posix: realTimeoutPosix, windows: realTimeoutWindows, diff --git a/internal/report/grouped_evaluation.go b/internal/report/grouped_evaluation.go index 1bd5e533..9eb7d6c0 100644 --- a/internal/report/grouped_evaluation.go +++ b/internal/report/grouped_evaluation.go @@ -4,10 +4,12 @@ import "github.com/alibaba/skill-up/internal/judge" // GroupedEvaluation is the canonical machine-readable multi-judge result. type GroupedEvaluation struct { - Version int `json:"version"` - Gates *judge.ExpectResult `json:"gates,omitempty"` - JudgeResults []judge.Outcome `json:"judge_results"` - Aggregation GroupAggregation `json:"aggregation"` + Version int `json:"version"` + Gates *judge.ExpectResult `json:"gates,omitempty"` + JudgeResults []judge.Outcome `json:"judge_results"` + Aggregation GroupAggregation `json:"aggregation"` + Execution *judge.ExecutionMetadata `json:"execution,omitempty"` + Metrics *judge.MetricsEnvelope `json:"metrics,omitempty"` } // GroupAggregation records the case-level decision without flattening criteria. diff --git a/internal/report/metrics_persistence_test.go b/internal/report/metrics_persistence_test.go index c38d5258..36b3141a 100644 --- a/internal/report/metrics_persistence_test.go +++ b/internal/report/metrics_persistence_test.go @@ -24,7 +24,7 @@ func collectedEnvelope(t *testing.T, judgeID string) *judge.MetricsEnvelope { if problem != judge.ProblemNone { t.Fatalf("payload rejected: %s", problem) } - env := judge.MaterializeSlots(judgeID, slots, emitted, judge.ProblemNone, false) + env := judge.MaterializeSlots(judgeID, slots, emitted, judge.ProblemNone, judge.MetricProduced) if env.Coverage.Available != 2 { t.Fatalf("envelope = %+v", env) } @@ -48,7 +48,7 @@ func TestGroupedEvaluationPersistsMetricsAndExecution(t *testing.T) { }, { ID: "b", Type: "script", Status: judge.StatusSkip, SkipReason: "cancelled before invocation", - Metrics: judge.MaterializeSlots("b", plannedSlots(), nil, judge.ProblemNone, true), + Metrics: judge.MaterializeSlots("b", plannedSlots(), nil, judge.ProblemNone, judge.MetricPrevented), Execution: &judge.ExecutionMetadata{ Stage: judge.StageNotInvoked, Completion: judge.CompletionInterrupted, Cause: judge.CauseCancellation, CancelSource: judge.CancelOSSignal, @@ -74,6 +74,11 @@ func TestGroupedEvaluationPersistsMetricsAndExecution(t *testing.T) { if err := json.Unmarshal(raw, &back); err != nil { t.Fatalf("read back: %v", err) } + assertPersistedGroup(t, back) +} + +func assertPersistedGroup(t *testing.T, back GroupedEvaluation) { + t.Helper() if back.Aggregation.Completion != judge.CompletionInterrupted || back.Aggregation.Cause != judge.CauseCancellation { t.Fatalf("group state lost: %+v", back.Aggregation) } @@ -94,6 +99,15 @@ func TestGroupedEvaluationPersistsMetricsAndExecution(t *testing.T) { if zeroValue == nil || *zeroValue != 0 { t.Fatalf("explicit zero not persisted: %v", zeroValue) } + assertPersistedSecondMember(t, second) + // Legacy compatibility fields are untouched. + if first.Status != judge.StatusPass || second.Status != judge.StatusSkip || second.SkipReason == "" { + t.Fatalf("compat fields changed: %+v %+v", first, second) + } +} + +func assertPersistedSecondMember(t *testing.T, second judge.Outcome) { + t.Helper() if second.Execution == nil || second.Execution.Stage != judge.StageNotInvoked || second.Execution.Completion != judge.CompletionInterrupted || second.Execution.CancelSource != judge.CancelOSSignal { @@ -107,8 +121,4 @@ func TestGroupedEvaluationPersistsMetricsAndExecution(t *testing.T) { t.Fatalf("persisted slot = %+v", metric) } } - // Legacy compatibility fields are untouched. - if first.Status != judge.StatusPass || second.Status != judge.StatusSkip || second.SkipReason == "" { - t.Fatalf("compat fields changed: %+v %+v", first, second) - } } diff --git a/internal/report/reporter.go b/internal/report/reporter.go index f898df37..4d7914ab 100644 --- a/internal/report/reporter.go +++ b/internal/report/reporter.go @@ -133,28 +133,30 @@ func (in Input) PrimaryCaseResults() []CaseResult { // CaseResult represents the result of a single case execution. type CaseResult struct { - CaseID string `json:"case_id"` - Title string `json:"title"` - Status judge.Status `json:"status"` - DurationMs int64 `json:"duration_ms"` - Turns int `json:"turns"` - InputTokens int `json:"input_tokens"` - OutputTokens int `json:"output_tokens"` - JudgeDurationMs int64 `json:"judge_duration_ms,omitempty"` - JudgeInputTokens int `json:"judge_input_tokens,omitempty"` - JudgeOutputTokens int `json:"judge_output_tokens,omitempty"` - JudgeRecovery *judge.RecoveryMetadata `json:"judge_recovery,omitempty"` - Error string `json:"error,omitempty"` - Grading *judge.Result `json:"grading"` - Gates *judge.ExpectResult `json:"gates,omitempty"` - JudgeResults []judge.Outcome `json:"judge_results,omitempty"` - JudgeSkills []judge.SkillInfo `json:"judge_skills,omitempty"` - Configuration string `json:"configuration,omitempty"` // "with_skill" or "without_skill" - Prompt string `json:"prompt,omitempty"` // input prompt sent to the agent - Response string `json:"response,omitempty"` // agent final message - ObservedModel string `json:"observed_model,omitempty"` // model explicitly reported by the agent - ObservedVersion string `json:"observed_version,omitempty"` // CLI version reported by static runtime inspection - TurnResults []CaseTurnResult `json:"turn_results,omitempty"` // per-turn outcomes; nil for single-turn + CaseID string `json:"case_id"` + Title string `json:"title"` + Status judge.Status `json:"status"` + DurationMs int64 `json:"duration_ms"` + Turns int `json:"turns"` + InputTokens int `json:"input_tokens"` + OutputTokens int `json:"output_tokens"` + JudgeDurationMs int64 `json:"judge_duration_ms,omitempty"` + JudgeInputTokens int `json:"judge_input_tokens,omitempty"` + JudgeOutputTokens int `json:"judge_output_tokens,omitempty"` + JudgeRecovery *judge.RecoveryMetadata `json:"judge_recovery,omitempty"` + Error string `json:"error,omitempty"` + Grading *judge.Result `json:"grading"` + Gates *judge.ExpectResult `json:"gates,omitempty"` + JudgeResults []judge.Outcome `json:"judge_results,omitempty"` + Execution *judge.ExecutionMetadata `json:"execution,omitempty"` + Metrics *judge.MetricsEnvelope `json:"metrics,omitempty"` + JudgeSkills []judge.SkillInfo `json:"judge_skills,omitempty"` + Configuration string `json:"configuration,omitempty"` // "with_skill" or "without_skill" + Prompt string `json:"prompt,omitempty"` // input prompt sent to the agent + Response string `json:"response,omitempty"` // agent final message + ObservedModel string `json:"observed_model,omitempty"` // model explicitly reported by the agent + ObservedVersion string `json:"observed_version,omitempty"` // CLI version reported by static runtime inspection + TurnResults []CaseTurnResult `json:"turn_results,omitempty"` // per-turn outcomes; nil for single-turn } // CaseTurnResult holds the outcome of a single turn for reporting purposes. diff --git a/internal/runner/metrics_single_persistence_test.go b/internal/runner/metrics_single_persistence_test.go new file mode 100644 index 00000000..56f60541 --- /dev/null +++ b/internal/runner/metrics_single_persistence_test.go @@ -0,0 +1,50 @@ +package runner + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/alibaba/skill-up/internal/evaluator" + "github.com/alibaba/skill-up/internal/judge" + "github.com/alibaba/skill-up/internal/report" +) + +func TestWriteCaseArtifactsPersistsSingleJudgeMetricsAndExecution(t *testing.T) { + ws, err := report.NewIterationWorkspace(t.TempDir(), "skill", 1) + if err != nil { + t.Fatal(err) + } + if err := ws.EnsureDirs([]string{"single"}); err != nil { + t.Fatal(err) + } + score := 0.0 + result := &evaluator.EvalResult{ + CaseID: "single", CaseName: "single judge metrics", Status: judge.StatusPass, + Metrics: judge.MaterializeSlots("configured-script-id", []judge.MetricSlot{{ID: "correctness", Required: true}}, []judge.EmittedMetric{{ + MetricID: "correctness", Availability: "AVAILABLE", Value: &score, + }}, judge.ProblemNone, judge.MetricProduced), + Execution: &judge.ExecutionMetadata{Stage: judge.StageFinished, Completion: judge.CompletionFinished, Cause: judge.CauseNone}, + } + if err := (&Runner{}).writeCaseArtifacts(ws, result, "with_skill"); err != nil { + t.Fatal(err) + } + raw, err := os.ReadFile(filepath.Join(ws.ConfigDir("single", "with_skill"), "evaluation.json")) + if err != nil { + t.Fatal(err) + } + var saved report.GroupedEvaluation + if err := json.Unmarshal(raw, &saved); err != nil { + t.Fatal(err) + } + if len(saved.JudgeResults) != 0 || saved.Execution == nil || saved.Execution.Completion != judge.CompletionFinished { + t.Fatalf("single execution metadata lost: %+v", saved) + } + if saved.Metrics == nil || saved.Metrics.JudgeID != "configured-script-id" || saved.Metrics.Coverage.Available != 1 { + t.Fatalf("single metrics lost: %+v", saved.Metrics) + } + if saved.Metrics.Metrics[0].Value == nil || *saved.Metrics.Metrics[0].Value != 0 { + t.Fatalf("explicit zero lost: %+v", saved.Metrics.Metrics[0]) + } +} diff --git a/internal/runner/runner.go b/internal/runner/runner.go index c5e85189..06d0db63 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -587,7 +587,7 @@ func (r *Runner) writeCaseArtifacts(ws *report.IterationWorkspace, res *evaluato return err } } - if len(res.JudgeResults) > 0 { + if len(res.JudgeResults) > 0 || res.Execution != nil || res.Metrics != nil { strategy := res.AggregationStrategy if strategy == "" { strategy = judge.DefaultOutcomeAggregator().Strategy() @@ -596,6 +596,8 @@ func (r *Runner) writeCaseArtifacts(ws *report.IterationWorkspace, res *evaluato Version: 1, Gates: res.ExpectResult, JudgeResults: res.JudgeResults, + Execution: res.Execution, + Metrics: res.Metrics, Aggregation: report.GroupAggregation{ Strategy: strategy, Status: res.Status, @@ -793,6 +795,8 @@ func evalResultToCaseResult(res *evaluator.EvalResult) report.CaseResult { Grading: res.Grading, Gates: res.ExpectResult, JudgeResults: res.JudgeResults, + Execution: res.Execution, + Metrics: res.Metrics, JudgeSkills: res.JudgeSkills, Configuration: res.Configuration, Prompt: res.Prompt, From 5209b99890d7f4e964b7df415478522e4c81bdc1 Mon Sep 17 00:00:00 2001 From: Max_Guobaixin Date: Fri, 30 Oct 2026 03:50:41 +0800 Subject: [PATCH 07/12] fix(judge): keep an expired deadline visible on interrupted members interruptedExecution recorded termination_cause and deadline_source but never the timeout metadata, so a grouped member stopped by its own judge deadline reported the cause without the structured record that README requires. Build it once in interruptedExecution and drop the duplicate in deriveSingleExecution. --- internal/evaluator/metrics_finalize.go | 6 +----- internal/evaluator/multi_judge.go | 6 +++++- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/evaluator/metrics_finalize.go b/internal/evaluator/metrics_finalize.go index ce69341c..d2a42da0 100644 --- a/internal/evaluator/metrics_finalize.go +++ b/internal/evaluator/metrics_finalize.go @@ -103,11 +103,7 @@ func deriveSingleExecution(ctx context.Context, result *EvalResult, env *judge.M switch { case ctx.Err() != nil: stop := judge.ClassifyStop(ctx.Err(), judge.StopObservation{DeadlineSource: DeadlineSource(ctx), SignalObserved: SignalObserved(ctx)}) - execution := interruptedExecution(stage, stop) - if stop.Cause == judge.CauseDeadlineExpiry { - execution.Timeout = &judge.TimeoutMetadata{Occurred: true, Source: stop.DeadlineSource} - } - return execution + return interruptedExecution(stage, stop) case result.Grading != nil && result.Grading.Execution != nil: return result.Grading.Execution case contractError != "": diff --git a/internal/evaluator/multi_judge.go b/internal/evaluator/multi_judge.go index f895e405..ebb7a602 100644 --- a/internal/evaluator/multi_judge.go +++ b/internal/evaluator/multi_judge.go @@ -174,7 +174,7 @@ func stopDetail(stop judge.Stop) string { } func interruptedExecution(stage judge.Stage, stop judge.Stop) *judge.ExecutionMetadata { - return &judge.ExecutionMetadata{ + meta := &judge.ExecutionMetadata{ Stage: stage, Completion: judge.CompletionInterrupted, Cause: stop.Cause, @@ -182,6 +182,10 @@ func interruptedExecution(stage judge.Stage, stop judge.Stop) *judge.ExecutionMe CancelSource: stop.CancelSource, Detail: stop.Detail, } + if stop.Cause == judge.CauseDeadlineExpiry { + meta.Timeout = &judge.TimeoutMetadata{Occurred: true, Source: stop.DeadlineSource} + } + return meta } // preventedMetrics materializes every planned slot as NOT_PRODUCED for a member From 1143eb813896c2bde2722bac4c356616cf25682c Mon Sep 17 00:00:00 2001 From: Max_Guobaixin Date: Fri, 30 Oct 2026 03:50:53 +0800 Subject: [PATCH 08/12] test(judge): pin metric contract outcomes on the real evaluator path Add a grouped judge-timeout case with a live but unexpired case deadline, the shape the review asked about, and assert the stop names judge.timeout_seconds. Assert the coverage buckets behind each contract verdict, the configured judge id on grouped members, and the folded group state on every skip short circuit. Move the two tests that kill a running child onto a scratch workspace, since Windows harness cleanup of a held working directory fails after all assertions pass. --- .../evaluator/metrics_integration_test.go | 105 ++++++++++++++++-- internal/evaluator/metrics_wiring_test.go | 3 + internal/evaluator/multi_judge_test.go | 20 +++- 3 files changed, 114 insertions(+), 14 deletions(-) diff --git a/internal/evaluator/metrics_integration_test.go b/internal/evaluator/metrics_integration_test.go index 0f123271..d76f79ae 100644 --- a/internal/evaluator/metrics_integration_test.go +++ b/internal/evaluator/metrics_integration_test.go @@ -52,6 +52,15 @@ const ( longWaitWindows = "Start-Sleep -Seconds 5\nexit 0\n" ) +// scoreThenWaitBody emits valid scores and then outlives the judge deadline, so +// a collected result must survive the stop. +func scoreThenWaitBody() string { + if goruntime.GOOS == windowsGOOS { + return strings.Replace(powershellScore, "Write-Output scored\nexit 0", "Write-Output scored\nStart-Sleep -Seconds 6\nexit 0", 1) + } + return strings.Replace(posixScore, "echo \"scored\"\nexit 0", "echo \"scored\"\nsleep 6\nexit 0", 1) +} + func scoreRegistry() []config.MetricConfig { return []config.MetricConfig{{ID: "correctness", Required: true}, {ID: "completeness"}} } @@ -84,6 +93,27 @@ func newScriptRuntime(t *testing.T) runtime.Runtime { return rt } +// metricsScratchDir allocates a workspace outside the directory tree the test +// harness removes itself. On Windows the runtime kills only the direct child, so +// the timed-out script's own wait keeps this directory as its working directory +// until it exits long after the test finished; POSIX process-group kill removes +// it right away, where cleanup succeeds. Leaving it is a harness cost, not a +// verdict: harness cleanup of that path would otherwise fail the test after +// every assertion passed. +func metricsScratchDir(t *testing.T) string { + t.Helper() + dir, err := os.MkdirTemp("", "skill-up-metrics-scratch-*") //nolint:usetesting // t.TempDir() cannot own a workspace a killed child still holds + if err != nil { + t.Fatalf("scratch dir: %v", err) + } + t.Cleanup(func() { + if err := os.RemoveAll(dir); err != nil { + t.Logf("scratch workspace %s still held by the timed-out script, left for the OS temp cleaner: %v", dir, err) + } + }) + return dir +} + func TestExecuteCaseSingleScriptJudgePersistsMetrics(t *testing.T) { dir := t.TempDir() script := metricsScript(t, dir, pick(posixScore, powershellScore)) @@ -164,30 +194,35 @@ func TestExecuteCaseSingleScriptMetricContractControlsVerdict(t *testing.T) { want judge.Status quality judge.Quality complete judge.Completion + coverage judge.Coverage }{ { name: "threshold miss fails", payload: `{"schema_version":1,"invocation_id":"__INVOCATION__","metrics":[{"metric_id":"correctness","availability":"AVAILABLE","value":0.4}]}`, registry: []config.MetricConfig{{ID: "correctness", Required: true, MinScore: new(0.7)}}, want: judge.StatusFail, quality: judge.QualityNotMet, complete: judge.CompletionFinished, + coverage: judge.Coverage{Planned: 1, Available: 1}, }, { name: "required missing is error", payload: `{"schema_version":1,"invocation_id":"__INVOCATION__","metrics":[{"metric_id":"optional","availability":"AVAILABLE","value":0.8}]}`, registry: []config.MetricConfig{{ID: "correctness", Required: true}, {ID: "optional"}}, want: judge.StatusError, quality: judge.QualityNotConfigured, complete: judge.CompletionIncomplete, + coverage: judge.Coverage{Planned: 2, Available: 1, Missing: 1}, }, { name: "malformed payload is error", payload: `not-json`, registry: []config.MetricConfig{{ID: "correctness", Required: true}}, want: judge.StatusError, quality: judge.QualityNotAssessed, complete: judge.CompletionFailed, + coverage: judge.Coverage{Planned: 1, Errored: 1}, }, { name: "optional-only empty output remains completed", payload: `{"schema_version":1,"invocation_id":"__INVOCATION__","metrics":[]}`, registry: []config.MetricConfig{{ID: "optional"}}, want: judge.StatusPass, quality: judge.QualityNotAssessed, complete: judge.CompletionFinished, + coverage: judge.Coverage{Planned: 1, Missing: 1}, }, } { t.Run(tc.name, func(t *testing.T) { @@ -206,6 +241,11 @@ func TestExecuteCaseSingleScriptMetricContractControlsVerdict(t *testing.T) { if result.Metrics == nil || result.Metrics.Quality != tc.quality { t.Fatalf("metrics=%+v, want quality %s", result.Metrics, tc.quality) } + if got := result.Metrics.Coverage; got.Planned != tc.coverage.Planned || got.Available != tc.coverage.Available || + got.Missing != tc.coverage.Missing || got.Errored != tc.coverage.Errored || + got.Skipped != tc.coverage.Skipped || got.NotProduced != tc.coverage.NotProduced { + t.Fatalf("coverage = %+v, want %+v", got, tc.coverage) + } if result.Execution == nil || result.Execution.Completion != tc.complete { t.Fatalf("execution=%+v, want completion %s", result.Execution, tc.complete) } @@ -263,20 +303,14 @@ func TestExecuteCaseExpectGateFoldsGroupState(t *testing.T) { func TestExecuteCaseSingleScriptTimeoutRecordsSourceAndKeepsScores(t *testing.T) { dir := t.TempDir() - body := pick(posixScore, powershellScore) - if goruntime.GOOS == windowsGOOS { - body = strings.Replace(body, "Write-Output scored\nexit 0", "Write-Output scored\nStart-Sleep -Seconds 5\nexit 0", 1) - } else { - body = strings.Replace(body, "echo \"scored\"\nexit 0", "echo \"scored\"\nsleep 5\nexit 0", 1) - } - script := metricsScript(t, dir, body) + script := metricsScript(t, dir, scoreThenWaitBody()) one := 1 member := config.JudgeConfig{ID: "scorer", Type: "script", ScriptPath: script, TimeoutSeconds: &one, Metrics: scoreRegistry()} e := newTestEvaluator(EvalOptions{ Agent: &mockAgent{name: "test", output: "done"}, OutputDir: t.TempDir(), EvalCfg: &config.EvalConfig{Environment: config.Environment{Type: "none"}, Judge: member}, }) - result := e.executeCase(context.Background(), &config.CaseConfig{ID: "single-timeout", Input: config.Input{Prompt: "hello"}}, "with_skill", newScriptRuntime(t), nil) + result := e.executeCase(context.Background(), &config.CaseConfig{ID: "single-timeout", Input: config.Input{Prompt: "hello"}}, "with_skill", newScratchRuntime(t, metricsScratchDir(t)), nil) if result.Status != judge.StatusFail || result.Metrics == nil || result.Metrics.Coverage.Available != 2 { t.Fatalf("timeout verdict/scores status=%s metrics=%+v", result.Status, result.Metrics) } @@ -286,6 +320,58 @@ func TestExecuteCaseSingleScriptTimeoutRecordsSourceAndKeepsScores(t *testing.T) } } +// TestExecuteCaseGroupedJudgeTimeoutUnderLiveCaseDeadline drives the reviewer's +// shape for real: the case knob is armed at 300s and never fires while the judge +// knob is armed at 2s and does. The member runs as a child process, emits its +// scores, and is then killed by its own deadline, so the stop must name the knob +// that expired rather than the live case deadline above it. +func TestExecuteCaseGroupedJudgeTimeoutUnderLiveCaseDeadline(t *testing.T) { + scratch := metricsScratchDir(t) + slowDir := filepath.Join(scratch, "slow") + if err := os.MkdirAll(slowDir, 0o755); err != nil { + t.Fatal(err) + } + slow := metricsScript(t, slowDir, scoreThenWaitBody()) + two := 2 + members := []config.JudgeConfig{ + {ID: "scorer", Type: "script", ScriptPath: slow, TimeoutSeconds: &two, Metrics: scoreRegistry()}, + } + e := newTestEvaluator(EvalOptions{ + Agent: &mockAgent{name: "test", output: "done"}, OutputDir: t.TempDir(), + EvalCfg: &config.EvalConfig{Environment: config.Environment{Type: "none"}, Judges: &members}, + }) + result := e.executeCase(context.Background(), &config.CaseConfig{ + ID: "live-case-deadline", Input: config.Input{Prompt: "hello"}, + Constraints: config.Constraints{TimeoutSeconds: 300}, + }, "with_skill", newScratchRuntime(t, scratch), nil) + + if len(result.JudgeResults) != 1 { + t.Fatalf("outcomes = %d", len(result.JudgeResults)) + } + member := result.JudgeResults[0] + if member.Execution == nil || member.Execution.Cause != judge.CauseDeadlineExpiry || + member.Execution.DeadlineSource != judgeTimeoutSource { + t.Fatalf("grouped timeout provenance = %+v", member.Execution) + } + if member.Execution.Timeout == nil || !member.Execution.Timeout.Occurred || + member.Execution.Timeout.Source != judgeTimeoutSource { + t.Fatalf("timeout metadata = %+v", member.Execution.Timeout) + } + // Upstream (before #317) arms grouped script judges with TimeoutIsError=true + // and the single-judge path with false, so the same judge deadline is an ERROR + // verdict when grouped and FAIL when alone. What #317 must keep is that the + // expired knob stays attributable and the scores already collected survive. + if member.Status != judge.StatusError { + t.Fatalf("grouped member verdict = %s, want ERROR under the pre-existing strict script timeout", member.Status) + } + if member.Metrics == nil || member.Metrics.Coverage.Available != 2 { + t.Fatalf("collected scores lost on stop: metrics=%+v", member.Metrics) + } + if result.GroupCompletion != judge.CompletionInterrupted || result.GroupCause != judge.CauseDeadlineExpiry { + t.Fatalf("group = %s/%s, want INTERRUPTED/deadline_expiry", result.GroupCompletion, result.GroupCause) + } +} + func TestExecuteCaseMultiJudgeDerivesGroupStateFromMembers(t *testing.T) { dir := t.TempDir() script := metricsScript(t, dir, pick(posixScore, powershellScore)) @@ -309,6 +395,9 @@ func TestExecuteCaseMultiJudgeDerivesGroupStateFromMembers(t *testing.T) { if outcome.Metrics == nil || outcome.Metrics.Coverage.Available != 2 { t.Fatalf("member %s envelope = %+v", outcome.ID, outcome.Metrics) } + if outcome.Metrics.JudgeID != outcome.ID { + t.Fatalf("member %s persisted producer identity %q, want the configured id", outcome.ID, outcome.Metrics.JudgeID) + } if outcome.Execution == nil || outcome.Execution.Completion != judge.CompletionFinished || outcome.Execution.Stage != judge.StageFinished { t.Fatalf("member %s execution = %+v", outcome.ID, outcome.Execution) diff --git a/internal/evaluator/metrics_wiring_test.go b/internal/evaluator/metrics_wiring_test.go index 352b5021..e36cd608 100644 --- a/internal/evaluator/metrics_wiring_test.go +++ b/internal/evaluator/metrics_wiring_test.go @@ -292,6 +292,9 @@ func TestSkippedJudgesRetainSlotsPerCause(t *testing.T) { t.Fatalf("metrics = %+v, want one NOT_PRODUCED slot", outcome.Metrics) } } + if result.GroupCompletion != judge.CompletionInterrupted || result.GroupCause != tc.want { + t.Fatalf("group state on the %s short circuit = %s/%s, want INTERRUPTED/%s", tc.reason, result.GroupCompletion, result.GroupCause, tc.want) + } }) } } diff --git a/internal/evaluator/multi_judge_test.go b/internal/evaluator/multi_judge_test.go index 2fd719e6..49be0386 100644 --- a/internal/evaluator/multi_judge_test.go +++ b/internal/evaluator/multi_judge_test.go @@ -369,17 +369,25 @@ func TestMultiJudgeMultiTurnEarlyExitPreservesSkippedOutcomes(t *testing.T) { if ag.turnCall != 1 || ag.runCall.Load() != 0 { t.Fatalf("unexpected agent or judge calls: turns=%d runs=%d", ag.turnCall, ag.runCall.Load()) } - assertSkippedJudgeOutcomes(t, result.JudgeResults, members, tc.wantReason) + assertSkippedJudgeOutcomes(t, result, members, tc.wantReason) }) } } -func assertSkippedJudgeOutcomes(t *testing.T, outcomes []JudgeOutcome, members []config.JudgeConfig, reason string) { +func assertSkippedJudgeOutcomes(t *testing.T, result EvalResult, members []config.JudgeConfig, reason string) { t.Helper() - for i, outcome := range outcomes { + cause := causeForReason(reason) + for i, outcome := range result.JudgeResults { if outcome.ID != members[i].ID || outcome.Type != members[i].Type || outcome.Status != judge.StatusSkip || outcome.SkipReason != reason || outcome.Result != nil { t.Fatalf("unexpected outcome: %+v", outcome) } + if outcome.Execution == nil || outcome.Execution.Completion != judge.CompletionInterrupted || + outcome.Execution.Stage != judge.StageNotInvoked || outcome.Execution.Cause != cause { + t.Fatalf("skipped outcome %s execution = %+v, want interrupted/not-invoked/%s", outcome.ID, outcome.Execution, cause) + } + } + if result.GroupCompletion != judge.CompletionInterrupted || result.GroupCause != cause { + t.Fatalf("group state on the %s short circuit = %s/%s, want INTERRUPTED/%s", reason, result.GroupCompletion, result.GroupCause, cause) } } @@ -403,7 +411,7 @@ func TestMultiJudgeSingleTurnExecutionErrorPreservesSkippedOutcomes(t *testing.T if result.Status != judge.StatusError || result.AggregationStrategy != "all_required" || len(result.JudgeResults) != len(members) || ag.runCall.Load() != 1 { t.Fatalf("unexpected result: %+v", result) } - assertSkippedJudgeOutcomes(t, result.JudgeResults, members, "agent_execution_failed") + assertSkippedJudgeOutcomes(t, result, members, "agent_execution_failed") }) } } @@ -452,7 +460,7 @@ func TestMultiJudgeUserSimulatorGroupedOutcomes(t *testing.T) { func assertSimulatedJudgeStatuses(t *testing.T, result EvalResult, members []config.JudgeConfig) { t.Helper() if result.Status == judge.StatusError { - assertSkippedJudgeOutcomes(t, result.JudgeResults, members, "agent_execution_failed") + assertSkippedJudgeOutcomes(t, result, members, "agent_execution_failed") return } for _, outcome := range result.JudgeResults { @@ -478,5 +486,5 @@ func TestMultiJudgeUnsupportedSessionResumerPreservesSkippedOutcomes(t *testing. if result.Status != judge.StatusError || len(result.JudgeResults) != len(members) || ag.runCall.Load() != 0 { t.Fatalf("unexpected result: %+v", result) } - assertSkippedJudgeOutcomes(t, result.JudgeResults, members, "session_resumption_unsupported") + assertSkippedJudgeOutcomes(t, result, members, "session_resumption_unsupported") } From 32b98f284b134c507119952f829693f37a180bd2 Mon Sep 17 00:00:00 2001 From: Max_Guobaixin Date: Fri, 30 Oct 2026 03:50:53 +0800 Subject: [PATCH 09/12] test(runner): persist single-judge metrics through evaluator and runner Run a real script judge through NewEvaluator/EvaluateAll, write the case artifacts, then read evaluation.json back from disk and assert the configured judge id, FINISHED execution and the explicit zero score. Split the case into helpers so the complexity gate accepts it, and provide the PowerShell fixture so the path is verified on Windows instead of skipped. --- .../runner/metrics_single_persistence_test.go | 115 ++++++++++++++++++ 1 file changed, 115 insertions(+) diff --git a/internal/runner/metrics_single_persistence_test.go b/internal/runner/metrics_single_persistence_test.go index 56f60541..7027c0ef 100644 --- a/internal/runner/metrics_single_persistence_test.go +++ b/internal/runner/metrics_single_persistence_test.go @@ -1,16 +1,131 @@ package runner import ( + "context" "encoding/json" "os" "path/filepath" + "runtime" + "strings" "testing" + "github.com/alibaba/skill-up/internal/agent" + "github.com/alibaba/skill-up/internal/config" "github.com/alibaba/skill-up/internal/evaluator" "github.com/alibaba/skill-up/internal/judge" "github.com/alibaba/skill-up/internal/report" + evalruntime "github.com/alibaba/skill-up/internal/runtime" + "github.com/alibaba/skill-up/pkg/transcript" ) +// metricsPersistenceAgent is the smallest agent.Agent that lets a case reach the +// judge phase, so the script judge really runs and its envelope is persisted. +type metricsPersistenceAgent struct{} + +func (metricsPersistenceAgent) Name() string { return "metrics-test" } +func (metricsPersistenceAgent) Check(context.Context, agent.Runtime) error { return nil } +func (metricsPersistenceAgent) CheckCredentials(context.Context) error { return nil } +func (metricsPersistenceAgent) Install(context.Context, agent.Runtime) error { return nil } + +func (metricsPersistenceAgent) InstallMCP(context.Context, agent.Runtime, evalruntime.MCPConfig) error { + return nil +} + +func (metricsPersistenceAgent) InstallSkill(context.Context, agent.Runtime, evalruntime.SkillConfig) error { + return nil +} + +func (metricsPersistenceAgent) Run(context.Context, agent.Runtime, agent.ExecOptions, []transcript.Message) (*agent.SessionResult, error) { + return &agent.SessionResult{FinalMessage: "done"}, nil +} + +// metricsPersistenceScript writes the platform's judge script, which emits one +// AVAILABLE metric carrying an explicit zero score. +func metricsPersistenceScript(t *testing.T) string { + t.Helper() + const payload = `{"schema_version":1,"invocation_id":"__INVOCATION__","metrics":[{"metric_id":"correctness","availability":"AVAILABLE","value":0}]}` + name, body := "judge.sh", "#!/bin/sh\ncat > \"$EVAL_METRICS_RESULT_PATH\" < Date: Fri, 30 Oct 2026 19:29:00 +0800 Subject: [PATCH 10/12] fix(judge): let execution outcomes outrank metric policy A below-threshold score must not turn a timeout ERROR into FAIL, because the case then loses the retryable timeout reason, and a script that exited nonzero must not be reclassified as a missing-required-score contract error. Report both metric decisions in the additive metric metadata instead, and stop injecting a synthetic compatibility assertion: the #309 assertion and member denominators stay exactly as the execution layer produced them. --- internal/evaluator/evaluator.go | 7 +- internal/evaluator/metrics_finalize.go | 104 ++++++++++++++++++------- internal/evaluator/multi_judge.go | 6 ++ internal/judge/metrics.go | 13 ++++ internal/judge/script.go | 61 ++++++++++++--- internal/judge/script_metrics_test.go | 35 +++++---- 6 files changed, 174 insertions(+), 52 deletions(-) diff --git a/internal/evaluator/evaluator.go b/internal/evaluator/evaluator.go index 5fb20203..e7d9db1e 100644 --- a/internal/evaluator/evaluator.go +++ b/internal/evaluator/evaluator.go @@ -1162,12 +1162,17 @@ func retryAllowed(policy config.RetryPolicy, reason string) bool { return false } +// retryReasonTimeout is the retry_policy reason a case fails with when a +// deadline expired; it must survive a metric policy decision so the case stays +// retryable for the right reason. +const retryReasonTimeout = "timeout" + func retryReasonForResult(result EvalResult) (string, bool) { if result.Status != judge.StatusError || result.Error == nil { return "", false } if isTimeoutError(result.Error) { - return "timeout", true + return retryReasonTimeout, true } return "error", true } diff --git a/internal/evaluator/metrics_finalize.go b/internal/evaluator/metrics_finalize.go index d2a42da0..889851e9 100644 --- a/internal/evaluator/metrics_finalize.go +++ b/internal/evaluator/metrics_finalize.go @@ -3,6 +3,7 @@ package evaluator import ( "context" "errors" + "strings" "github.com/alibaba/skill-up/internal/config" "github.com/alibaba/skill-up/internal/judge" @@ -32,16 +33,12 @@ func finalizeJudgeMetrics(ctx context.Context, result *EvalResult, cfg config.Ju result.Grading.Metrics = env } - production := judge.MetricProduced - if ctx.Err() != nil || (result.Grading != nil && result.Grading.Execution != nil && result.Grading.Execution.Completion == judge.CompletionInterrupted) { - production = judge.MetricPrevented - } - contractError := metricContractError(env, planned, production) + contractError := metricContractError(env, planned, metricProduction(ctx, result, stage)) switch { case contractError != "": applyMetricError(result, contractError) case env.Quality == judge.QualityNotMet: - applyMetricFailure(result) + applyMetricFailure(result, unmetThresholds(env, planned)) } result.Execution = deriveSingleExecution(ctx, result, env, planned, contractError, stage) } @@ -50,15 +47,32 @@ func caseMetricsEnvelope(ctx context.Context, result *EvalResult, cfg config.Jud if result.Grading != nil && result.Grading.Metrics != nil { return result.Grading.Metrics } - production := judge.MetricProduced - if stage != judge.StageStarted || ctx.Err() != nil { - production = judge.MetricPrevented - } problem := judge.ProblemNone if cfg.Type != "script" && stage == judge.StageStarted { problem = judge.ProblemChannelUnsupported } - return judge.MaterializeSlots(cfg.ID, planned, nil, problem, production) + return judge.MaterializeSlots(cfg.ID, planned, nil, problem, metricProduction(ctx, result, stage)) +} + +// metricProduction reports whether a scoring pass was allowed to reach a +// successful final outcome. A pass that never started, was stopped mid-flight, or +// ended as an unsuccessful process leaves its planned slots NOT_PRODUCED rather +// than MISSING, so an execution failure is never reported as omitted output. +func metricProduction(ctx context.Context, result *EvalResult, stage judge.Stage) judge.MetricProduction { + if stage != judge.StageStarted || ctx.Err() != nil { + return judge.MetricPrevented + } + if ex := gradingExecution(result); ex != nil && ex.Completion != judge.CompletionFinished { + return judge.MetricPrevented + } + return judge.MetricProduced +} + +func gradingExecution(result *EvalResult) *judge.ExecutionMetadata { + if result == nil || result.Grading == nil { + return nil + } + return result.Grading.Execution } func metricContractError(env *judge.MetricsEnvelope, planned []judge.MetricSlot, production judge.MetricProduction) string { @@ -74,28 +88,77 @@ func metricContractError(env *judge.MetricsEnvelope, planned []judge.MetricSlot, } for _, slot := range planned { metric, ok := byID[slot.ID] - if (!ok || metric.Availability != judge.Available) && (slot.Required || slot.MinScore != nil) { + if !ok { + if slot.Required || slot.MinScore != nil { + return "required named metric unavailable: " + slot.ID + } + continue + } + // A slot the execution never reached is an execution fact, not omitted + // scoring output: it must not reclassify the verdict or its cause. + if metric.Availability == judge.NotProduced { + continue + } + if metric.Availability != judge.Available && (slot.Required || slot.MinScore != nil) { return "required named metric unavailable: " + slot.ID } } return "" } +// unmetThresholds names the metrics whose collected value fell below the +// configured threshold, so the policy decision stays evidenced after the +// compatibility assertions are left untouched. +func unmetThresholds(env *judge.MetricsEnvelope, planned []judge.MetricSlot) []string { + byID := make(map[string]judge.MetricResult, len(env.Metrics)) + for _, metric := range env.Metrics { + byID[metric.MetricID] = metric + } + var unmet []string + for _, slot := range planned { + if slot.MinScore == nil { + continue + } + metric, ok := byID[slot.ID] + if !ok || metric.Availability != judge.Available || metric.Value == nil { + continue + } + if *metric.Value < *slot.MinScore { + unmet = append(unmet, slot.ID) + } + } + return unmet +} + +// applyMetricError records a metric contract violation. An execution error the +// judge already reported outranks it: the verdict, its error text and therefore +// its retry classification must survive. func applyMetricError(result *EvalResult, message string) { + if result.Status == judge.StatusError { + result.Metrics.AddDiagnostic("metric_contract_error: " + message) + return + } result.Status = judge.StatusError result.Error = errors.New(message) if result.Grading != nil { result.Grading.Status = judge.StatusError result.Grading.ErrorReason = &message - appendMetricFailure(result.Grading, message) } } -func applyMetricFailure(result *EvalResult) { +// applyMetricFailure turns a below-threshold score into a FAIL verdict without +// touching the assertion or member denominators, and never downgrades an +// execution outcome that already failed or errored. +func applyMetricFailure(result *EvalResult, unmet []string) { + if len(unmet) > 0 { + result.Metrics.AddDiagnostic("threshold_not_met: " + strings.Join(unmet, ",")) + } + if result.Status != judge.StatusPass { + return + } result.Status = judge.StatusFail if result.Grading != nil { result.Grading.Status = judge.StatusFail - appendMetricFailure(result.Grading, "one or more configured score thresholds were not met") } } @@ -114,14 +177,3 @@ func deriveSingleExecution(ctx context.Context, result *EvalResult, env *judge.M return &judge.ExecutionMetadata{Stage: judge.StageFinished, Completion: completionForContract(env, planned), Cause: judge.CauseNone} } } - -func appendMetricFailure(result *judge.Result, evidence string) { - result.AssertionResults = append(result.AssertionResults, judge.AssertionResult{ - Text: "named metric contract", Passed: false, Evidence: evidence, - }) - result.Summary.Total++ - result.Summary.Failed++ - if result.Summary.Total > 0 { - result.Summary.PassRate = float64(result.Summary.Passed) / float64(result.Summary.Total) - } -} diff --git a/internal/evaluator/multi_judge.go b/internal/evaluator/multi_judge.go index ebb7a602..6d6c4b0a 100644 --- a/internal/evaluator/multi_judge.go +++ b/internal/evaluator/multi_judge.go @@ -251,6 +251,12 @@ func memberExecution(graded EvalResult, stop judge.Stop, decision MemberDecision } func acceptedMemberExecution(graded EvalResult, stop judge.Stop, planned []judge.MetricSlot) *judge.ExecutionMetadata { + // A judge that reported how its own pass ended outranks a completion derived + // from slots: a script that exited unsuccessfully keeps FAILED/process_exit + // even though it reached a final outcome. + if graded.Execution != nil && graded.Execution.Cause == judge.CauseProcessExit { + return graded.Execution + } completion := judge.CompletionFinished env := graded.Metrics if graded.Grading != nil && graded.Grading.Metrics != nil { diff --git a/internal/judge/metrics.go b/internal/judge/metrics.go index 646a523c..8d308f4d 100644 --- a/internal/judge/metrics.go +++ b/internal/judge/metrics.go @@ -4,6 +4,7 @@ import ( "context" "errors" "math" + "slices" "sort" "strings" @@ -199,6 +200,18 @@ type MetricsEnvelope struct { Diagnostics []string `json:"diagnostics,omitempty"` } +// AddDiagnostic records an informational note about the metric pass. Diagnostics +// are additive output detail: they never create a planned slot, change coverage, +// or alter a verdict, so a policy decision can be evidenced without touching the +// compatibility assertion or member denominators. +func (e *MetricsEnvelope) AddDiagnostic(note string) { + if e == nil || note == "" || slices.Contains(e.Diagnostics, note) { + return + } + e.Diagnostics = append(e.Diagnostics, note) + sort.Strings(e.Diagnostics) +} + // ExecutionMetadata preserves how far evaluation got and why it stopped. It is // carried even when no scoring Result exists at all. type ExecutionMetadata struct { diff --git a/internal/judge/script.go b/internal/judge/script.go index 186eb214..1ff908ea 100644 --- a/internal/judge/script.go +++ b/internal/judge/script.go @@ -157,14 +157,7 @@ func (j *ScriptJudge) evaluateInRuntime(ctx context.Context, rt evalruntime.Runt } } if result.ExitCode != 0 { - detail := evidence - if detail == "" { - detail = fmt.Sprintf("script exited with code %d", result.ExitCode) - } - res := j.buildResult(false, detail, debugInfo, in) - // A nonzero exit never collects scores as a valid result (row P5): the file - // stays a diagnostic artifact only. - return res, nil + return j.unsuccessfulExitResult(evidence, debugInfo, result.ExitCode, in), nil } if evidence == "" { @@ -211,6 +204,53 @@ func (j *ScriptJudge) buildResult(passed bool, evidence, debugInfo string, in In return NewResult([]AssertionResult{assertion}, in.TurnsExecuted, in.TurnsTotal) } +// unsuccessfulExitResult maps a nonzero script exit onto the legacy FAIL verdict. +// With a metric registry configured, the execution record names the exit as the +// reason the case failed and no score the script wrote can be accepted: an +// unsuccessful process must not be reported as omitted scoring output. +func (j *ScriptJudge) unsuccessfulExitResult(evidence, debugInfo string, exitCode int, in Input) *Result { + detail := evidence + if detail == "" { + detail = fmt.Sprintf("script exited with code %d", exitCode) + } + res := j.buildResult(false, detail, debugInfo, in) + if len(j.MetricSlots) == 0 { + return res + } + res.Execution = &ExecutionMetadata{ + Stage: StageFinished, + Completion: CompletionFailed, + Cause: CauseProcessExit, + Detail: detail, + } + res.Metrics = j.declinedMetrics() + return res +} + +// producerID is the judge identity recorded in a metric envelope: the configured +// id, falling back to the judge type when a judge was built without one. +func (j *ScriptJudge) producerID() string { + if j.MetricJudgeID != "" { + return j.MetricJudgeID + } + return "script" +} + +// declinedMetrics materializes every planned slot as NOT_PRODUCED for a script +// that exited unsuccessfully. The result file is deliberately not read: it stays +// a diagnostic artifact, and its presence is noted so the run is explainable +// without pretending a failed process produced a score. +func (j *ScriptJudge) declinedMetrics() *MetricsEnvelope { + judgeID := j.producerID() + env := MaterializeSlots(judgeID, j.MetricSlots, nil, ProblemNone, MetricPrevented) + if j.MetricsResultPath != "" { + if info, err := os.Lstat(j.MetricsResultPath); err == nil && info.Mode().IsRegular() { + env.AddDiagnostic("result_file_not_accepted") + } + } + return env +} + // attachMetrics collects this invocation's structured metric result file and // materializes one slot per planned registry entry onto the judge result. // @@ -225,10 +265,7 @@ func (j *ScriptJudge) attachMetrics(res *Result, production MetricProduction) { if res == nil || len(j.MetricSlots) == 0 { return } - judgeID := j.MetricJudgeID - if judgeID == "" { - judgeID = "script" - } + judgeID := j.producerID() if j.MetricsResultPath == "" { // The channel is not available for this judge (a non-local runtime, or a // judge built without one). Report that honestly rather than implying a diff --git a/internal/judge/script_metrics_test.go b/internal/judge/script_metrics_test.go index ca42a9f7..87c96b4f 100644 --- a/internal/judge/script_metrics_test.go +++ b/internal/judge/script_metrics_test.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" "runtime" + "slices" "strings" "testing" @@ -28,8 +29,9 @@ type metricsScriptCase struct { expectValues map[string]float64 expectQuality Quality expectPlanned int - expectNilEnvelope bool expectReasons map[string]string + // expectDiagnostic is one required entry of the envelope's diagnostics list. + expectDiagnostic string } func (tc metricsScriptCase) scriptName() string { @@ -228,11 +230,21 @@ func metricsScriptAdditionalCases() []metricsScriptCase { }, }, { - name: "valid file then nonzero exit collects nothing", - posix: posixValidFileThenFail, - windows: windowsValidFileThenFail, - registry: twoMetricRegistry(), - expectNilEnvelope: true, + // A nonzero exit is an unsuccessful process: nothing it wrote can become + // a score, and the planned slots record that instead of looking omitted. + name: "valid file then nonzero exit collects no score", + posix: posixValidFileThenFail, + windows: windowsValidFileThenFail, + registry: twoMetricRegistry(), + expectAvailability: map[string]Availability{ + "correctness": NotProduced, "completeness": NotProduced, + }, + expectReasons: map[string]string{ + "correctness": "evaluation_prevented", "completeness": "evaluation_prevented", + }, + expectQuality: QualityNotAssessed, + expectPlanned: 2, + expectDiagnostic: "result_file_not_accepted", }, { name: "judge-declared SKIP keeps its own reason", @@ -284,14 +296,8 @@ func TestScriptJudgeCollectsNamedMetrics(t *testing.T) { func checkMetricsScriptCase(t *testing.T, tc metricsScriptCase) { t.Helper() res := runMetricsScriptCase(t, tc) - if tc.expectNilEnvelope { - if res.Metrics != nil { - t.Fatalf("expected no envelope after a nonzero exit, got %+v", res.Metrics) - } - return - } if res.Metrics == nil { - t.Fatal("expected a metrics envelope from the collected file") + t.Fatal("a configured registry must report one slot per planned metric") } env := res.Metrics if env.Coverage.Planned != tc.expectPlanned { @@ -304,6 +310,9 @@ func checkMetricsScriptCase(t *testing.T, tc metricsScriptCase) { t.Fatalf("quality = %s, want %s", env.Quality, tc.expectQuality) } checkMetricValues(t, env, tc) + if tc.expectDiagnostic != "" && !slices.Contains(env.Diagnostics, tc.expectDiagnostic) { + t.Fatalf("diagnostics = %v, want %q", env.Diagnostics, tc.expectDiagnostic) + } // The verdict mapping itself must not change because metrics exist. if res.Status != StatusPass && !tc.nameIsFailing() { t.Fatalf("status = %s, expected the legacy PASS for a zero-exit script", res.Status) From 0dfde3ad9508c63d685dc48311bcae1f30e82b6b Mon Sep 17 00:00:00 2001 From: Max_Guobaixin Date: Fri, 30 Oct 2026 19:29:02 +0800 Subject: [PATCH 11/12] test(judge): cover timeout, exit and denominator interactions on the real path Each case runs a child script through executeCase and asserts the execution outcome and the metric state together: a timeout with a below-threshold score on the single and grouped paths, a nonzero exit after writing a valid result file, and unchanged compatibility counts when the metric policy decides. --- .../evaluator/metrics_integration_test.go | 303 +++++++++++++++++- 1 file changed, 301 insertions(+), 2 deletions(-) diff --git a/internal/evaluator/metrics_integration_test.go b/internal/evaluator/metrics_integration_test.go index d76f79ae..697488ae 100644 --- a/internal/evaluator/metrics_integration_test.go +++ b/internal/evaluator/metrics_integration_test.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" goruntime "runtime" + "slices" "strings" "testing" "time" @@ -254,15 +255,313 @@ func TestExecuteCaseSingleScriptMetricContractControlsVerdict(t *testing.T) { } func metricsPayloadScript(t *testing.T, dir, payload string) string { + t.Helper() + return metricsScriptWithTail(t, dir, payload, "exit 0\n") +} + +// metricsScriptWithTail writes an invocation-bound result file, then runs tail, +// which carries the script's own wait and exit status. +func metricsScriptWithTail(t *testing.T, dir, payload, tail string) string { t.Helper() if goruntime.GOOS == windowsGOOS { - body := "$body = '" + payload + "' -replace '__INVOCATION__', $env:EVAL_METRICS_INVOCATION_ID\nSet-Content -LiteralPath $env:EVAL_METRICS_RESULT_PATH -Value $body\nexit 0\n" + body := "$body = '" + payload + "' -replace '__INVOCATION__', $env:EVAL_METRICS_INVOCATION_ID\nSet-Content -LiteralPath $env:EVAL_METRICS_RESULT_PATH -Value $body\n" + tail return metricsScript(t, dir, body) } - body := "#!/bin/sh\ncat > \"$EVAL_METRICS_RESULT_PATH\" < Date: Tue, 3 Nov 2026 03:46:49 +0800 Subject: [PATCH 12/12] fix(evaluator): make metric results and verdicts agree end to end A relative --output-dir used to hand the script judge a relative result path, which the child resolves against its case workspace while the collector resolves it against this process, so scores vanished; the channel now binds an absolute path. The judge phase and the metric finalizer work on a copy of the case record, so a threshold miss or a contract violation persisted FAIL/ERROR while the evaluator.case span and skill_up.case.run.count kept reporting the pre-judge PASS on both the single-shot and the multi-turn route: each now writes the final verdict back before the deferred telemetry reads it. Termination causes stay conservative. A skipped judge adopts a context stop only when that stop is the failure it was handed, so an unrelated engine error that lands after the case deadline expired still reports execution_error instead of blaming a knob skill-up never armed, and a judge that fails before producing any grading result counts as prevented rather than materializing MISSING slots. A metric payload carrying a number beyond float64 poisons only its own slot, because the structural scan no longer converts numbers, and a non-finite min_score is rejected at configuration validation since every comparison against NaN is false and quality derivation would then report MET for any score. --- internal/config/metrics_validation_test.go | 46 ++++ internal/config/validator.go | 7 +- internal/evaluator/evaluator.go | 44 +++- internal/evaluator/metrics_finalize.go | 12 +- .../evaluator/metrics_integration_test.go | 221 ++++++++++++++++++ internal/evaluator/metrics_otel_test.go | 151 ++++++++++++ internal/evaluator/metrics_wiring_test.go | 61 ++++- internal/evaluator/multi_judge.go | 32 ++- internal/evaluator/multiturn.go | 8 +- internal/judge/metrics_payload.go | 5 + internal/judge/script_metrics_test.go | 62 +++++ 11 files changed, 623 insertions(+), 26 deletions(-) create mode 100644 internal/evaluator/metrics_otel_test.go diff --git a/internal/config/metrics_validation_test.go b/internal/config/metrics_validation_test.go index 091caf66..d627b78b 100644 --- a/internal/config/metrics_validation_test.go +++ b/internal/config/metrics_validation_test.go @@ -1,6 +1,9 @@ package config import ( + "math" + "os" + "path/filepath" "strings" "testing" ) @@ -43,6 +46,16 @@ func TestValidateJudgeMetrics(t *testing.T) { metrics: []MetricConfig{{ID: "correctness"}, {ID: "correctness"}}, wantErrs: []string{`metrics[1].id "correctness" is duplicated`}, }, + { + name: "a NaN threshold is rejected because every comparison against it is false", + metrics: []MetricConfig{{ID: "correctness", MinScore: new(math.NaN())}}, + wantErrs: []string{"min_score must be between 0.0 and 1.0"}, + }, + { + name: "an infinite threshold is rejected", + metrics: []MetricConfig{{ID: "correctness", MinScore: new(math.Inf(1))}}, + wantErrs: []string{"min_score must be between 0.0 and 1.0"}, + }, { name: "an unsafe identifier is rejected", metrics: []MetricConfig{{ID: "Correctness!"}}, @@ -88,3 +101,36 @@ func TestMetricRegistryKeepsAnAbsentThresholdDistinctFromZero(t *testing.T) { t.Fatal("an explicit zero threshold was lost") } } + +// TestYAMLRegistryParsesNanThreshold pins the premise behind the non-finite +// rejection: the real YAML loader turns `min_score: .nan` into a NaN *float64, and +// every comparison against NaN is false, so an unvalidated registry would reach +// quality derivation and report MET for any available score. +func TestYAMLRegistryParsesNanThreshold(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "eval.yaml") + content := `schema_version: v1alpha1 +judge: + type: script + script_path: score.sh + metrics: + - id: correctness + min_score: .nan +cases: + files: [] +` + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + cfg, err := NewLoader(path).LoadEvalConfig() + if err != nil { + t.Fatalf("load: %v", err) + } + metric := cfg.Judge.Metrics[0] + if metric.MinScore == nil || !math.IsNaN(*metric.MinScore) { + t.Fatalf("min_score = %+v, want NaN parsed from `.nan`", metric.MinScore) + } + if errs := validateJudgeMetrics("judge.metrics", cfg.Judge.Metrics); len(errs) == 0 || !strings.Contains(errs[0], "min_score") { + t.Fatalf("validation errors = %v, want the NaN threshold rejected", errs) + } +} diff --git a/internal/config/validator.go b/internal/config/validator.go index 586f3973..9b014eca 100644 --- a/internal/config/validator.go +++ b/internal/config/validator.go @@ -3,6 +3,7 @@ package config import ( "fmt" "maps" + "math" "path" "regexp" "slices" @@ -657,7 +658,11 @@ func validateJudgeMetrics(configPath string, metrics []MetricConfig) []string { errs = append(errs, fmt.Sprintf("%s.metrics[%d].id %q is duplicated", configPath, i, metric.ID)) } seen[metric.ID] = true - if metric.MinScore != nil && (*metric.MinScore < 0.0 || *metric.MinScore > 1.0) { + // NaN slips through a bare range check because every comparison against it + // is false, and a non-finite threshold would then make the quality + // derivation report MET for any score. + if metric.MinScore != nil && (math.IsNaN(*metric.MinScore) || math.IsInf(*metric.MinScore, 0) || + *metric.MinScore < 0.0 || *metric.MinScore > 1.0) { errs = append(errs, fmt.Sprintf("%s.metrics[%d].min_score must be between 0.0 and 1.0", configPath, i)) } } diff --git a/internal/evaluator/evaluator.go b/internal/evaluator/evaluator.go index e7d9db1e..8e6c9536 100644 --- a/internal/evaluator/evaluator.go +++ b/internal/evaluator/evaluator.go @@ -42,6 +42,10 @@ var ( const judgeTypeAgentJudge = "agent_judge" +// caseTimeoutDeadlineSource names the configuration key that arms a case-level +// deadline, so an interrupted run can point users at the knob to raise. +const caseTimeoutDeadlineSource = "case.constraints.timeout_seconds" + // ProgressObserver receives progress notifications during evaluation. // Implementations must be safe for concurrent use. type ProgressObserver interface { @@ -467,7 +471,7 @@ func (e *defaultEvaluator) executeCaseOnce(ctx context.Context, caseCfg *config. result.Status = judge.StatusError result.Error = fmt.Errorf("agent %s preflight failed: %w", runAgent.Name(), err) result.Configuration = configName - e.recordSkippedJudgesForCase(&result, caseCfg, "agent_execution_failed") + e.recordSkippedJudgesForCase(ctx, err, &result, caseCfg, "agent_execution_failed") return result } } else { @@ -477,7 +481,7 @@ func (e *defaultEvaluator) executeCaseOnce(ctx context.Context, caseCfg *config. result.Status = judge.StatusError result.Error = err result.Configuration = configName - e.recordSkippedJudgesForCase(&result, caseCfg, "agent_execution_failed") + e.recordSkippedJudgesForCase(ctx, err, &result, caseCfg, "agent_execution_failed") return result } defer func() { _ = rt.Close() }() @@ -491,7 +495,7 @@ func (e *defaultEvaluator) executeCaseOnce(ctx context.Context, caseCfg *config. result.Status, result.Error = judge.StatusError, planErr return result } - if err := e.validateJudgePlan(judgePlan, &result); err != nil { + if err := e.validateJudgePlan(ctx, judgePlan, &result); err != nil { result.Status, result.Error = judge.StatusError, err return result } @@ -513,10 +517,13 @@ func (e *defaultEvaluator) executeCaseOnce(ctx context.Context, caseCfg *config. } multiTurnResult := e.executeMultiTurnCase(ctx, rt, caseCfg, configName, runAgent, agentExecOpts, startTime, judgeCfg, &result) multiTurnResult.Version = observation.Version - return multiTurnResult + // Same rule as the single-shot path: the deferred telemetry reads this + // record, so the multi-turn verdict has to land on it. + result = multiTurnResult + return result } if caseCfg.UserSimulator != nil { - recordSkippedJudges(&result, judgePlan, "session_resumption_unsupported") + recordSkippedJudges(ctx, nil, &result, judgePlan, "session_resumption_unsupported") result.Status = judge.StatusError result.Error = fmt.Errorf("agent %s does not support session resumption required by user simulation", runAgent.Name()) result.Configuration = configName @@ -570,7 +577,7 @@ func (e *defaultEvaluator) executeCaseOnce(ctx context.Context, caseCfg *config. // they are captured even when the agent failed or timed out. e.collectGlobArtifacts(ctx, rt, configName, caseCfg) if shouldReturn := e.handleExecutionResult(ctx, caseCfg, configName, startTime, &result, execErr); shouldReturn { - recordSkippedJudges(&result, judgePlan, "agent_execution_failed") + recordSkippedJudges(ctx, execErr, &result, judgePlan, "agent_execution_failed") return result } // A genuine non-zero exit may still be handled by expect.exit_code or a @@ -595,7 +602,12 @@ func (e *defaultEvaluator) executeCaseOnce(ctx context.Context, caseCfg *config. logging.DebugContextf(ctx, "Evaluator: case %s agent exited with code %d, proceeding to evaluation", caseCfg.ID, sessionResult.ExitCode) } - return e.evaluateCaseSession(ctx, rt, caseCfg, configName, judgeCfg, turnsTotal, runAgent, sessionResult, &result) + finalResult := e.evaluateCaseSession(ctx, rt, caseCfg, configName, judgeCfg, turnsTotal, runAgent, sessionResult, &result) + // The judge phase and the metric finalizer work on a copy of the case record. + // Write the final verdict back before the deferred telemetry reads it, or the + // case span and skill_up.case.run.count keep reporting the pre-judge status. + result = finalResult + return result } //nolint:spancheck // caller owns the returned span and must end it. @@ -639,7 +651,7 @@ func (e *defaultEvaluator) evaluateCaseSession( } if failed := e.runExpectPreCheck(ctx, caseCfg, configName, judgeCfg, judgeInput, result); failed { - recordSkippedJudges(result, plan, "gate_failed") + recordSkippedJudges(ctx, nil, result, plan, "gate_failed") if plan.Multi { result.Grading, _ = judge.DefaultOutcomeAggregator().Aggregate(result.JudgeResults, result.Turns, turnsTotal) result.Grading.Status = judge.StatusFail @@ -842,7 +854,15 @@ func (e *defaultEvaluator) configureMetricsChannel( logging.WarnContextf(ctx, "Evaluator: cannot generate metrics invocation id for case %s: %v", caseCfg.ID, err) return } - sj.MetricsResultPath = filepath.Join(dir, "metrics-"+invocation+".json") + // The path is handed to a child process whose working directory is the case + // workspace, while the collector reads it from this process: only an absolute + // path means the same file for both. + absDir, err := filepath.Abs(filepath.Clean(dir)) + if err != nil { + logging.WarnContextf(ctx, "Evaluator: cannot resolve metrics dir for case %s: %v", caseCfg.ID, err) + return + } + sj.MetricsResultPath = filepath.Join(absDir, "metrics-"+invocation+".json") sj.MetricsInvocationID = invocation } @@ -1071,12 +1091,12 @@ func (e *defaultEvaluator) resolveJudgeAgentConfig(judgeCfg config.JudgeConfig) return resolved, agent.ValidateJudgeAdapterConfig(resolved, e.resolver) } -func (e *defaultEvaluator) validateJudgePlan(plan config.JudgePlan, result *EvalResult) error { +func (e *defaultEvaluator) validateJudgePlan(ctx context.Context, plan config.JudgePlan, result *EvalResult) error { if plan.Multi { result.AggregationStrategy = judge.DefaultOutcomeAggregator().Strategy() } if err := e.validateJudgeAgents(plan); err != nil { - recordSkippedJudges(result, plan, "invalid_judge_config") + recordSkippedJudges(ctx, nil, result, plan, "invalid_judge_config") return err } return nil @@ -1141,7 +1161,7 @@ func annotateCaseTimeoutError(result *EvalResult, source string, seconds int) { func withCaseTimeout(ctx context.Context, defaultTimeoutSec, caseTimeoutSec int) (context.Context, context.CancelFunc, string, int) { if caseTimeoutSec > 0 { c, cancel := context.WithTimeout(ctx, time.Duration(caseTimeoutSec)*time.Second) - return c, cancel, "case.constraints.timeout_seconds", caseTimeoutSec + return c, cancel, caseTimeoutDeadlineSource, caseTimeoutSec } if defaultTimeoutSec > 0 { c, cancel := context.WithTimeout(ctx, time.Duration(defaultTimeoutSec)*time.Second) diff --git a/internal/evaluator/metrics_finalize.go b/internal/evaluator/metrics_finalize.go index 889851e9..f1b1dcb1 100644 --- a/internal/evaluator/metrics_finalize.go +++ b/internal/evaluator/metrics_finalize.go @@ -9,10 +9,13 @@ import ( "github.com/alibaba/skill-up/internal/judge" ) -func (e *defaultEvaluator) recordSkippedJudgesForCase(result *EvalResult, caseCfg *config.CaseConfig, reason string) { +// recordSkippedJudgesForCase materializes skipped judges for a case that never +// reached judging. cause is the error that ended the agent run, if any: it lets a +// case deadline or cancellation keep its own termination cause. +func (e *defaultEvaluator) recordSkippedJudgesForCase(ctx context.Context, cause error, result *EvalResult, caseCfg *config.CaseConfig, reason string) { plan, err := config.ResolveJudgePlan(e.evalCfg, caseCfg) if err == nil { - recordSkippedJudges(result, plan, reason) + recordSkippedJudges(ctx, cause, result, plan, reason) } } @@ -62,6 +65,11 @@ func metricProduction(ctx context.Context, result *EvalResult, stage judge.Stage if stage != judge.StageStarted || ctx.Err() != nil { return judge.MetricPrevented } + if result.Error != nil && result.Grading == nil { + // The judge failed before producing any grading result, so scoring never + // reached an outcome to omit: the slots were not produced. + return judge.MetricPrevented + } if ex := gradingExecution(result); ex != nil && ex.Completion != judge.CompletionFinished { return judge.MetricPrevented } diff --git a/internal/evaluator/metrics_integration_test.go b/internal/evaluator/metrics_integration_test.go index 697488ae..30243fcd 100644 --- a/internal/evaluator/metrics_integration_test.go +++ b/internal/evaluator/metrics_integration_test.go @@ -2,6 +2,7 @@ package evaluator import ( "context" + "errors" "os" "path/filepath" goruntime "runtime" @@ -14,6 +15,7 @@ import ( "github.com/alibaba/skill-up/internal/config" "github.com/alibaba/skill-up/internal/judge" "github.com/alibaba/skill-up/internal/runtime" + "github.com/alibaba/skill-up/pkg/transcript" ) // Integration tests for the named-metrics path through the real evaluator: a @@ -938,3 +940,222 @@ func TestExecuteCaseCancellationInterruptsRunningMember(t *testing.T) { t.Fatalf("group = %s/%s", result.GroupCompletion, result.GroupCause) } } + +// TestExecuteCaseRelativeOutputDirStillBindsMetricsResult covers the CLI shape +// where --output-dir is relative (the flag is documented as a plain directory and +// is passed through without resolution). The injected host path is handed to the +// script, whose working directory is the case workspace, while the collector reads +// the same string from the skill-up process's working directory: both sides have to +// agree on one file or every score is lost. +func TestExecuteCaseRelativeOutputDirStillBindsMetricsResult(t *testing.T) { + root := t.TempDir() + scriptDir := t.TempDir() + script := metricsScript(t, scriptDir, pick(posixScore, powershellScore)) + limit := 30 + member := config.JudgeConfig{ID: "scorer", Type: "script", ScriptPath: script, TimeoutSeconds: &limit, Metrics: scoreRegistry()} + t.Chdir(root) + e := newTestEvaluator(EvalOptions{ + Agent: &mockAgent{name: "test", output: "done"}, OutputDir: "results", + EvalCfg: &config.EvalConfig{Environment: config.Environment{Type: "none"}, Judge: member}, + }) + result := e.executeCase(context.Background(), &config.CaseConfig{ID: "relative-out", Input: config.Input{Prompt: "hello"}}, "with_skill", newScriptRuntime(t), nil) + if result.Status != judge.StatusPass || result.Metrics == nil || result.Metrics.Coverage.Available != 2 { + written, _ := filepath.Glob(filepath.Join(root, "results", "relative-out", "with_skill", "outputs", "judge", "run", "metrics-*.json")) + t.Fatalf("status=%s metrics=%+v: writer and reader disagreed on the result path; artifacts found under the process cwd: %v", + result.Status, result.Metrics, written) + } +} + +// TestExecuteCaseJudgeExecutionErrorWithoutResultKeepsExecutionCause covers the +// path where the judge never produced a grading result at all (an unreadable +// script, an upload failure). Inferring "scoring reached a final outcome" turns a +// plain execution failure into a metric contract error and reports MISSING instead +// of the slots that were never produced. +func TestExecuteCaseJudgeExecutionErrorWithoutResultKeepsExecutionCause(t *testing.T) { + limit := 30 + missing := filepath.Join(t.TempDir(), "absent.sh") + member := config.JudgeConfig{ID: "scorer", Type: "script", ScriptPath: missing, TimeoutSeconds: &limit, Metrics: scoreRegistry()} + e := newTestEvaluator(EvalOptions{ + Agent: &mockAgent{name: "test", output: "done"}, OutputDir: t.TempDir(), + EvalCfg: &config.EvalConfig{Environment: config.Environment{Type: "none"}, Judge: member}, + }) + result := e.executeCase(context.Background(), &config.CaseConfig{ID: "judge-no-result", Input: config.Input{Prompt: "hello"}}, "with_skill", newScriptRuntime(t), nil) + + if result.Status != judge.StatusError { + t.Fatalf("status = %s, want ERROR", result.Status) + } + if result.Execution == nil || result.Execution.Stage != judge.StageStarted || + result.Execution.Completion != judge.CompletionFailed || result.Execution.Cause != judge.CauseExecutionError { + t.Fatalf("execution = %+v, want STARTED/FAILED/execution_error", result.Execution) + } + if result.Metrics == nil || result.Metrics.Coverage.NotProduced != 2 { + t.Fatalf("metrics = %+v, want both planned slots NOT_PRODUCED", result.Metrics) + } +} + +// TestExecuteCaseContextStopSurvivesSkippedJudgeMaterialization checks that a case +// deadline or cancellation which stops the agent before judging keeps its real +// termination cause. Deriving the cause from the generic "agent_execution_failed" +// reason alone discards deadline/cancellation provenance for every skipped member. +func TestExecuteCaseContextStopSurvivesSkippedJudgeMaterialization(t *testing.T) { + registry := scoreRegistry() + blocking := &mockAgent{name: "test", runFunc: func(ctx context.Context, _ runtime.Runtime, _ agent.ExecOptions, _ []transcript.Message) (*agent.SessionResult, error) { + <-ctx.Done() + return nil, ctx.Err() + }} + + t.Run("case deadline", func(t *testing.T) { + members := []config.JudgeConfig{{ID: "a", Type: "script", ScriptPath: "unused-a", Metrics: registry}} + e := newTestEvaluator(EvalOptions{ + Agent: blocking, OutputDir: t.TempDir(), + EvalCfg: &config.EvalConfig{Environment: config.Environment{Type: "none"}, Judges: &members}, + }) + result := e.executeCase(context.Background(), &config.CaseConfig{ + ID: "deadline-skip", Input: config.Input{Prompt: "hello"}, + Constraints: config.Constraints{TimeoutSeconds: 1}, + }, "with_skill", newScriptRuntime(t), nil) + if len(result.JudgeResults) != 1 { + t.Fatalf("outcomes = %d", len(result.JudgeResults)) + } + member := result.JudgeResults[0] + if member.Execution == nil || member.Execution.Cause != judge.CauseDeadlineExpiry || + member.Execution.DeadlineSource != caseTimeoutDeadlineSource || + member.Execution.Timeout == nil || !member.Execution.Timeout.Occurred { + t.Fatalf("member execution = %+v, want the expired case deadline", member.Execution) + } + if result.GroupCause != judge.CauseDeadlineExpiry { + t.Fatalf("group cause = %s, want deadline_expiry", result.GroupCause) + } + }) + + t.Run("cancellation with a single judge", func(t *testing.T) { + member := config.JudgeConfig{ID: "solo", Type: "script", ScriptPath: "unused-solo", Metrics: registry} + e := newTestEvaluator(EvalOptions{ + Agent: blocking, OutputDir: t.TempDir(), + EvalCfg: &config.EvalConfig{Environment: config.Environment{Type: "none"}, Judge: member}, + }) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + result := e.executeCase(ctx, &config.CaseConfig{ID: "cancel-skip", Input: config.Input{Prompt: "hello"}}, "with_skill", newScriptRuntime(t), nil) + if result.Execution == nil || result.Execution.Cause != judge.CauseCancellation { + t.Fatalf("single execution = %+v, want cancellation", result.Execution) + } + if result.Metrics == nil || result.Metrics.Coverage.NotProduced != 2 { + t.Fatalf("single metrics = %+v, want both slots NOT_PRODUCED", result.Metrics) + } + }) +} + +// TestExecuteCaseUnrelatedAgentFailureAfterContextExpiryKeepsExecutionCause checks +// the other side of the skipped-judge precedence: an unrelated engine failure must +// not be relabelled as a cancellation just because the case context expired during +// or after it. Only a failure that is itself the context stop carries that cause. +func TestExecuteCaseUnrelatedAgentFailureAfterContextExpiryKeepsExecutionCause(t *testing.T) { + boom := errors.New("engine exploded") + ag := &mockAgent{name: "test", runFunc: func(context.Context, runtime.Runtime, agent.ExecOptions, []transcript.Message) (*agent.SessionResult, error) { + return nil, boom + }} + limit := 30 + members := []config.JudgeConfig{{ID: "a", Type: "script", ScriptPath: "unused-a", TimeoutSeconds: &limit, Metrics: scoreRegistry()}} + e := newTestEvaluator(EvalOptions{ + Agent: ag, OutputDir: t.TempDir(), + EvalCfg: &config.EvalConfig{Environment: config.Environment{Type: "none"}, Judges: &members}, + }) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + result := e.executeCase(ctx, &config.CaseConfig{ID: "unrelated-failure", Input: config.Input{Prompt: "hello"}}, "with_skill", newScriptRuntime(t), nil) + + if len(result.JudgeResults) != 1 { + t.Fatalf("outcomes = %d", len(result.JudgeResults)) + } + member := result.JudgeResults[0] + if member.Execution == nil || member.Execution.Cause != judge.CauseExecutionError { + t.Fatalf("member execution = %+v, want execution_error for an unrelated failure", member.Execution) + } + if member.Execution.Timeout != nil { + t.Fatalf("member execution invented a timeout: %+v", member.Execution) + } + if member.SkipReason != "agent_execution_failed" { + t.Fatalf("compat skip reason = %q", member.SkipReason) + } + if result.GroupCause != judge.CauseExecutionError { + t.Fatalf("group cause = %s, want execution_error", result.GroupCause) + } +} + +// multiTurnStopEvaluator builds an evaluator whose resumed turns die on the case +// context, with one registry-carrying script judge planned: a multi-turn case is +// exactly where a long run is cut short by the budget, and its skipped judges have +// to record that stop rather than a generic execution error. +func multiTurnStopEvaluator(t *testing.T, runTurn func(context.Context, runtime.Runtime, agent.ExecOptions, transcript.Message, string) (*agent.SessionResult, error)) *defaultEvaluator { + t.Helper() + ag := &mockResumerAgent{mockAgent: mockAgent{name: "test"}, runTurnFunc: runTurn} + members := []config.JudgeConfig{{ID: "a", Type: "script", ScriptPath: "unused-a", Metrics: scoreRegistry()}} + return newTestEvaluator(EvalOptions{ + Agent: ag, OutputDir: t.TempDir(), + EvalCfg: &config.EvalConfig{Environment: config.Environment{Type: "none"}, Judges: &members}, + }) +} + +func twoTurnCase(id string, timeoutSeconds int) *config.CaseConfig { + return &config.CaseConfig{ + ID: id, + Input: config.Input{Turns: []config.Turn{{Role: "user", Content: "first"}, {Role: "user", Content: "second"}}}, + Constraints: config.Constraints{TimeoutSeconds: timeoutSeconds}, + } +} + +// onlyStoppedMember returns the planned member's skipped outcome and checks the +// parts every context stop shares: the compat reason and the registry denominator. +func onlyStoppedMember(t *testing.T, result EvalResult) JudgeOutcome { + t.Helper() + if len(result.JudgeResults) != 1 { + t.Fatalf("outcomes = %d, want the one planned member", len(result.JudgeResults)) + } + member := result.JudgeResults[0] + if member.SkipReason != "agent_execution_failed" { + t.Fatalf("member reason = %q, want agent_execution_failed", member.SkipReason) + } + if member.Metrics == nil || member.Metrics.Coverage.NotProduced != 2 { + t.Fatalf("member metrics = %+v, want both planned slots NOT_PRODUCED", member.Metrics) + } + return member +} + +func TestExecuteCaseMultiTurnCaseDeadlineReachesSkippedJudgeCause(t *testing.T) { + e := multiTurnStopEvaluator(t, func(ctx context.Context, _ runtime.Runtime, _ agent.ExecOptions, _ transcript.Message, _ string) (*agent.SessionResult, error) { + <-ctx.Done() + return nil, ctx.Err() + }) + result := e.executeCase(context.Background(), twoTurnCase("multi-deadline", 1), "with_skill", newScriptRuntime(t), nil) + + member := onlyStoppedMember(t, result) + if member.Execution == nil || member.Execution.Cause != judge.CauseDeadlineExpiry || + member.Execution.DeadlineSource != caseTimeoutDeadlineSource || + member.Execution.Timeout == nil || !member.Execution.Timeout.Occurred { + t.Fatalf("member execution = %+v, want the expired case deadline", member.Execution) + } + if result.GroupCause != judge.CauseDeadlineExpiry { + t.Fatalf("group cause = %s, want deadline_expiry", result.GroupCause) + } +} + +func TestExecuteCaseMultiTurnCancellationReachesSkippedJudgeCause(t *testing.T) { + e := multiTurnStopEvaluator(t, func(ctx context.Context, _ runtime.Runtime, _ agent.ExecOptions, _ transcript.Message, _ string) (*agent.SessionResult, error) { + return nil, ctx.Err() + }) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + result := e.executeCase(ctx, twoTurnCase("multi-cancel", 0), "with_skill", newScriptRuntime(t), nil) + + member := onlyStoppedMember(t, result) + if member.Execution == nil || member.Execution.Cause != judge.CauseCancellation { + t.Fatalf("member execution = %+v, want cancellation", member.Execution) + } + if member.Execution.Timeout != nil { + t.Fatalf("a cancellation must not be reported as a timeout: %+v", member.Execution) + } + if result.Status != judge.StatusError { + t.Fatalf("status = %s, want ERROR", result.Status) + } +} diff --git a/internal/evaluator/metrics_otel_test.go b/internal/evaluator/metrics_otel_test.go new file mode 100644 index 00000000..ebb89016 --- /dev/null +++ b/internal/evaluator/metrics_otel_test.go @@ -0,0 +1,151 @@ +package evaluator + +import ( + "context" + "testing" + + "go.opentelemetry.io/otel" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + + "github.com/alibaba/skill-up/internal/config" + "github.com/alibaba/skill-up/internal/judge" +) + +// TestExecuteCaseMetricPolicyVerdictReachesCaseTelemetry checks that the metric +// policy's revised verdict reaches the shared case record before the deferred +// telemetry runs. The judge phase writes its grading into the caller's record and +// the finalizer then works on a copy, so a threshold miss or a contract violation +// can leave the persisted case FAIL/ERROR while the case span and the +// skill_up.case.run.count still say PASS. +func TestExecuteCaseMetricPolicyVerdictReachesCaseTelemetry(t *testing.T) { + cases := []struct { + name string + caseID string + payload string + want judge.Status + }{ + {name: "threshold miss", caseID: "otel-threshold-miss", payload: scorePayload("0.4"), want: judge.StatusFail}, + {name: "contract violation", caseID: "otel-contract-error", payload: `{"schema_version":1,"invocation_id":"__INVOCATION__","metrics":[]}`, want: judge.StatusError}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + recorder := tracetest.NewSpanRecorder() + provider := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder)) + original := otel.GetTracerProvider() + otel.SetTracerProvider(provider) + t.Cleanup(func() { + _ = provider.Shutdown(context.Background()) + otel.SetTracerProvider(original) + }) + + dir := t.TempDir() + limit := 30 + member := config.JudgeConfig{ + ID: "scorer", Type: "script", + ScriptPath: metricsPayloadScript(t, dir, tc.payload), + TimeoutSeconds: &limit, + Metrics: thresholdRegistry(), + } + e := newTestEvaluator(EvalOptions{ + Agent: &mockAgent{name: "test", output: "done"}, OutputDir: t.TempDir(), + EvalCfg: &config.EvalConfig{Environment: config.Environment{Type: "none"}, Judge: member}, + }) + result := e.executeCase(context.Background(), &config.CaseConfig{ID: tc.caseID, Input: config.Input{Prompt: "hello"}}, "with_skill", newScriptRuntime(t), nil) + // Both axes are reported, so a failing run shows the exact divergence the + // fix removes: persisted verdict vs case span status. + spanStatus := caseSpanStatus(t, recorder.Ended(), tc.caseID) + t.Logf("persisted verdict = %s, case span status = %s", result.Status, spanStatus) + if result.Status != tc.want { + t.Errorf("persisted verdict = %s error=%v, want %s", result.Status, result.Error, tc.want) + } + if spanStatus != string(tc.want) { + t.Errorf("case span status = %q, want %q to match the persisted verdict", spanStatus, tc.want) + } + }) + } +} + +// findEndedCaseSpan returns the completed evaluator.case span for one case id, so +// consecutive sub-tests cannot read each other's recording. +func findEndedCaseSpan(t *testing.T, spans []sdktrace.ReadOnlySpan, caseID string) sdktrace.ReadOnlySpan { + t.Helper() + for _, span := range spans { + if span.Name() != "evaluator.case" { + continue + } + for _, kv := range span.Attributes() { + if string(kv.Key) == "skill_up.case.id" && kv.Value.AsString() == caseID { + return span + } + } + } + t.Fatalf("no evaluator.case span recorded for case %q: %v spans", caseID, len(spans)) + return nil +} + +// TestExecuteCaseMultiTurnMetricPolicyVerdictReachesCaseTelemetry covers the same +// write-back on the multi-turn route: executeCaseOnce delegates to the multi-turn +// engine and used to return its result without updating the record the deferred +// telemetry reads, so a metric decision on a multi-turn case left the case span +// (and the case counter, which shares that one read) on the pre-judge status. +func TestExecuteCaseMultiTurnMetricPolicyVerdictReachesCaseTelemetry(t *testing.T) { + recorder := tracetest.NewSpanRecorder() + provider := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder)) + original := otel.GetTracerProvider() + otel.SetTracerProvider(provider) + t.Cleanup(func() { + _ = provider.Shutdown(context.Background()) + otel.SetTracerProvider(original) + }) + + dir := t.TempDir() + limit := 30 + member := config.JudgeConfig{ + ID: "scorer", + Type: "script", + ScriptPath: metricsPayloadScript(t, dir, scorePayload("0.4")), + TimeoutSeconds: &limit, + Metrics: thresholdRegistry(), + } + e := newTestEvaluator(EvalOptions{ + Agent: &mockResumerAgent{mockAgent: mockAgent{name: "test", output: "done"}}, + OutputDir: t.TempDir(), + EvalCfg: &config.EvalConfig{Environment: config.Environment{Type: "none"}, Judge: member}, + }) + caseCfg := &config.CaseConfig{ID: "otel-multi-turn", Input: config.Input{Turns: []config.Turn{ + {Role: "user", Content: "first"}, + {Role: "user", Content: "second"}, + }}} + result := e.executeCase(context.Background(), caseCfg, "with_skill", newScriptRuntime(t), &mockResumerAgent{mockAgent: mockAgent{name: "test", output: "done"}}) + + spanStatus := caseSpanStatus(t, recorder.Ended(), "otel-multi-turn") + t.Logf("persisted verdict = %s, case span status = %s", result.Status, spanStatus) + if result.Status != judge.StatusFail { + t.Errorf("persisted verdict = %s error=%v, want FAIL", result.Status, result.Error) + } + if result.Metrics == nil || result.Metrics.Quality != judge.QualityNotMet { + t.Fatalf("metrics = %+v, want NOT_MET", result.Metrics) + } + if result.Turns != 2 { + t.Fatalf("turns = %d, want the multi-turn route to have run both turns", result.Turns) + } + if spanStatus != string(judge.StatusFail) { + t.Errorf("case span status = %q, want FAIL to match the persisted verdict", spanStatus) + } +} + +// caseSpanStatus reads the case-status attribute the deferred telemetry writes. +// The counter datapoint uses the same read site (one local computed once), so this +// is where a stale case record would show up. +func caseSpanStatus(t *testing.T, spans []sdktrace.ReadOnlySpan, caseID string) string { + t.Helper() + span := findEndedCaseSpan(t, spans, caseID) + for _, kv := range span.Attributes() { + if string(kv.Key) == "skill_up.case.status" { + return kv.Value.AsString() + } + } + t.Fatalf("case span for %q has no skill_up.case.status attribute: %v", caseID, span.Attributes()) + return "" +} diff --git a/internal/evaluator/metrics_wiring_test.go b/internal/evaluator/metrics_wiring_test.go index e36cd608..036d8b60 100644 --- a/internal/evaluator/metrics_wiring_test.go +++ b/internal/evaluator/metrics_wiring_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "strings" "testing" "time" @@ -275,7 +276,7 @@ func TestSkippedJudgesRetainSlotsPerCause(t *testing.T) { for _, tc := range cases { t.Run(tc.reason, func(t *testing.T) { result := &EvalResult{} - recordSkippedJudges(result, config.JudgePlan{Judges: members, Multi: true}, tc.reason) + recordSkippedJudges(context.Background(), nil, result, config.JudgePlan{Judges: members, Multi: true}, tc.reason) if len(result.JudgeResults) != 2 { t.Fatalf("outcomes = %d, want 2", len(result.JudgeResults)) } @@ -299,9 +300,65 @@ func TestSkippedJudgesRetainSlotsPerCause(t *testing.T) { } } +// TestSkippedJudgeCauseKeepsSpecificReasonsWhenContextEnded checks the precedence +// between a context stop and the recorded reason: a context stop is only adopted +// when it is the same stop that ended the agent, so an unrelated failure after the +// context expired, and an agent reporting its own internal timeout while the case +// budget is still live, both keep the execution cause. A reason naming a specific +// cause (an expect gate, a contract violation) is never reclassified. +func TestSkippedJudgeCauseKeepsSpecificReasonsWhenContextEnded(t *testing.T) { + deadlineErr := fmt.Errorf("agent run: %w", context.DeadlineExceeded) + canceledErr := fmt.Errorf("agent run: %w", context.Canceled) + unrelatedErr := errors.New("engine exploded") + cases := []struct { + name string + stopped string + reason string + cause error + want judge.Cause + }{ + {"expired case deadline caused the failure", "expired", "agent_execution_failed", deadlineErr, judge.CauseDeadlineExpiry}, + {"cancellation caused the failure", "cancelled", "agent_execution_failed", canceledErr, judge.CauseCancellation}, + {"unrelated failure after the context expired", "cancelled", "agent_execution_failed", unrelatedErr, judge.CauseExecutionError}, + {"generic failure without an error object", "cancelled", "agent_execution_failed", nil, judge.CauseExecutionError}, + {"agent-internal timeout under a live case budget", "live", "agent_execution_failed", deadlineErr, judge.CauseExecutionError}, + {"turn error without a cause", "cancelled", "turn_error", nil, judge.CauseExecutionError}, + {"expect gate keeps its own cause", "cancelled", "gate_failed", canceledErr, judge.CauseExpectGate}, + {"contract reason is never reclassified", "expired", "invalid_judge_config", deadlineErr, judge.CauseContractError}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + stop := skippedJudgeStop(stoppedContext(tc.stopped), tc.cause, tc.reason) + if stop.Cause != tc.want || stop.Detail == "" { + t.Fatalf("stop = %+v, want cause %s with a reason detail", stop, tc.want) + } + }) + } + expiredStop := skippedJudgeStop(stoppedContext("expired"), deadlineErr, "agent_execution_failed") + if expiredStop.DeadlineSource != "case.constraints.timeout_seconds" { + t.Fatalf("expired case deadline must name the knob that armed it: %+v", expiredStop) + } + if execution := interruptedExecution(judge.StageNotInvoked, expiredStop); execution.Timeout == nil || !execution.Timeout.Occurred { + t.Fatalf("an expired deadline must stay visible as a timeout: %+v", execution) + } +} + +// stoppedContext builds the case context a skipped-judge call would observe. The +// table stores a state name instead of a context so no row carries a Context value. +func stoppedContext(state string) context.Context { + switch state { + case "expired": + return expiredDeadlineContext("case.constraints.timeout_seconds") + case "cancelled": + return canceledContext() + default: + return context.Background() + } +} + func TestSkippedJudgesAreNoopForSingleJudgePlans(t *testing.T) { result := &EvalResult{} - recordSkippedJudges(result, config.JudgePlan{Judges: []config.JudgeConfig{{ID: "a"}}}, "gate_failed") + recordSkippedJudges(context.Background(), nil, result, config.JudgePlan{Judges: []config.JudgeConfig{{ID: "a"}}}, "gate_failed") if len(result.JudgeResults) != 0 { t.Fatalf("single-judge plan gained member outcomes: %+v", result.JudgeResults) } diff --git a/internal/evaluator/multi_judge.go b/internal/evaluator/multi_judge.go index 6d6c4b0a..adf85167 100644 --- a/internal/evaluator/multi_judge.go +++ b/internal/evaluator/multi_judge.go @@ -340,15 +340,15 @@ func summarizeJudgeErrors(outcomes []JudgeOutcome) string { // keeps execution metadata and its registry-derived metric slots even though no // judge ran, so a gate or execution failure never loses that information. The // reason strings are unchanged for compatibility; the machine-readable cause is -// the new source of truth. -func recordSkippedJudges(result *EvalResult, plan config.JudgePlan, reason string) { - cause := causeForReason(reason) +// the new source of truth and prefers the stop the case context actually observed. +func recordSkippedJudges(ctx context.Context, cause error, result *EvalResult, plan config.JudgePlan, reason string) { + stop := skippedJudgeStop(ctx, cause, reason) if !plan.Multi { if len(plan.Judges) > 0 { member := plan.Judges[0] result.Metrics = preventedMetrics(member.ID, member.Metrics) if result.Metrics != nil { - result.Execution = interruptedExecution(judge.StageNotInvoked, judge.Stop{Cause: cause, Detail: reason}) + result.Execution = interruptedExecution(judge.StageNotInvoked, stop) } } return @@ -356,13 +356,35 @@ func recordSkippedJudges(result *EvalResult, plan config.JudgePlan, reason strin for _, member := range plan.Judges { result.JudgeResults = append(result.JudgeResults, JudgeOutcome{ ID: member.ID, Type: member.Type, Status: judge.StatusSkip, SkipReason: reason, - Execution: interruptedExecution(judge.StageNotInvoked, judge.Stop{Cause: cause, Detail: reason}), + Execution: interruptedExecution(judge.StageNotInvoked, stop), Metrics: preventedMetrics(member.ID, member.Metrics), }) } result.GroupCompletion, result.GroupCause = foldGroupState(result.JudgeResults) } +// skippedJudgeStop records why judging was skipped. A context stop wins over the +// generic reason only when it is the same stop that ended the agent: the case +// context is expired and the failure wraps that very error. Otherwise the cause +// would claim a knob skill-up never armed — an agent reporting its own internal +// timeout, or an unrelated failure after the context expired, stays an execution +// error. A reason that already names a specific cause (an expect gate, an invalid +// configuration) is never reclassified. +func skippedJudgeStop(ctx context.Context, cause error, reason string) judge.Stop { + stop := judge.Stop{Cause: causeForReason(reason), Detail: reason} + if stop.Cause != judge.CauseExecutionError || cause == nil { + return stop + } + ctxErr := ctx.Err() + if ctxErr == nil || !errors.Is(cause, ctxErr) { + return stop + } + return judge.ClassifyStop(cause, judge.StopObservation{ + DeadlineSource: DeadlineSource(ctx), + SignalObserved: SignalObserved(ctx), + }) +} + func causeForReason(reason string) judge.Cause { switch reason { case "gate_failed": diff --git a/internal/evaluator/multiturn.go b/internal/evaluator/multiturn.go index 6e81b5aa..8b4b129c 100644 --- a/internal/evaluator/multiturn.go +++ b/internal/evaluator/multiturn.go @@ -1092,7 +1092,7 @@ func (e *defaultEvaluator) executeMultiTurnCase( } result.Status = judge.StatusError result.Error = fmt.Errorf("agent execution failed: %w", execErr) - recordSkippedJudges(result, plan, "agent_execution_failed") + recordSkippedJudges(ctx, execErr, result, plan, "agent_execution_failed") result.Configuration = configName return *result } @@ -1101,13 +1101,13 @@ func (e *defaultEvaluator) executeMultiTurnCase( status := multiTurnStatus(turnResults) switch status { case judge.StatusError: - recordSkippedJudges(result, plan, "turn_error") + recordSkippedJudges(ctx, nil, result, plan, "turn_error") result.Status = judge.StatusError result.Configuration = configName return *result case judge.StatusFail: // Post-condition failure — mark FAIL, skip judge. - recordSkippedJudges(result, plan, "post_condition_failed") + recordSkippedJudges(ctx, nil, result, plan, "post_condition_failed") if result.DurationMs == 0 { result.DurationMs = time.Since(startTime).Milliseconds() } @@ -1115,7 +1115,7 @@ func (e *defaultEvaluator) executeMultiTurnCase( result.Configuration = configName return *result case judge.StatusSkip: - recordSkippedJudges(result, plan, "turns_skipped") + recordSkippedJudges(ctx, nil, result, plan, "turns_skipped") result.Status = judge.StatusSkip result.Configuration = configName return *result diff --git a/internal/judge/metrics_payload.go b/internal/judge/metrics_payload.go index 46d56746..cab1b4ac 100644 --- a/internal/judge/metrics_payload.go +++ b/internal/judge/metrics_payload.go @@ -120,6 +120,11 @@ func decodeMetricsEntry(entry map[string]json.RawMessage) (EmittedMetric, Payloa // payload can be trusted. func checkTopLevelKeys(raw []byte) PayloadProblem { decoder := json.NewDecoder(bytes.NewReader(raw)) + // Numbers stay as json.Number here: this scan only checks structure and key + // uniqueness, and converting a syntactically valid number that overflows + // float64 (1e309) would reject the whole payload instead of letting the entry + // decoder poison just that slot. + decoder.UseNumber() token, err := decoder.Token() if err != nil { return ProblemUnparseable diff --git a/internal/judge/script_metrics_test.go b/internal/judge/script_metrics_test.go index 87c96b4f..b4d0fd57 100644 --- a/internal/judge/script_metrics_test.go +++ b/internal/judge/script_metrics_test.go @@ -97,6 +97,17 @@ exit 0 windowsOneBadValue = "$body = '{\"schema_version\":1,\"invocation_id\":\"' + $env:EVAL_METRICS_INVOCATION_ID + '\",\"metrics\":[{\"metric_id\":\"correctness\",\"availability\":\"AVAILABLE\",\"value\":1.4,\"evidence\":[\"out of range\"]},{\"metric_id\":\"completeness\",\"availability\":\"AVAILABLE\",\"value\":0.4,\"evidence\":[\"fine\"]}]}'\n" + "Set-Content -LiteralPath $env:EVAL_METRICS_RESULT_PATH -Value $body\nexit 0\n" + // A JSON number that is syntactically valid but outside float64's range must + // poison only its own slot, because it is still structurally valid JSON. + posixOverflowNumber = `#!/bin/sh +cat > "$EVAL_METRICS_RESULT_PATH" < "$EVAL_METRICS_RESULT_PATH" < "$EVAL_METRICS_RESULT_PATH" <