diff --git a/plugins/modelopt/skills/day0-release/SKILL.md b/plugins/modelopt/skills/day0-release/SKILL.md index ee58d078219..9b740512be2 100644 --- a/plugins/modelopt/skills/day0-release/SKILL.md +++ b/plugins/modelopt/skills/day0-release/SKILL.md @@ -270,6 +270,13 @@ the final path after any move. ## Triage (gate failure → decision) +For completed evaluations, apply +[evaluation run validation](../evaluation/references/run-validation.md) before +triage; its policy overrides retry actions below. Warn but do not retry solely +for failures below tolerance. For runs that fail tolerance or an independent +validation check, return findings and a recommendation to the parent/user for +review; do not automatically resubmit. + Map a gate's `failure_class` to the next action: | `failure_class` | Action | @@ -281,7 +288,7 @@ Map a gate's `failure_class` to the next action: | `CHECKPOINT_NOT_SERVABLE` | The Step 2b canary could not load/generate. Usually a tensor-naming or config-schema mismatch between the exporter and the serving stack, or missing/dangling auxiliary files (tokenizer). Fix the export; do not evaluate. | | `VERBOSITY_EXCEEDED` | Re-check run hygiene first (mixed reasoning effort, partial runs, unequal sample counts) — that has explained every false positive so far. If the delta survives matched, complete runs, it is a real behavioural change; do not publish on accuracy alone. | | `DEPLOYMENT_HEALTH_FAILED` | Drop to the **deployment** skill: reproduce serving standalone (`/health` + one generation), debug flags / image / TP / env, then carry the working command into NEL's `deployment.command` and retry the eval. If it can't serve, `POINT_INFEASIBLE`. | -| `EVAL_JUDGE_FAILED` | Usually transient (auth / rate limit) — wait and retry. | +| `EVAL_JUDGE_FAILED` | For completed runs, apply evaluation run validation above; return tolerance failures to the parent/user without automatic retry. Otherwise investigate auth/rate limits before retrying. | | `SAMPLE_ACCOUNTING_FAILED` | Investigate dropped/failed samples before trusting scores. | | `EXTERNAL_BASELINE_MISMATCH` | Investigate baseline configuration, correct it, rerun the baseline, and repeat external sanity before comparison. | | `USER_CONFIG_ERROR` | Correct it from the request, workspace, or model/config metadata and retry; if irrecoverable, return `ANOMALOUS` with evidence. | diff --git a/plugins/modelopt/skills/day0-release/scripts/gate_run.py b/plugins/modelopt/skills/day0-release/scripts/gate_run.py index d5dcbe94a70..408383819aa 100644 --- a/plugins/modelopt/skills/day0-release/scripts/gate_run.py +++ b/plugins/modelopt/skills/day0-release/scripts/gate_run.py @@ -28,10 +28,14 @@ "expected_samples": int, "scored_samples": int, "score": float | null, # canonical score, if extracted - "errors": [str, ...] # judge/parse/sample errors, if any + "errors": [str, ...], # unresolved validation blockers + "warnings": [str, ...] # policy-validated failure diagnostics } } } +Apply run-validation.md's bounded failure policy before constructing this +summary; this gate does not verify failure rates or raw evidence. Preserve +accepted failures in warnings and their original artifacts, never erase blockers. Only a terminal SUCCESS with complete, numeric scores passes. Non-terminal statuses (RUNNING/PENDING/TIMEOUT/RESUMING) do NOT pass — the run hasn't finished — but they classify as INFRA_TRANSIENT (wait for NEL to resume/finish; diff --git a/plugins/modelopt/skills/day0-release/tests/test_gates.py b/plugins/modelopt/skills/day0-release/tests/test_gates.py index b8217883e17..ddb576dd38d 100644 --- a/plugins/modelopt/skills/day0-release/tests/test_gates.py +++ b/plugins/modelopt/skills/day0-release/tests/test_gates.py @@ -151,6 +151,22 @@ def test_run_judge_error(): assert not r["pass"] and r["failure_class"] == "EVAL_JUDGE_FAILED" +def test_run_validated_warnings_do_not_waive_blockers(): + """Accept validated warnings without overlooking coverage or integrity blockers.""" + task = _task(warnings=["protocol-valid scored timeout; trial rate 1/100 = 1%"]) + assert evaluate_run({"tasks": {"t": task}})["pass"] + for blockers in ( + {"scored_samples": 99}, + {"score": None}, + {"errors": ["unscored harness crash"]}, + {"errors": ["secret leak"]}, + {"errors": ["wrong model/task/version/config"]}, + {"errors": ["cross-invocation contamination; no provenance"]}, + {"errors": ["systemic broken scoring"]}, + ): + assert not evaluate_run({"tasks": {"t": task | blockers}})["pass"] + + def test_run_missing_score(): r = evaluate_run({"tasks": {"gpqa": _task(score=None)}}) assert not r["pass"] and r["failure_class"] == "SAMPLE_ACCOUNTING_FAILED" diff --git a/plugins/modelopt/skills/evaluation/SKILL.md b/plugins/modelopt/skills/evaluation/SKILL.md index bf107f68e07..50c2b8eb66f 100644 --- a/plugins/modelopt/skills/evaluation/SKILL.md +++ b/plugins/modelopt/skills/evaluation/SKILL.md @@ -1,6 +1,6 @@ --- name: evaluation -description: Evaluates accuracy of quantized or unquantized LLMs using NeMo Evaluator Launcher (NEL). Triggers on "evaluate model", "benchmark accuracy", "run MMLU", "evaluate quantized model", "run nel". Handles deployment, config generation, and evaluation execution. Not for quantizing models (use ptq), deploying/serving models (use deployment), or comparing completed baseline-vs-quantized results (use compare-results). +description: Evaluates accuracy of quantized or unquantized LLMs using NeMo Evaluator Launcher (NEL). Triggers on "evaluate model", "benchmark accuracy", "run MMLU", "evaluate quantized model", "run nel". Handles deployment, config generation, and evaluation execution. Always load when validating completed runs or reporting evaluation scores, including standalone launching-evals analysis. Not for quantizing models (use ptq), deploying/serving models (use deployment), or comparing completed baseline-vs-quantized results (use compare-results). license: Apache-2.0 # Based on nel-assistant skill from NeMo Evaluator Launcher (commit f1fa073). # https://github.com/NVIDIA-NeMo/Evaluator/tree/f1fa073/packages/nemo-evaluator-launcher/.claude/skills/nel-assistant @@ -10,6 +10,10 @@ license: Apache-2.0 Guide the user through creating NEL YAML configs, running evaluations, and monitoring progress. +### Completed-run analysis + +For completed runs, skip launch/config; use launching-evals for artifacts/analysis and apply [run-validation.md](references/run-validation.md) before scores. Its policy overrides blanket uncapping and fixed token/context limits. + ### Workspace integration If `MODELOPT_WORKSPACE_ROOT` is set, use the common skill's `workspace-management.md` and reuse existing workspaces (this skill is usually the final stage of PTQ → Deploy → Eval; carry any deployment-time patches into `deployment.command`). @@ -130,10 +134,10 @@ Read [Step 8](references/launcher-workflow.md#step-8--run-evaluation-gated-dry-r ### Step 9 — Verify completed run -Read [run-validation.md](references/run-validation.md) before reporting scores: -validate logs and sample coverage, complete **Timeout and Output-Limit Accounting** -for every task, and report missing telemetry as unknown. For comparisons, also -apply its **External Baseline Sanity Check**, then use `compare-results`. +Apply [run-validation.md](references/run-validation.md)'s **Bounded Evaluation-Failure +Policy** and **Timeout and Output-Limit Accounting** before reporting scores. For +comparisons, also apply its **External Baseline Sanity Check**, then use +`compare-results`. --- diff --git a/plugins/modelopt/skills/evaluation/recipes/tasks/aa/scicode.md b/plugins/modelopt/skills/evaluation/recipes/tasks/aa/scicode.md index 29f2afce54b..d581ec7974e 100644 --- a/plugins/modelopt/skills/evaluation/recipes/tasks/aa/scicode.md +++ b/plugins/modelopt/skills/evaluation/recipes/tasks/aa/scicode.md @@ -44,7 +44,8 @@ when you have them, never fewer. - **Report the mean** of the n pooled runs, with the **sample** stdev (n-1 denominator) over `sqrt(n)` as the standard error, and n itself. Validate each run (`references/run-validation.md`) before averaging it in; - resubmit to replace invalid runs rather than pooling a sandbox-crashed one. + bounded protocol-valid sandbox/verifier failures may pass with warnings. Do not + pool incomplete/unscored runs; return blockers for parent review before reruns. - **Expect an export to time out.** The exports land on the CPU partition together and each reinstalls the launcher against a fixed 30 min sbatch limit. The score survives in the run artifacts — re-submit that run's diff --git a/plugins/modelopt/skills/evaluation/recipes/tasks/aa_next/terminal_bench_2_1.md b/plugins/modelopt/skills/evaluation/recipes/tasks/aa_next/terminal_bench_2_1.md index 5d0289f54c9..08e60b1f6bc 100644 --- a/plugins/modelopt/skills/evaluation/recipes/tasks/aa_next/terminal_bench_2_1.md +++ b/plugins/modelopt/skills/evaluation/recipes/tasks/aa_next/terminal_bench_2_1.md @@ -81,6 +81,11 @@ intended trial set. Sharding or concurrency changes can still affect scores when serving speed or queueing changes timeout rates. `shards: 4` suits 89 × r8 = 712 trials. Check `N × concurrency` against the Fargate quota and `N × gpus_per_node` against your allocation. +Apply `references/run-validation.md`'s bounded policy before reporting scores: +solver/harness timeouts and terminal transport/action errors may count only when +recorded and scored as protocol-valid failures. Keep all expected trials and +repeats, including zero scores; incomplete coverage remains a blocker. + ## Score Extraction Before reporting `pass@1`, complete the evaluation skill's diff --git a/plugins/modelopt/skills/evaluation/references/nel-next.md b/plugins/modelopt/skills/evaluation/references/nel-next.md index 14f4420aa06..ee9884eb3a7 100644 --- a/plugins/modelopt/skills/evaluation/references/nel-next.md +++ b/plugins/modelopt/skills/evaluation/references/nel-next.md @@ -246,4 +246,6 @@ kwargs after `--` (e.g. `-- -o copy_artifacts=true`). retries (≤8); lower `max_concurrent` if frequent. - `solver failed — grading 0.0` agent crashes are rare but real (e.g. terminus-2 tmux `send-keys`) — distinguish from genuine misses + task timeouts; the final - report breaks them out. + report breaks them out. Apply `run-validation.md`'s bounded failure policy: + verify protocol-valid scored failures and complete trial accounting. A harness + crash leaving missing/unscored trials is a blocker, not a zero score. diff --git a/plugins/modelopt/skills/evaluation/references/run-validation.md b/plugins/modelopt/skills/evaluation/references/run-validation.md index 2d3122dc60b..7e2713b3925 100644 --- a/plugins/modelopt/skills/evaluation/references/run-validation.md +++ b/plugins/modelopt/skills/evaluation/references/run-validation.md @@ -25,18 +25,59 @@ For each completed invocation/run directory, whether baseline, quantized, or a single-model run: 1. Inspect client, server/deployment, SLURM, judge, and task-specific/code-execution logs as applicable. Search for `Traceback`, `Exception`, `ERROR`, `FAILED`, `OOM`, `Killed`, `timeout`, `rate limit`, `unauthorized`, `connection refused/reset`, `health check`, `sandbox`, `container`, `judge`, `parse`, `scoring`, and task-specific failure strings. -2. Confirm the inference server loaded the intended checkpoint/model and stayed healthy through the run: no startup failure, mid-run crash/restart, OOM, request validation failure, max-context truncation, quantization load error, or repeated 4xx/5xx responses. -3. For judge-backed tasks, confirm judge calls succeeded and were parsed/scored correctly: no auth/rate-limit failures, malformed judge responses, invalid JSON, missing scores, or fallback/default scores. -4. For code-execution tasks, inspect executor/sandbox/container logs for setup failures, package install failures, timeouts, thread/process exhaustion, permission errors, harness crashes, or skipped tests that would make scores non-comparable. -5. Confirm sample accounting: expected samples/repeats match completed, scored samples; no unexpected dropped/skipped/failed samples, `unknown_agent_error`, `failed_samples_policy` aborts, empty outputs, or partial result files. -6. If reasoning traces are present, confirm they are parsed/stripped/ignored before scoring consistently. Assess output-limit termination such as `finish_reason: length` using the accounting below; it is not by itself a parsing failure. Check for parser errors, unmatched reasoning delimiters, reasoning text leaked into answers, answers stripped with the reasoning, or reasoning disabled when the config intended it to be active. +2. Verify the intended checkpoint loaded; inspect startup failures, crashes/restarts, OOMs, request errors, input/context clipping, and quantization load errors. Classify attributable per-trial failures under the policy below; systemic serving failures remain blockers. +3. For judge-backed tasks, inspect authentication, rate limits, malformed responses, parsing, and scoring. Distinguish protocol-valid failed scores from missing scores or undocumented fallback/default scores. +4. For code-execution tasks, inspect executor/sandbox/container setup, installation, timeout, resource, permission, harness, and skipped-test diagnostics. Verify each failed trial's protocol-defined outcome rather than treating every error as invalid. +5. Confirm coverage at every available level: expected, selected, accounted-for, and scored samples/repeats/trajectories must match. Explicit protocol-valid failed trials count as accounted-for; missing/unscored trials, dropped submissions, `failed_samples_policy` aborts leaving gaps, and partial results do not. +6. If reasoning traces are present, confirm they are parsed/stripped/ignored before scoring consistently. Assess unusable model outputs under the policy below. Check for parser or fallback errors, unmatched reasoning delimiters, reasoning text leaked into answers, answers stripped with the reasoning, or reasoning disabled when the config intended it to be active. 7. Complete the **Timeout and Output-Limit Accounting** below for every task, including non-reasoning models and successful runs. -Report the run-validation summary before any score: log scan status, sample -accounting, reasoning/answer parsing status, and any errors or warnings found. -If any validation item fails, either rerun/fix it or label the result as -incomplete or invalid. +Report the run-validation summary before any score: log scan status, coverage, +reasoning/answer parsing status, and any errors or warnings found. If an +independent validation item fails, label the result incomplete or invalid and +return findings and a recommendation to the parent (or user); do not +automatically resubmit a completed run. + +### Bounded Evaluation-Failure Policy (Parent and Evaluator) + +Accept complete runs with **<2% failures in each applicable gate**, measured +before rounding. Warn for nonzero rates; do not abort, invalidate, or retry solely +for bounded failures. At or above 2%, return findings to the parent/user without +automatic retry or a success verdict. + +| Gate | Failures | Denominator | +|---|---|---| +| Response | Truncated, empty-final, or malformed/unusable model outputs | Verified unique successful raw model responses selected for evaluation | +| Trial | Terminal request/server/transport, judge, sandbox/verifier, terminal action, solver, or harness errors/timeouts | Expected trials including repeats | + +Failures qualify only when preserved and recorded as incorrect/zero under the +existing benchmark protocol. Verify this from artifacts and logs, not exception +strings or summary zeros alone. Failed requests without raw responses belong only +in the trial gate; ordinary wrong answers are not runtime failures. + +Report category counts, deduplicated unions, denominators, and rates per +benchmark/run. Deduplicate overlaps, caches, and resumes by response/trial identity; +exclude judge responses and failed attempts from the response denominator. Never +add the two rates or pool benchmarks. Unknown/zero applicable denominators leave +validation incomplete; identify inapplicable gates explicitly. + +Retain all trials, zero scores, original score denominators, and private raw +evidence. Continue after a harness crash only if it records a protocol-valid +scored failure. Missing/unscored trials, fabricated or undocumented fallback +scores, secret leaks, wrong identity/configuration, contamination, provenance +failures, input/context clipping, systemic serving/scoring failures, and violated +benchmark rules remain blockers. Never select passing repeats or silently regrade. + +Keep protocol limits fixed, verify prompt/history plus output fits deployed +context, and report mismatches or unknown reference settings; proposed changes +need parent/user approval. Tolerance does not prove negligible score impact or +leaderboard comparability. + +Keep score, validation, and MLflow delivery separate. Carry counts, rates, and +warnings into handoffs. In day0 summaries, accepted diagnostics go in `warnings`; +unresolved blockers stay in `errors`. Apply this policy before `gate_run.py`, +which does not verify rates or raw evidence. ## Timeout and Output-Limit Accounting @@ -58,12 +99,13 @@ Missing termination metadata, sampled-only artifacts, or omitted failures make full-run rates **unknown**, not zero. Report coverage and what evidence is missing; do not extrapolate sampled rates. -- **Valid with warnings:** small fractions of output-limit or benchmark-timeout - events can pass when sample/repeat/scoring coverage is complete, limits match - the protocol, and other checks pass. Apply explicit task/user tolerances, - retain failures per benchmark scoring, and report observed impact. A nonzero - rate alone must not trigger failure or automatic rerun; a low rate does not - prove negligible impact or excuse infrastructure failures. +- **Terminal failures:** apply the bounded policy above to output faults and + protocol-valid scored runtime/trial failures, including harness timeouts. + Report timeout type, effective limits, concurrency/queueing, and protocol + differences; tolerance alone does not establish comparability. +- **Recovered retries:** report them separately. They do not enter the fault + numerator or denominator as extra responses, and they do not excuse a terminal + unscored failure or missing coverage. - **Provisional/inconclusive:** unknown accounting leaves validation incomplete. Investigate infrastructure failures, unexpected exclusions, or mismatched limits before a model-quality verdict. A valid run alone does not establish diff --git a/plugins/modelopt/skills/evaluation/tests/evals.json b/plugins/modelopt/skills/evaluation/tests/evals.json index 3a33a6af053..90947a46ba8 100644 --- a/plugins/modelopt/skills/evaluation/tests/evals.json +++ b/plugins/modelopt/skills/evaluation/tests/evals.json @@ -1,4 +1,37 @@ [ + { + "name": "bounded-failures-strict-boundary", + "skills": ["evaluation"], + "query": "Classify independent complete runs with 10000 expected scored trials and 10000 verified unique successful model responses each. Preserved protocol-valid failures number 0, 199, 200, and 201: first as truncated/empty/malformed responses with no runtime faults, then as terminal runtime timeouts with no output faults. Independent checks pass. Also classify 19999/1000000 failures, which displays as 2.00%. Do not launch jobs.", + "files": [], + "expected_behavior": [ + "Applies each gate separately: 0% passes without a failure warning; 1.99% passes with a warning; exactly 2% and 2.01% require parent review without success or automatic retry", + "Accepts 1.9999% with a warning despite its rounded 2.00% display", + "Retains failed scores and original denominators; does not infer negligible score impact" + ] + }, + { + "name": "bounded-failures-overlap-and-resume", + "skills": ["evaluation"], + "query": "A complete run has 1000 expected scored trials and 10000 verified unique successful model responses. Solver timeouts affect trials 1-12, harness errors 9-16, and terminal action failures 15-19. Resumed logs duplicate these records. Model response faults affect IDs 1-120 (truncated), 81-170 (empty-final), and 151-190 (malformed). All failures are preserved protocol-valid incorrect/zero outcomes; other checks pass. A second run has the same response faults but 20 runtime-failed trials. Do not launch jobs.", + "files": [], + "expected_behavior": [ + "Reports runtime categories 12, 8, 5, deduplicated to 19/1000 = 1.9%; response categories 120, 90, 40, deduplicated to 190/10000 = 1.9%", + "Accepts the first run with warnings without counting resumed records twice, pooling benchmarks, or adding gate rates", + "Does not accept the second run: its trial gate is exactly 2%, even though its response gate passes" + ] + }, + { + "name": "bounded-failures-scored-zeros-versus-missing", + "skills": ["evaluation"], + "query": "Run A has 100 expected trials, 100 scored, and one preserved harness timeout recorded zero under the benchmark protocol; its response gate is verified fault-free. Run B has 100 expected but only 99 scored after a crash. Run C fabricates a zero for that missing trial. Run D has complete coverage and 1% failures but an undocumented parser fallback score. Can these pass? Explain how to build day0 summaries; do not launch jobs or regrade.", + "files": [], + "expected_behavior": [ + "Accepts A with a 1% trial warning; blocks B, C, and D independently of apparent failure rate", + "Verifies coverage, identities, failure outcomes, and denominators from artifacts/logs before constructing summaries; a zero or exception string alone is insufficient", + "Keeps accepted diagnostics in warnings and unresolved blockers in errors; gate_run.py does not enforce failure rates or inspect raw evidence" + ] + }, { "name": "nemotron3-nano-bf16-reasoning", "skills": ["evaluation"], @@ -82,7 +115,7 @@ "Re-submits SciCode alone with -t ns_scicode rather than re-running the whole suite", "Reports the SciCode score as the mean of the pooled runs with standard error and run count", "Confirms cache replay from run provenance rather than equal scores before excluding a run", - "Re-submits replacements until at least 8 valid independent runs exist before reporting a mean", + "Requests parent/user approval to replace invalid runs until at least 8 valid independent runs exist; does not retry solely for bounded failures", "Reports SciCode as INDETERMINATE when fewer than 8 valid runs are available" ] }