feat(evaluator): roll up trial errors into summary and trial

packages/nemo_evaluator_sdk/examples/harbor/README.md

@@ -44,7 +44,7 @@ folders:
 
 ```
 hello_world_dataset/
-  hello-world/          # [task] name = "harbor/hello-world"
+  hello-world/                 # [task] name = "harbor/hello-world"
     task.toml
     instruction.md
     environment/Dockerfile
@@ -95,10 +95,10 @@ From the repository root:
 
 ```bash
 # Native path: run and print the SDK summary.
-python -m packages.nemo_evaluator_sdk.examples.harbor.run_harbor_example --mode native
+uv run python -m packages.nemo_evaluator_sdk.examples.harbor.run_harbor_example --mode native
 
 # Optimizer path: run, then rebuild NeMo Optimizer's legacy reward payload.
-python -m packages.nemo_evaluator_sdk.examples.harbor.run_harbor_example --mode optimizer
+uv run python -m packages.nemo_evaluator_sdk.examples.harbor.run_harbor_example --mode optimizer
 ```
 
 Both modes call `run_harbor_eval`; the only difference is what they print.

packages/nemo_evaluator_sdk/examples/harbor/hello_world_dataset/hello-world/environment/Dockerfile

@@ -1,7 +1,8 @@
 # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
-
-FROM alpine:3.22
+# Matches the tag CI pre-pulls in the integration job, so building this task image reuses that
+# layer
+FROM alpine:3.23
 
 # bash is required for Harbor's docker environment
 RUN apk add --no-cache bash

packages/nemo_evaluator_sdk/examples/harbor/run_harbor_example.py

@@ -9,8 +9,8 @@
 runs Harbor's ``JobConfig`` and scores the results; the caller never imports
 ``harbor`` or assembles a job.
 
-Two modes, both over the bundled ``hello_world_dataset`` (Harbor's ``hello-world``
-task) scored with the deterministic **oracle** agent, so no LLM/API key is needed:
+Two modes, both over the bundled ``hello_world_dataset`` scored with the
+deterministic **oracle** agent, so no LLM/API key is needed:
 
 * ``--mode native`` — print the SDK summary.
 * ``--mode optimizer`` — collapse the result into NeMo Optimizer's legacy
@@ -20,9 +20,9 @@
 
 Run it as a module from the repository root::
 
-    python -m packages.nemo_evaluator_sdk.examples.harbor.run_harbor_example --mode native
-    python -m packages.nemo_evaluator_sdk.examples.harbor.run_harbor_example --mode native --n-attempts 2
-    python -m packages.nemo_evaluator_sdk.examples.harbor.run_harbor_example --mode optimizer
+    uv run python -m packages.nemo_evaluator_sdk.examples.harbor.run_harbor_example --mode native
+    uv run python -m packages.nemo_evaluator_sdk.examples.harbor.run_harbor_example --mode native --n-attempts 2
+    uv run python -m packages.nemo_evaluator_sdk.examples.harbor.run_harbor_example --mode optimizer
 """
 
 from __future__ import annotations
@@ -70,13 +70,16 @@ async def _main(mode: str, jobs_dir: Path, *, n_attempts: int, job_name: str | N
     for score in result.scores:
         reward = score.outputs[0].value if score.outputs else None
         print(f"  {score.task_id}: reward={reward} status={score.status.value}")
+    for trial in result.trials:
+        if trial.error is not None:
+            print(f"  {trial.id}: error={trial.error.type}: {trial.error.message}")
 
 
 if __name__ == "__main__":
     if __package__ in {None, ""}:
         raise SystemExit(
             "Run this example as a module from the repository root:\n"
-            "  python -m packages.nemo_evaluator_sdk.examples.harbor.run_harbor_example --mode native"
+            "  uv run python -m packages.nemo_evaluator_sdk.examples.harbor.run_harbor_example --mode native"
         )
     logging.basicConfig(level=logging.INFO, format="%(levelname)s:%(name)s:%(message)s")
     parser = argparse.ArgumentParser(description=__doc__)

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/evaluator.py

@@ -192,7 +192,9 @@ async def run(
             tasks=task_list,
             trials=trial_list,
             scores=scores,
-            summary=AgentEvalSummary.from_scores(scores, tasks=task_list, extra_scores=runner_scores),
+            summary=AgentEvalSummary.from_scores(
+                scores, tasks=task_list, trials=trial_list, extra_scores=runner_scores
+            ),
             metadata=metadata,
             work_dir=runtime_config.work_dir,
         )
@@ -691,6 +693,8 @@ def _metric_row(task: AgentEvalTask, trial: AgentEvalTrial) -> dict[str, Any]:
             "id": trial.id,
             "task_id": trial.task_id,
             "status": trial.status.value,
+            # How the trial failed, for a metric that grades on it. None when the producer reported no failure.
+            "error": trial.error.model_dump(mode="json") if trial.error is not None else None,
             "metadata": trial.metadata,
         },
     }

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py

@@ -398,9 +398,56 @@ class AgentEvalSummary(BaseModel):
             }
         ],
     )
+    error_trial_ids: dict[str, list[str]] = Field(
+        default_factory=dict,
+        description=(
+            "Trials that errored, grouped by error type -- Harbor's 'exception_stats' shape. Values "
+            "are trial ids, not task ids: they join to AgentEvalTrial.id (trials.jsonl), "
+            "AgentEvalTaskScore.trial_id (scores.jsonl), and TrialMetricValue.trial_id in "
+            "task_metric_values. Membership is 'the trial carries an error', with no status filter -- "
+            "an errored Harbor trial is PARTIAL rather than FAILED so that it is still scored, and it "
+            "belongs here regardless. A trial that both errored and produced a reward therefore "
+            "appears here AND in task_metric_values, where it may even count as a pass; that is what "
+            "Harbor does too. Ids are appended in trial order and never deduplicated. Key order is "
+            "not meaningful -- summary.json is written with sorted keys. Empty is ambiguous and "
+            "stays that way: it means either no trial errored or no trials were supplied to "
+            "from_scores(). The field always serializes (it defaults to {}), so the two cases are "
+            "indistinguishable in summary.json -- read trial_count, or the trials themselves, to "
+            "tell them apart."
+        ),
+        examples=[
+            {
+                "RuntimeError": [
+                    "contract-review-msa-indemnity__k3f9wq2",
+                    "nda-scope-carveouts__p2hn8sc",
+                ],
+                "TimeoutError": ["merger-hsr-filing-threshold__w5db3qy"],
+            }
+        ],
+    )
     task_count: int = Field(default=0, description="Number of tasks represented in the run.")
     trial_count: int = Field(default=0, description="Number of distinct trials scored.")
     score_count: int = Field(default=0, description="Total number of metric scores.")
+    error_count: int = Field(
+        default=0,
+        description=(
+            "Number of trials that errored -- Harbor's 'n_errors'. Equals the total ids across "
+            "error_trial_ids; stated rather than derived so a non-Python reader of summary.json need "
+            "not sum a nested structure, matching the other counts here."
+        ),
+    )
+
+    @model_validator(mode="after")
+    def _error_count_matches_rollup(self) -> AgentEvalSummary:
+        """Keep the two error fields from disagreeing when a summary is built by hand.
+
+        ``from_scores`` derives both from one walk, but the model is public and directly
+        constructible -- and a count that contradicts the rollup beside it is worse than no count.
+        """
+        total = sum(len(ids) for ids in self.error_trial_ids.values())
+        if self.error_count != total:
+            raise ValueError(f"error_count {self.error_count} does not match {total} ids in error_trial_ids")
+        return self
 
     @property
     def scores_by_name(self) -> Mapping[str, AggregateScore]:
@@ -457,15 +504,23 @@ def from_scores(
         scores: Sequence[AgentEvalTaskScore],
         *,
         tasks: Sequence[AgentEvalTask] | None = None,
+        trials: Sequence[AgentEvalTrial] | None = None,
         extra_scores: Sequence[AggregateScore] = (),
     ) -> AgentEvalSummary:
-        """Build aggregated scores, task values, and coverage for a set of metric scores.
+        """Build aggregated scores, task values, coverage, and the error rollup for a set of scores.
 
         ``extra_scores`` are already-aggregated scores contributed by the runner (namespaced
         ``runner.<name>.``), merged in so a backend's own figures are addressable the same way as ours.
+
+        ``trials`` supplies the only thing scores cannot carry: what went wrong. Omitting it leaves
+        :attr:`error_trial_ids` empty rather than raising -- the same silent-skip contract ``tasks``
+        already has for pass@k. It may legitimately be *wider* than ``scores`` (a caller
+        re-aggregating a subset), so the rollup can name trial ids absent from
+        :attr:`task_metric_values`.
         """
         task_list = list(tasks) if tasks is not None else None
         task_metric_values = _task_metric_values(scores, task_list)
+        error_trial_ids = _error_trial_ids(trials)
         return AgentEvalSummary(
             scores=_aggregate_scores(
                 scores,
@@ -475,9 +530,11 @@ def from_scores(
             ),
             metric_coverage=_metric_coverage(scores, task_list),
             task_metric_values=task_metric_values,
+            error_trial_ids=error_trial_ids,
             task_count=len(task_list) if task_list is not None else len({score.task_id for score in scores}),
             trial_count=len({score.trial_id for score in scores}),
             score_count=len(scores),
+            error_count=sum(len(ids) for ids in error_trial_ids.values()),
         )
 
 
@@ -934,6 +991,32 @@ def _scorelike_outputs(tasks: Sequence[AgentEvalTask] | None) -> set[tuple[str,
     return scorelike
 
 
+def _error_trial_ids(trials: Sequence[AgentEvalTrial] | None) -> dict[str, list[str]]:
+    """Trial ids grouped by error type, in trial order — Harbor's ``exception_stats``.
+
+    Three trials, the middle one fine::
+
+        in   t0  error RuntimeError
+             t1  (no error)
+             t2  error RuntimeError
+             t3  error TimeoutError
+
+        out  {"RuntimeError": ["t0", "t2"], "TimeoutError": ["t3"]}
+
+    Selection is on ``trial.error``, never on ``trial.status``: an errored Harbor trial is ``PARTIAL``
+    so that it still scores, and filtering by status would drop exactly the trials this exists to name.
+
+    Ids are **appended**, never collected into a set or used as dict keys. Nothing enforces trial-id
+    uniqueness (Gym derives ids from a rollout index in two separate loops), and losing cardinality
+    here would understate the error count — the same rule ``task_metric_values`` follows.
+    """
+    grouped: dict[str, list[str]] = {}
+    for trial in trials or ():
+        if trial.error is not None:
+            grouped.setdefault(trial.error.type, []).append(trial.id)
+    return grouped
+
+
 def _task_metric_values(
     scores: Sequence[AgentEvalTaskScore],
     tasks: Sequence[AgentEvalTask] | None,