From 0b1281e91e194359caddc13d2a459ac8bd980a95 Mon Sep 17 00:00:00 2001 From: Sandy Chapman Date: Mon, 10 Aug 2026 12:21:35 -0300 Subject: [PATCH 1/3] feat(evaluator): give AgentEvalResult the dataset path's display surface `EvaluationResult` ships format_summary/print_summary/to_records/to_table/ to_pandas/__str__. `AgentEvalResult` had none of them, so anyone inspecting an agent-eval run wrote their own formatting -- two result types in one SDK with unrelated ergonomics. Adds the same six methods, with the same signatures and the same `view` values. ## No mixin The ticket proposed extracting a shared mixin. `BenchmarkEvaluationResult` (values/multi_metric_results.py) already faced this and answered it differently: import the module-level helpers, implement the methods directly. The genuinely shared logic -- format_table, summary_aggregate_record, serialize_value, flatten_dict -- is already free functions, so a mixin would share method *names* while every body still needed overriding, and would add a third pattern to unify two that already agree. This follows the established one instead. ## What a row is here A record per metric score. The fan-out is preserved rather than collapsed: task_id and trial_id are columns, so a consumer can still group by task, which is what pass@k depends on. The aggregate view is byte-for-byte the dataset path's -- percentiles flattened, histograms as JSON strings. The error section is agent-eval's own, because it has a distinction the dataset path cannot make: a failed *trial* is an attempt the agent is answerable for, a failed *metric* is a measurement that never happened. Both arrive as FAILED. ## to_table column union (deliberate divergence) `pa.Table.from_pylist` takes its schema from the first record alone. Here `error` and `diagnostics.*` appear only on failed scores, so a run whose first score passed exported a table missing exactly the columns you needed, while to_pandas -- which unions keys -- included them. Verified: 5 columns vs 7. Columns are now unioned before the table is built. Both siblings have the same latent behaviour; matching a known-lossy export seemed worse than diverging and saying so. Worth a follow-up for EvaluationResult and BenchmarkEvaluationResult. Signed-off-by: Sandy Chapman --- .../nemo_evaluator_sdk/agent_eval/results.py | 248 +++++++++++++++++- .../tests/agent_eval/test_result_display.py | 209 +++++++++++++++ 2 files changed, 454 insertions(+), 3 deletions(-) create mode 100644 packages/nemo_evaluator_sdk/tests/agent_eval/test_result_display.py diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py index b31d1b5078..5fd447ac1d 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py @@ -5,19 +5,35 @@ from __future__ import annotations +import json import math from collections.abc import Sequence from datetime import datetime from pathlib import Path - -from nemo_evaluator_sdk.agent_eval.scores import AgentEvalScoreStatus, AgentEvalTaskScore, is_trial_failure +from typing import Any + +from nemo_evaluator_sdk.agent_eval.scores import ( + AgentEvalDiagnosticSeverity, + AgentEvalScoreStatus, + AgentEvalTaskScore, + is_trial_failure, +) from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalTask, SemanticReducer, ViewSignal from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial, RunnerInfo from nemo_evaluator_sdk.metrics.aggregation import compute_percentiles from nemo_evaluator_sdk.metrics.protocol import MetricOutput from nemo_evaluator_sdk.metrics.utils import metric_type_name from nemo_evaluator_sdk.values.protocol import BooleanValue, ContinuousScore -from nemo_evaluator_sdk.values.results import AggregatedMetricResult, AggregateRangeScore, AggregateScore +from nemo_evaluator_sdk.values.results import ( + AggregatedMetricResult, + AggregateRangeScore, + AggregateScore, + ResultView, + flatten_dict, + format_table, + serialize_value, + summary_aggregate_record, +) from pydantic import BaseModel, ConfigDict, Field #: Metric-output value schemas eligible for pass@k (a per-attempt "did it pass?" signal). Labels, @@ -179,6 +195,232 @@ def persist(self, output_dir: str | Path | None = None, *, write_dashboard: bool ) return persist_run(self, target, write_html_dashboard=write_dashboard) + def to_records(self, view: ResultView = "rows") -> list[dict[str, Any]]: + """Convert this run into flat dictionaries for export or inspection. + + ``view="rows"`` yields one record per metric score — the agent-eval analogue of the dataset + path's row. The fan-out is preserved rather than collapsed: ``task_id`` and ``trial_id`` are + columns, so a consumer can still group by task, which is what pass@k depends on. + + ``view="aggregate"`` matches the dataset path exactly — percentiles flattened, histograms + kept as JSON strings so the view stays tabular. + + Args: + view: Output projection, either ``"rows"`` or ``"aggregate"``. + + Returns: + Flat record dictionaries for downstream table/dataframe conversion. + + Raises: + ValueError: If ``view`` is unsupported. + """ + if view == "rows": + return [_score_record(score) for score in self.scores] + + if view == "aggregate": + records: list[dict[str, Any]] = [] + for score in self.summary.scores.scores: + record: dict[str, Any] = {} + for key, value in score.model_dump(mode="json").items(): + if key == "percentiles" and isinstance(value, dict): + flatten_dict("percentiles", value, record) + elif key == "histogram" and value is not None: + # Histograms stay as JSON strings so aggregate views remain tabular instead + # of expanding variable-width nested columns. + record[key] = json.dumps(value, sort_keys=True) + else: + record[key] = value + records.append(record) + return records + + raise ValueError(f"Unsupported view {view!r}. Expected 'rows' or 'aggregate'.") + + def to_table(self, view: ResultView = "rows"): + """Convert records into a ``pyarrow.Table``. + + Args: + view: Output projection, either ``"rows"`` or ``"aggregate"``. + + Columns are unioned across every record before the table is built. ``pa.Table.from_pylist`` + takes its schema from the first record alone, and in a row view ``error`` and + ``diagnostics.*`` appear only on failed scores — so a run whose first score succeeded would + otherwise export a table with the failure columns silently missing. ``to_pandas`` already + unions keys, and the two should not disagree about what a run contains. + + Args: + view: Output projection, either ``"rows"`` or ``"aggregate"``. + + Returns: + Table built from ``to_records(view=view)``. + """ + import pyarrow as pa + + records = self.to_records(view=view) + # dict-of-None preserves first-appearance order, matching how format_table derives columns. + columns = {key: None for record in records for key in record} + return pa.Table.from_pylist([{key: record.get(key) for key in columns} for record in records]) + + def to_pandas(self, view: ResultView = "rows"): + """Convert records into a pandas ``DataFrame``. + + Args: + view: Output projection, either ``"rows"`` or ``"aggregate"``. + + Returns: + DataFrame built from ``to_records(view=view)``. + """ + import pandas as pd + + return pd.DataFrame.from_records(self.to_records(view=view)) + + def format_summary(self, max_rows: int = 10, *, max_error_rows: int | None = None) -> str: + """Render a human-readable summary with aggregates and a score preview. + + Args: + max_rows: Maximum number of score records included in the preview. + max_error_rows: Maximum number of failed scores included in the error-details section. + Defaults to ``max_rows``. + + Returns: + Multi-line summary string suitable for terminal/notebook display. + """ + if max_error_rows is None: + max_error_rows = max_rows + aggregate_records = [summary_aggregate_record(score) for score in self.summary.scores.scores] + preview = [_score_preview_record(score) for score in self.scores[:max_rows]] + parts = [ + _agent_eval_summary_header(self), + "", + "Aggregate scores", + format_table(aggregate_records), + ] + if preview: + parts.extend( + [ + "", + f"Score preview (first {len(preview)} of {len(self.scores)})", + format_table(preview), + ] + ) + parts.extend(_format_score_errors(self.scores, max_error_rows=max_error_rows)) + return "\n".join(parts) + + def print_summary(self, max_rows: int = 10, *, max_error_rows: int | None = None) -> None: + """Print ``format_summary`` output. + + Args: + max_rows: Maximum number of score records included in the preview. + max_error_rows: Maximum number of failed scores included in the error-details section. + Defaults to ``max_rows``. + """ + print(self.format_summary(max_rows=max_rows, max_error_rows=max_error_rows)) + + def __str__(self) -> str: + """Return the default compact summary representation. + + Returns: + Summary string with up to five preview scores. + """ + return self.format_summary(max_rows=5) + + +def _score_error_text(score: AgentEvalTaskScore) -> str | None: + """Join the error-severity diagnostic messages for a score, or None when it has none.""" + messages = [ + diagnostic.message + for diagnostic in score.diagnostics + if diagnostic.severity is AgentEvalDiagnosticSeverity.ERROR + ] + return "; ".join(messages) if messages else None + + +def _score_diagnostics_columns(score: AgentEvalTaskScore) -> dict[str, str]: + """JSON-encoded diagnostic columns, keyed ``diagnostics.``. + + Encoded as compact JSON for the same reason the dataset path does it: diagnostics have a + metric-defined shape, and exports stay flat only if that shape is a string. + """ + if not score.diagnostics: + return {} + return { + f"diagnostics.{score.metric_type}": json.dumps( + [serialize_value(diagnostic) for diagnostic in score.diagnostics], sort_keys=True + ) + } + + +def _score_preview_record(score: AgentEvalTaskScore) -> dict[str, Any]: + """Identity and status columns shared by the row export and the summary preview.""" + record: dict[str, Any] = { + "task_id": score.task_id, + "trial_id": score.trial_id, + "metric_type": score.metric_type, + "status": score.status.value, + } + for output in score.outputs: + record[f"output.{output.name}"] = serialize_value(output.value) + return record + + +def _score_record(score: AgentEvalTaskScore) -> dict[str, Any]: + """Full export record for one score: preview columns plus error text and diagnostics.""" + record = _score_preview_record(score) + if error_text := _score_error_text(score): + record["error"] = error_text + record.update(_score_diagnostics_columns(score)) + return record + + +def _agent_eval_summary_header(result: AgentEvalResult) -> str: + """Build the header line, mirroring the shape :func:`summary_header` produces for row results. + + The counts differ because the units do — a run has tasks, trials, and scores where the dataset + path has rows — but the ``Name(field=value, ...)`` shape is the same, and a status the run never + produced is left out, matching what that header does with its zero counts. + + Statuses are counted by tallying the scores present, so an absent status simply never becomes a + key; there is no zero to filter out. + """ + status_counts: dict[str, int] = {} + for score in result.scores: + status_counts[score.status.value] = status_counts.get(score.status.value, 0) + 1 + fields = [ + f"tasks={len(result.tasks)}", + f"trials={len(result.trials)}", + f"scores={len(result.scores)}", + f"aggregate_scores={len(result.summary.scores.scores)}", + ] + fields.extend(f"{status}={count}" for status, count in sorted(status_counts.items())) + return f"AgentEvalResult({', '.join(fields)})" + + +def _format_score_errors( + scores: Sequence[AgentEvalTaskScore], + *, + max_error_rows: int | None, +) -> list[str]: + """Render the failed-score detail section, separating a failed trial from a failed metric. + + Both arrive as ``FAILED``, but they mean different things to a reader: a failed trial is an + attempt the agent is answerable for, a failed metric is a measurement that never happened. The + dataset path has no equivalent distinction to make, so this section is agent-eval's own rather + than a reuse of :func:`format_error_details`. + """ + failed = [score for score in scores if score.status is AgentEvalScoreStatus.FAILED] + if not failed: + return [] + + shown_limit = len(failed) if max_error_rows is None else max(0, max_error_rows) + shown = failed[:shown_limit] + parts = ["", f"Error details ({len(shown)} of {len(failed)} failed scores)"] + for score in shown: + kind = "failed trial" if is_trial_failure(score) else "failed metric" + parts.extend(["", f"[{score.task_id} / {score.trial_id} / {score.metric_type}] {kind}"]) + parts.append(_score_error_text(score) or "(no error-severity diagnostic recorded)") + if len(shown) < len(failed): + parts.extend(["", f"... {len(failed) - len(shown)} more failed scores omitted"]) + return parts + def _aggregate_scores( scores: Sequence[AgentEvalTaskScore], diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_result_display.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_result_display.py new file mode 100644 index 0000000000..79d09e88a7 --- /dev/null +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_result_display.py @@ -0,0 +1,209 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Display and export surface on AgentEvalResult, mirroring the dataset path's.""" + +from __future__ import annotations + +import pytest +from nemo_evaluator_sdk.agent_eval.results import AgentEvalResult, AgentEvalSummary +from nemo_evaluator_sdk.agent_eval.scores import ( + TRIAL_STATUS_DETAIL, + AgentEvalDiagnostic, + AgentEvalDiagnosticSeverity, + AgentEvalScoreStatus, + AgentEvalTaskScore, +) +from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrialStatus +from nemo_evaluator_sdk.values.protocol import MetricOutput + +_TRIAL_FAILURE = AgentEvalDiagnostic( + severity=AgentEvalDiagnosticSeverity.ERROR, + message="agent crashed", + details={TRIAL_STATUS_DETAIL: AgentEvalTrialStatus.FAILED.value}, +) +_METRIC_FAILURE = AgentEvalDiagnostic( + severity=AgentEvalDiagnosticSeverity.ERROR, + message="judge timed out", + details={"exception_type": "TimeoutError"}, +) + + +def _score( + task_id: str, + trial_id: str, + value: float | None = None, + *, + status: AgentEvalScoreStatus = AgentEvalScoreStatus.COMPLETED, + diagnostics: tuple[AgentEvalDiagnostic, ...] = (), +) -> AgentEvalTaskScore: + return AgentEvalTaskScore( + id=f"{task_id}-{trial_id}", + run_id="run-1", + task_id=task_id, + trial_id=trial_id, + metric_type="reward", + status=status, + outputs=[MetricOutput(name="reward", value=value)] if value is not None else [], + diagnostics=list(diagnostics), + ) + + +def _result(*scores: AgentEvalTaskScore) -> AgentEvalResult: + return AgentEvalResult( + run_id="run-1", + tasks=[], + trials=[], + scores=list(scores), + summary=AgentEvalSummary.from_scores(scores), + ) + + +def _passing_run() -> AgentEvalResult: + return _result(_score("t1", "a", 1.0), _score("t1", "b", 0.0), _score("t2", "a", 1.0)) + + +def _run_with_failures() -> AgentEvalResult: + return _result( + _score("t1", "a", 1.0), + _score("t2", "b", status=AgentEvalScoreStatus.FAILED, diagnostics=(_TRIAL_FAILURE,)), + _score("t3", "a", status=AgentEvalScoreStatus.FAILED, diagnostics=(_METRIC_FAILURE,)), + ) + + +def test_row_records_keep_the_task_grouping_that_pass_at_k_depends_on() -> None: + # The fan-out is load-bearing: flattening trials into anonymous rows would make it impossible to + # regroup by task, so task_id and trial_id have to survive as columns. + records = _passing_run().to_records() + + assert [record["task_id"] for record in records] == ["t1", "t1", "t2"] + assert [record["trial_id"] for record in records] == ["a", "b", "a"] + assert records[0]["output.reward"] == 1.0 + + +def test_row_records_carry_error_text_and_diagnostics_only_for_failures() -> None: + records = _run_with_failures().to_records() + + assert "error" not in records[0] + assert records[1]["error"] == "agent crashed" + assert "diagnostics.reward" in records[1] + + +def test_aggregate_view_flattens_percentiles_like_the_dataset_path() -> None: + records = _passing_run().to_records(view="aggregate") + + assert records[0]["name"] == "reward.reward" + # Percentiles are flattened into columns rather than left nested, so the view stays tabular. + assert "percentiles.p50" in records[0] + assert "percentiles" not in records[0] + + +def test_unsupported_view_is_rejected_with_the_same_message_as_the_dataset_path() -> None: + with pytest.raises(ValueError, match=r"Unsupported view 'nope'. Expected 'rows' or 'aggregate'."): + _passing_run().to_records(view="nope") # ty: ignore[invalid-argument-type] + + +def test_to_table_keeps_columns_that_only_appear_on_later_records() -> None: + # pa.Table.from_pylist takes its schema from the first record. Error and diagnostics columns + # appear only on failures, so a run whose first score passed would silently export without them. + table = _run_with_failures().to_table() + + assert "error" in table.column_names + assert "diagnostics.reward" in table.column_names + + +def test_to_table_and_to_pandas_agree_on_columns() -> None: + result = _run_with_failures() + + assert set(result.to_table().column_names) == set(result.to_pandas().columns) + + +def test_to_table_handles_a_run_with_no_scores() -> None: + assert _result().to_table().num_rows == 0 + + +def test_summary_header_reports_the_units_a_run_actually_has() -> None: + header = _run_with_failures().format_summary().splitlines()[0] + + assert header.startswith("AgentEvalResult(") + assert "scores=3" in header + assert "completed=1" in header + assert "failed=2" in header + + +def test_summary_header_lists_only_the_statuses_the_run_produced() -> None: + # Matches the dataset path's header, which leaves out statuses with nothing behind them. + header = _passing_run().format_summary().splitlines()[0] + + assert "completed=3" in header + assert "failed" not in header + assert "partial" not in header + + +def test_summary_separates_a_failed_trial_from_a_failed_metric() -> None: + # Both surface as FAILED but mean different things: a failed trial is an attempt the agent is + # answerable for, a failed metric is a measurement that never happened. Each label is asserted + # against the score it describes -- checking only that both strings appear would still pass if + # the two were swapped. + summary = _run_with_failures().format_summary() + + assert "[t2 / b / reward] failed trial\nagent crashed" in summary + assert "[t3 / a / reward] failed metric\njudge timed out" in summary + + +def test_summary_has_no_error_section_when_nothing_failed() -> None: + assert "Error details" not in _passing_run().format_summary() + + +def test_summary_preview_is_capped_and_says_what_it_is_showing() -> None: + result = _result(*(_score(f"t{index}", "a", 1.0) for index in range(8))) + + summary = result.format_summary(max_rows=3) + + assert "Score preview (first 3 of 8)" in summary + assert "t7" not in summary + + +def test_error_section_reports_how_many_failures_it_omitted() -> None: + result = _result( + *( + _score(f"t{index}", "a", status=AgentEvalScoreStatus.FAILED, diagnostics=(_METRIC_FAILURE,)) + for index in range(5) + ) + ) + + summary = result.format_summary(max_error_rows=2) + + assert "Error details (2 of 5 failed scores)" in summary + assert "3 more failed scores omitted" in summary + + +def test_max_error_rows_follows_max_rows_when_not_given() -> None: + # The documented default. Without this, tightening the preview would silently leave the error + # section at its own limit, and the two sections would disagree about how much they are showing. + result = _result( + *( + _score(f"t{index}", "a", status=AgentEvalScoreStatus.FAILED, diagnostics=(_METRIC_FAILURE,)) + for index in range(3) + ) + ) + + assert "Error details (1 of 3 failed scores)" in result.format_summary(max_rows=1) + + +def test_str_renders_the_compact_summary_capped_at_five_rows() -> None: + # Matches EvaluationResult.__str__, which previews five rows. + result = _result(*(_score(f"t{index}", "a", 1.0) for index in range(8))) + + rendered = str(result) + + assert rendered.startswith("AgentEvalResult(") + assert "Score preview (first 5 of 8)" in rendered + + +def test_print_summary_writes_format_summary_to_stdout(capsys: pytest.CaptureFixture[str]) -> None: + result = _passing_run() + + result.print_summary() + + assert capsys.readouterr().out == f"{result.format_summary()}\n" From 0eceacbe9312249b7e1421084c78a954115d439c Mon Sep 17 00:00:00 2001 From: Sandy Chapman Date: Mon, 10 Aug 2026 13:02:32 -0300 Subject: [PATCH 2/3] chore(evaluator): sync the vendored SDK with the new display surface `nemo_evaluator_sdk` is vendored into `sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/`, so a change to its source needs `make vendor` re-run and the result committed. Without it `lint-sdk-vendored` fails, and `lint-cli` fails after it because the vendor step leaves `sdk/python/` dirty and lint-cli diffs that same path. Signed-off-by: Sandy Chapman --- .../beta/evaluator/agent_eval/results.py | 248 +++++++++++++++++- 1 file changed, 245 insertions(+), 3 deletions(-) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py index cddf78e2ef..8bd35db38e 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py @@ -5,19 +5,35 @@ from __future__ import annotations +import json import math from collections.abc import Sequence from datetime import datetime from pathlib import Path - -from nemo_platform.beta.evaluator.agent_eval.scores import AgentEvalScoreStatus, AgentEvalTaskScore, is_trial_failure +from typing import Any + +from nemo_platform.beta.evaluator.agent_eval.scores import ( + AgentEvalDiagnosticSeverity, + AgentEvalScoreStatus, + AgentEvalTaskScore, + is_trial_failure, +) from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalTask, SemanticReducer, ViewSignal from nemo_platform.beta.evaluator.agent_eval.trials import AgentEvalTrial, RunnerInfo from nemo_platform.beta.evaluator.metrics.aggregation import compute_percentiles from nemo_platform.beta.evaluator.metrics.protocol import MetricOutput from nemo_platform.beta.evaluator.metrics.utils import metric_type_name from nemo_platform.beta.evaluator.values.protocol import BooleanValue, ContinuousScore -from nemo_platform.beta.evaluator.values.results import AggregatedMetricResult, AggregateRangeScore, AggregateScore +from nemo_platform.beta.evaluator.values.results import ( + AggregatedMetricResult, + AggregateRangeScore, + AggregateScore, + ResultView, + flatten_dict, + format_table, + serialize_value, + summary_aggregate_record, +) from pydantic import BaseModel, ConfigDict, Field #: Metric-output value schemas eligible for pass@k (a per-attempt "did it pass?" signal). Labels, @@ -179,6 +195,232 @@ def persist(self, output_dir: str | Path | None = None, *, write_dashboard: bool ) return persist_run(self, target, write_html_dashboard=write_dashboard) + def to_records(self, view: ResultView = "rows") -> list[dict[str, Any]]: + """Convert this run into flat dictionaries for export or inspection. + + ``view="rows"`` yields one record per metric score — the agent-eval analogue of the dataset + path's row. The fan-out is preserved rather than collapsed: ``task_id`` and ``trial_id`` are + columns, so a consumer can still group by task, which is what pass@k depends on. + + ``view="aggregate"`` matches the dataset path exactly — percentiles flattened, histograms + kept as JSON strings so the view stays tabular. + + Args: + view: Output projection, either ``"rows"`` or ``"aggregate"``. + + Returns: + Flat record dictionaries for downstream table/dataframe conversion. + + Raises: + ValueError: If ``view`` is unsupported. + """ + if view == "rows": + return [_score_record(score) for score in self.scores] + + if view == "aggregate": + records: list[dict[str, Any]] = [] + for score in self.summary.scores.scores: + record: dict[str, Any] = {} + for key, value in score.model_dump(mode="json").items(): + if key == "percentiles" and isinstance(value, dict): + flatten_dict("percentiles", value, record) + elif key == "histogram" and value is not None: + # Histograms stay as JSON strings so aggregate views remain tabular instead + # of expanding variable-width nested columns. + record[key] = json.dumps(value, sort_keys=True) + else: + record[key] = value + records.append(record) + return records + + raise ValueError(f"Unsupported view {view!r}. Expected 'rows' or 'aggregate'.") + + def to_table(self, view: ResultView = "rows"): + """Convert records into a ``pyarrow.Table``. + + Args: + view: Output projection, either ``"rows"`` or ``"aggregate"``. + + Columns are unioned across every record before the table is built. ``pa.Table.from_pylist`` + takes its schema from the first record alone, and in a row view ``error`` and + ``diagnostics.*`` appear only on failed scores — so a run whose first score succeeded would + otherwise export a table with the failure columns silently missing. ``to_pandas`` already + unions keys, and the two should not disagree about what a run contains. + + Args: + view: Output projection, either ``"rows"`` or ``"aggregate"``. + + Returns: + Table built from ``to_records(view=view)``. + """ + import pyarrow as pa + + records = self.to_records(view=view) + # dict-of-None preserves first-appearance order, matching how format_table derives columns. + columns = {key: None for record in records for key in record} + return pa.Table.from_pylist([{key: record.get(key) for key in columns} for record in records]) + + def to_pandas(self, view: ResultView = "rows"): + """Convert records into a pandas ``DataFrame``. + + Args: + view: Output projection, either ``"rows"`` or ``"aggregate"``. + + Returns: + DataFrame built from ``to_records(view=view)``. + """ + import pandas as pd + + return pd.DataFrame.from_records(self.to_records(view=view)) + + def format_summary(self, max_rows: int = 10, *, max_error_rows: int | None = None) -> str: + """Render a human-readable summary with aggregates and a score preview. + + Args: + max_rows: Maximum number of score records included in the preview. + max_error_rows: Maximum number of failed scores included in the error-details section. + Defaults to ``max_rows``. + + Returns: + Multi-line summary string suitable for terminal/notebook display. + """ + if max_error_rows is None: + max_error_rows = max_rows + aggregate_records = [summary_aggregate_record(score) for score in self.summary.scores.scores] + preview = [_score_preview_record(score) for score in self.scores[:max_rows]] + parts = [ + _agent_eval_summary_header(self), + "", + "Aggregate scores", + format_table(aggregate_records), + ] + if preview: + parts.extend( + [ + "", + f"Score preview (first {len(preview)} of {len(self.scores)})", + format_table(preview), + ] + ) + parts.extend(_format_score_errors(self.scores, max_error_rows=max_error_rows)) + return "\n".join(parts) + + def print_summary(self, max_rows: int = 10, *, max_error_rows: int | None = None) -> None: + """Print ``format_summary`` output. + + Args: + max_rows: Maximum number of score records included in the preview. + max_error_rows: Maximum number of failed scores included in the error-details section. + Defaults to ``max_rows``. + """ + print(self.format_summary(max_rows=max_rows, max_error_rows=max_error_rows)) + + def __str__(self) -> str: + """Return the default compact summary representation. + + Returns: + Summary string with up to five preview scores. + """ + return self.format_summary(max_rows=5) + + +def _score_error_text(score: AgentEvalTaskScore) -> str | None: + """Join the error-severity diagnostic messages for a score, or None when it has none.""" + messages = [ + diagnostic.message + for diagnostic in score.diagnostics + if diagnostic.severity is AgentEvalDiagnosticSeverity.ERROR + ] + return "; ".join(messages) if messages else None + + +def _score_diagnostics_columns(score: AgentEvalTaskScore) -> dict[str, str]: + """JSON-encoded diagnostic columns, keyed ``diagnostics.``. + + Encoded as compact JSON for the same reason the dataset path does it: diagnostics have a + metric-defined shape, and exports stay flat only if that shape is a string. + """ + if not score.diagnostics: + return {} + return { + f"diagnostics.{score.metric_type}": json.dumps( + [serialize_value(diagnostic) for diagnostic in score.diagnostics], sort_keys=True + ) + } + + +def _score_preview_record(score: AgentEvalTaskScore) -> dict[str, Any]: + """Identity and status columns shared by the row export and the summary preview.""" + record: dict[str, Any] = { + "task_id": score.task_id, + "trial_id": score.trial_id, + "metric_type": score.metric_type, + "status": score.status.value, + } + for output in score.outputs: + record[f"output.{output.name}"] = serialize_value(output.value) + return record + + +def _score_record(score: AgentEvalTaskScore) -> dict[str, Any]: + """Full export record for one score: preview columns plus error text and diagnostics.""" + record = _score_preview_record(score) + if error_text := _score_error_text(score): + record["error"] = error_text + record.update(_score_diagnostics_columns(score)) + return record + + +def _agent_eval_summary_header(result: AgentEvalResult) -> str: + """Build the header line, mirroring the shape :func:`summary_header` produces for row results. + + The counts differ because the units do — a run has tasks, trials, and scores where the dataset + path has rows — but the ``Name(field=value, ...)`` shape is the same, and a status the run never + produced is left out, matching what that header does with its zero counts. + + Statuses are counted by tallying the scores present, so an absent status simply never becomes a + key; there is no zero to filter out. + """ + status_counts: dict[str, int] = {} + for score in result.scores: + status_counts[score.status.value] = status_counts.get(score.status.value, 0) + 1 + fields = [ + f"tasks={len(result.tasks)}", + f"trials={len(result.trials)}", + f"scores={len(result.scores)}", + f"aggregate_scores={len(result.summary.scores.scores)}", + ] + fields.extend(f"{status}={count}" for status, count in sorted(status_counts.items())) + return f"AgentEvalResult({', '.join(fields)})" + + +def _format_score_errors( + scores: Sequence[AgentEvalTaskScore], + *, + max_error_rows: int | None, +) -> list[str]: + """Render the failed-score detail section, separating a failed trial from a failed metric. + + Both arrive as ``FAILED``, but they mean different things to a reader: a failed trial is an + attempt the agent is answerable for, a failed metric is a measurement that never happened. The + dataset path has no equivalent distinction to make, so this section is agent-eval's own rather + than a reuse of :func:`format_error_details`. + """ + failed = [score for score in scores if score.status is AgentEvalScoreStatus.FAILED] + if not failed: + return [] + + shown_limit = len(failed) if max_error_rows is None else max(0, max_error_rows) + shown = failed[:shown_limit] + parts = ["", f"Error details ({len(shown)} of {len(failed)} failed scores)"] + for score in shown: + kind = "failed trial" if is_trial_failure(score) else "failed metric" + parts.extend(["", f"[{score.task_id} / {score.trial_id} / {score.metric_type}] {kind}"]) + parts.append(_score_error_text(score) or "(no error-severity diagnostic recorded)") + if len(shown) < len(failed): + parts.extend(["", f"... {len(failed) - len(shown)} more failed scores omitted"]) + return parts + def _aggregate_scores( scores: Sequence[AgentEvalTaskScore], From e7d30d0495c1bfd9b09f1a1cc3109f837cb19cad Mon Sep 17 00:00:00 2001 From: Sandy Chapman Date: Mon, 10 Aug 2026 14:24:28 -0300 Subject: [PATCH 3/3] fix(evaluator): keep score identity and metadata in agent-eval row exports Review on #1201: `_score_record` dropped `AgentEvalTaskScore.id`, `run_id`, and `metadata`, so every row, table, and DataFrame export lost them. An export is the thing a caller joins, concatenates, and keeps. `id` is what a row is addressable by, `run_id` keeps a frame self-describing once several runs are stacked into one, and `metadata` is caller-supplied -- dropping it silently discarded data the SDK never owned. Metadata is flattened into dotted columns rather than JSON-encoded: it is free-form but usually shallow and scalar, so flattening keeps it queryable. Diagnostics keep the JSON treatment, because their shape is metric-defined and variable-width. The summary preview deliberately stays narrow -- it is read on a terminal -- which is the same split the dataset path makes between `to_records` and `summary_row_base_record`. Covered by a test so the two don't drift together. Also documents why the error-row limit is clamped with max(0, ...): slicing would read a negative limit as an offset from the end, so `failed[:-2]` would show all but the last two rather than none. An over-large limit needs no guard. Vendored SDK re-synced. Signed-off-by: Sandy Chapman --- .../nemo_evaluator_sdk/agent_eval/results.py | 20 ++++++- .../tests/agent_eval/test_result_display.py | 53 +++++++++++++++++++ .../beta/evaluator/agent_eval/results.py | 20 ++++++- 3 files changed, 89 insertions(+), 4 deletions(-) diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py index 5fd447ac1d..6fd8f2d811 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py @@ -363,11 +363,24 @@ def _score_preview_record(score: AgentEvalTaskScore) -> dict[str, Any]: def _score_record(score: AgentEvalTaskScore) -> dict[str, Any]: - """Full export record for one score: preview columns plus error text and diagnostics.""" - record = _score_preview_record(score) + """Full export record for one score: identity, preview columns, error text, and diagnostics. + + Carries ``id``, ``run_id``, and ``metadata`` that the summary preview leaves out. An export is + the thing a caller joins, concatenates, and keeps: ``id`` is what a row is addressable by, + ``run_id`` keeps a frame self-describing once several runs are stacked into one, and + ``metadata`` is caller-supplied — dropping it silently discards data the SDK never owned. The + preview stays narrow because it is read on a terminal, the same split the dataset path makes + between ``to_records`` and ``summary_row_base_record``. + """ + record: dict[str, Any] = {"id": score.id, "run_id": score.run_id} + record.update(_score_preview_record(score)) if error_text := _score_error_text(score): record["error"] = error_text record.update(_score_diagnostics_columns(score)) + # Flattened rather than JSON-encoded: metadata is free-form but usually shallow and scalar, so + # dotted columns keep it queryable. Diagnostics get the JSON treatment instead because their + # shape is metric-defined and variable-width. + flatten_dict("metadata", serialize_value(score.metadata), record) return record @@ -410,6 +423,9 @@ def _format_score_errors( if not failed: return [] + # max(0, ...) guards a negative limit, which slicing would otherwise read as an offset from the + # end: failed[:-2] shows all but the last two rather than none. An over-large limit needs no + # guard, since a slice past the end is simply the whole list. Mirrors format_error_details. shown_limit = len(failed) if max_error_rows is None else max(0, max_error_rows) shown = failed[:shown_limit] parts = ["", f"Error details ({len(shown)} of {len(failed)} failed scores)"] diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_result_display.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_result_display.py index 79d09e88a7..f866620d55 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_result_display.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_result_display.py @@ -81,6 +81,59 @@ def test_row_records_keep_the_task_grouping_that_pass_at_k_depends_on() -> None: assert records[0]["output.reward"] == 1.0 +def test_row_records_carry_score_identity_for_joining_and_stacking() -> None: + # An export is joined, concatenated, and kept: `id` addresses a row, and `run_id` keeps a frame + # self-describing once several runs are stacked together. + record = _passing_run().to_records()[0] + + assert record["id"] == "t1-a" + assert record["run_id"] == "run-1" + + +def test_row_records_flatten_caller_supplied_metadata() -> None: + # Metadata is the caller's, not the SDK's -- dropping it from the export discards data nothing + # else records. Flattened into dotted columns so the export stays tabular. + scored = _score("t1", "a", 1.0) + scored.metadata = {"backend": "gym", "attempt": {"index": 2}} + + record = _result(scored).to_records()[0] + + assert record["metadata.backend"] == "gym" + assert record["metadata.attempt.index"] == 2 + + +def test_summary_preview_stays_narrow_and_omits_export_only_columns() -> None: + # The preview is read on a terminal; identity and metadata columns belong in the export, which + # is the same split the dataset path makes between to_records and summary_row_base_record. + summary = _passing_run().format_summary() + + assert "run_id" not in summary + assert "metadata" not in summary + + +def test_summary_tolerates_max_rows_larger_than_the_run() -> None: + # A slice past the end is the whole list, so an over-large preview limit is not an error. + summary = _passing_run().format_summary(max_rows=1000) + + assert "Score preview (first 3 of 3)" in summary + + +def test_a_negative_error_limit_shows_no_failures_rather_than_all_but_some() -> None: + # Slicing would read a negative limit as an offset from the end -- failed[:-1] would show all + # but the last failure, which is not what "show at most -1" can sensibly mean. + result = _result( + *( + _score(f"t{index}", "a", status=AgentEvalScoreStatus.FAILED, diagnostics=(_METRIC_FAILURE,)) + for index in range(3) + ) + ) + + summary = result.format_summary(max_error_rows=-1) + + assert "Error details (0 of 3 failed scores)" in summary + assert "3 more failed scores omitted" in summary + + def test_row_records_carry_error_text_and_diagnostics_only_for_failures() -> None: records = _run_with_failures().to_records() diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py index 8bd35db38e..b97acdd823 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py @@ -363,11 +363,24 @@ def _score_preview_record(score: AgentEvalTaskScore) -> dict[str, Any]: def _score_record(score: AgentEvalTaskScore) -> dict[str, Any]: - """Full export record for one score: preview columns plus error text and diagnostics.""" - record = _score_preview_record(score) + """Full export record for one score: identity, preview columns, error text, and diagnostics. + + Carries ``id``, ``run_id``, and ``metadata`` that the summary preview leaves out. An export is + the thing a caller joins, concatenates, and keeps: ``id`` is what a row is addressable by, + ``run_id`` keeps a frame self-describing once several runs are stacked into one, and + ``metadata`` is caller-supplied — dropping it silently discards data the SDK never owned. The + preview stays narrow because it is read on a terminal, the same split the dataset path makes + between ``to_records`` and ``summary_row_base_record``. + """ + record: dict[str, Any] = {"id": score.id, "run_id": score.run_id} + record.update(_score_preview_record(score)) if error_text := _score_error_text(score): record["error"] = error_text record.update(_score_diagnostics_columns(score)) + # Flattened rather than JSON-encoded: metadata is free-form but usually shallow and scalar, so + # dotted columns keep it queryable. Diagnostics get the JSON treatment instead because their + # shape is metric-defined and variable-width. + flatten_dict("metadata", serialize_value(score.metadata), record) return record @@ -410,6 +423,9 @@ def _format_score_errors( if not failed: return [] + # max(0, ...) guards a negative limit, which slicing would otherwise read as an offset from the + # end: failed[:-2] shows all but the last two rather than none. An over-large limit needs no + # guard, since a slice past the end is simply the whole list. Mirrors format_error_details. shown_limit = len(failed) if max_error_rows is None else max(0, max_error_rows) shown = failed[:shown_limit] parts = ["", f"Error details ({len(shown)} of {len(failed)} failed scores)"]