diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py index b31d1b5078..6fd8f2d811 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py @@ -5,19 +5,35 @@ from __future__ import annotations +import json import math from collections.abc import Sequence from datetime import datetime from pathlib import Path - -from nemo_evaluator_sdk.agent_eval.scores import AgentEvalScoreStatus, AgentEvalTaskScore, is_trial_failure +from typing import Any + +from nemo_evaluator_sdk.agent_eval.scores import ( + AgentEvalDiagnosticSeverity, + AgentEvalScoreStatus, + AgentEvalTaskScore, + is_trial_failure, +) from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalTask, SemanticReducer, ViewSignal from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial, RunnerInfo from nemo_evaluator_sdk.metrics.aggregation import compute_percentiles from nemo_evaluator_sdk.metrics.protocol import MetricOutput from nemo_evaluator_sdk.metrics.utils import metric_type_name from nemo_evaluator_sdk.values.protocol import BooleanValue, ContinuousScore -from nemo_evaluator_sdk.values.results import AggregatedMetricResult, AggregateRangeScore, AggregateScore +from nemo_evaluator_sdk.values.results import ( + AggregatedMetricResult, + AggregateRangeScore, + AggregateScore, + ResultView, + flatten_dict, + format_table, + serialize_value, + summary_aggregate_record, +) from pydantic import BaseModel, ConfigDict, Field #: Metric-output value schemas eligible for pass@k (a per-attempt "did it pass?" signal). Labels, @@ -179,6 +195,248 @@ def persist(self, output_dir: str | Path | None = None, *, write_dashboard: bool ) return persist_run(self, target, write_html_dashboard=write_dashboard) + def to_records(self, view: ResultView = "rows") -> list[dict[str, Any]]: + """Convert this run into flat dictionaries for export or inspection. + + ``view="rows"`` yields one record per metric score — the agent-eval analogue of the dataset + path's row. The fan-out is preserved rather than collapsed: ``task_id`` and ``trial_id`` are + columns, so a consumer can still group by task, which is what pass@k depends on. + + ``view="aggregate"`` matches the dataset path exactly — percentiles flattened, histograms + kept as JSON strings so the view stays tabular. + + Args: + view: Output projection, either ``"rows"`` or ``"aggregate"``. + + Returns: + Flat record dictionaries for downstream table/dataframe conversion. + + Raises: + ValueError: If ``view`` is unsupported. + """ + if view == "rows": + return [_score_record(score) for score in self.scores] + + if view == "aggregate": + records: list[dict[str, Any]] = [] + for score in self.summary.scores.scores: + record: dict[str, Any] = {} + for key, value in score.model_dump(mode="json").items(): + if key == "percentiles" and isinstance(value, dict): + flatten_dict("percentiles", value, record) + elif key == "histogram" and value is not None: + # Histograms stay as JSON strings so aggregate views remain tabular instead + # of expanding variable-width nested columns. + record[key] = json.dumps(value, sort_keys=True) + else: + record[key] = value + records.append(record) + return records + + raise ValueError(f"Unsupported view {view!r}. Expected 'rows' or 'aggregate'.") + + def to_table(self, view: ResultView = "rows"): + """Convert records into a ``pyarrow.Table``. + + Args: + view: Output projection, either ``"rows"`` or ``"aggregate"``. + + Columns are unioned across every record before the table is built. ``pa.Table.from_pylist`` + takes its schema from the first record alone, and in a row view ``error`` and + ``diagnostics.*`` appear only on failed scores — so a run whose first score succeeded would + otherwise export a table with the failure columns silently missing. ``to_pandas`` already + unions keys, and the two should not disagree about what a run contains. + + Args: + view: Output projection, either ``"rows"`` or ``"aggregate"``. + + Returns: + Table built from ``to_records(view=view)``. + """ + import pyarrow as pa + + records = self.to_records(view=view) + # dict-of-None preserves first-appearance order, matching how format_table derives columns. + columns = {key: None for record in records for key in record} + return pa.Table.from_pylist([{key: record.get(key) for key in columns} for record in records]) + + def to_pandas(self, view: ResultView = "rows"): + """Convert records into a pandas ``DataFrame``. + + Args: + view: Output projection, either ``"rows"`` or ``"aggregate"``. + + Returns: + DataFrame built from ``to_records(view=view)``. + """ + import pandas as pd + + return pd.DataFrame.from_records(self.to_records(view=view)) + + def format_summary(self, max_rows: int = 10, *, max_error_rows: int | None = None) -> str: + """Render a human-readable summary with aggregates and a score preview. + + Args: + max_rows: Maximum number of score records included in the preview. + max_error_rows: Maximum number of failed scores included in the error-details section. + Defaults to ``max_rows``. + + Returns: + Multi-line summary string suitable for terminal/notebook display. + """ + if max_error_rows is None: + max_error_rows = max_rows + aggregate_records = [summary_aggregate_record(score) for score in self.summary.scores.scores] + preview = [_score_preview_record(score) for score in self.scores[:max_rows]] + parts = [ + _agent_eval_summary_header(self), + "", + "Aggregate scores", + format_table(aggregate_records), + ] + if preview: + parts.extend( + [ + "", + f"Score preview (first {len(preview)} of {len(self.scores)})", + format_table(preview), + ] + ) + parts.extend(_format_score_errors(self.scores, max_error_rows=max_error_rows)) + return "\n".join(parts) + + def print_summary(self, max_rows: int = 10, *, max_error_rows: int | None = None) -> None: + """Print ``format_summary`` output. + + Args: + max_rows: Maximum number of score records included in the preview. + max_error_rows: Maximum number of failed scores included in the error-details section. + Defaults to ``max_rows``. + """ + print(self.format_summary(max_rows=max_rows, max_error_rows=max_error_rows)) + + def __str__(self) -> str: + """Return the default compact summary representation. + + Returns: + Summary string with up to five preview scores. + """ + return self.format_summary(max_rows=5) + + +def _score_error_text(score: AgentEvalTaskScore) -> str | None: + """Join the error-severity diagnostic messages for a score, or None when it has none.""" + messages = [ + diagnostic.message + for diagnostic in score.diagnostics + if diagnostic.severity is AgentEvalDiagnosticSeverity.ERROR + ] + return "; ".join(messages) if messages else None + + +def _score_diagnostics_columns(score: AgentEvalTaskScore) -> dict[str, str]: + """JSON-encoded diagnostic columns, keyed ``diagnostics.``. + + Encoded as compact JSON for the same reason the dataset path does it: diagnostics have a + metric-defined shape, and exports stay flat only if that shape is a string. + """ + if not score.diagnostics: + return {} + return { + f"diagnostics.{score.metric_type}": json.dumps( + [serialize_value(diagnostic) for diagnostic in score.diagnostics], sort_keys=True + ) + } + + +def _score_preview_record(score: AgentEvalTaskScore) -> dict[str, Any]: + """Identity and status columns shared by the row export and the summary preview.""" + record: dict[str, Any] = { + "task_id": score.task_id, + "trial_id": score.trial_id, + "metric_type": score.metric_type, + "status": score.status.value, + } + for output in score.outputs: + record[f"output.{output.name}"] = serialize_value(output.value) + return record + + +def _score_record(score: AgentEvalTaskScore) -> dict[str, Any]: + """Full export record for one score: identity, preview columns, error text, and diagnostics. + + Carries ``id``, ``run_id``, and ``metadata`` that the summary preview leaves out. An export is + the thing a caller joins, concatenates, and keeps: ``id`` is what a row is addressable by, + ``run_id`` keeps a frame self-describing once several runs are stacked into one, and + ``metadata`` is caller-supplied — dropping it silently discards data the SDK never owned. The + preview stays narrow because it is read on a terminal, the same split the dataset path makes + between ``to_records`` and ``summary_row_base_record``. + """ + record: dict[str, Any] = {"id": score.id, "run_id": score.run_id} + record.update(_score_preview_record(score)) + if error_text := _score_error_text(score): + record["error"] = error_text + record.update(_score_diagnostics_columns(score)) + # Flattened rather than JSON-encoded: metadata is free-form but usually shallow and scalar, so + # dotted columns keep it queryable. Diagnostics get the JSON treatment instead because their + # shape is metric-defined and variable-width. + flatten_dict("metadata", serialize_value(score.metadata), record) + return record + + +def _agent_eval_summary_header(result: AgentEvalResult) -> str: + """Build the header line, mirroring the shape :func:`summary_header` produces for row results. + + The counts differ because the units do — a run has tasks, trials, and scores where the dataset + path has rows — but the ``Name(field=value, ...)`` shape is the same, and a status the run never + produced is left out, matching what that header does with its zero counts. + + Statuses are counted by tallying the scores present, so an absent status simply never becomes a + key; there is no zero to filter out. + """ + status_counts: dict[str, int] = {} + for score in result.scores: + status_counts[score.status.value] = status_counts.get(score.status.value, 0) + 1 + fields = [ + f"tasks={len(result.tasks)}", + f"trials={len(result.trials)}", + f"scores={len(result.scores)}", + f"aggregate_scores={len(result.summary.scores.scores)}", + ] + fields.extend(f"{status}={count}" for status, count in sorted(status_counts.items())) + return f"AgentEvalResult({', '.join(fields)})" + + +def _format_score_errors( + scores: Sequence[AgentEvalTaskScore], + *, + max_error_rows: int | None, +) -> list[str]: + """Render the failed-score detail section, separating a failed trial from a failed metric. + + Both arrive as ``FAILED``, but they mean different things to a reader: a failed trial is an + attempt the agent is answerable for, a failed metric is a measurement that never happened. The + dataset path has no equivalent distinction to make, so this section is agent-eval's own rather + than a reuse of :func:`format_error_details`. + """ + failed = [score for score in scores if score.status is AgentEvalScoreStatus.FAILED] + if not failed: + return [] + + # max(0, ...) guards a negative limit, which slicing would otherwise read as an offset from the + # end: failed[:-2] shows all but the last two rather than none. An over-large limit needs no + # guard, since a slice past the end is simply the whole list. Mirrors format_error_details. + shown_limit = len(failed) if max_error_rows is None else max(0, max_error_rows) + shown = failed[:shown_limit] + parts = ["", f"Error details ({len(shown)} of {len(failed)} failed scores)"] + for score in shown: + kind = "failed trial" if is_trial_failure(score) else "failed metric" + parts.extend(["", f"[{score.task_id} / {score.trial_id} / {score.metric_type}] {kind}"]) + parts.append(_score_error_text(score) or "(no error-severity diagnostic recorded)") + if len(shown) < len(failed): + parts.extend(["", f"... {len(failed) - len(shown)} more failed scores omitted"]) + return parts + def _aggregate_scores( scores: Sequence[AgentEvalTaskScore], diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_result_display.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_result_display.py new file mode 100644 index 0000000000..f866620d55 --- /dev/null +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_result_display.py @@ -0,0 +1,262 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Display and export surface on AgentEvalResult, mirroring the dataset path's.""" + +from __future__ import annotations + +import pytest +from nemo_evaluator_sdk.agent_eval.results import AgentEvalResult, AgentEvalSummary +from nemo_evaluator_sdk.agent_eval.scores import ( + TRIAL_STATUS_DETAIL, + AgentEvalDiagnostic, + AgentEvalDiagnosticSeverity, + AgentEvalScoreStatus, + AgentEvalTaskScore, +) +from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrialStatus +from nemo_evaluator_sdk.values.protocol import MetricOutput + +_TRIAL_FAILURE = AgentEvalDiagnostic( + severity=AgentEvalDiagnosticSeverity.ERROR, + message="agent crashed", + details={TRIAL_STATUS_DETAIL: AgentEvalTrialStatus.FAILED.value}, +) +_METRIC_FAILURE = AgentEvalDiagnostic( + severity=AgentEvalDiagnosticSeverity.ERROR, + message="judge timed out", + details={"exception_type": "TimeoutError"}, +) + + +def _score( + task_id: str, + trial_id: str, + value: float | None = None, + *, + status: AgentEvalScoreStatus = AgentEvalScoreStatus.COMPLETED, + diagnostics: tuple[AgentEvalDiagnostic, ...] = (), +) -> AgentEvalTaskScore: + return AgentEvalTaskScore( + id=f"{task_id}-{trial_id}", + run_id="run-1", + task_id=task_id, + trial_id=trial_id, + metric_type="reward", + status=status, + outputs=[MetricOutput(name="reward", value=value)] if value is not None else [], + diagnostics=list(diagnostics), + ) + + +def _result(*scores: AgentEvalTaskScore) -> AgentEvalResult: + return AgentEvalResult( + run_id="run-1", + tasks=[], + trials=[], + scores=list(scores), + summary=AgentEvalSummary.from_scores(scores), + ) + + +def _passing_run() -> AgentEvalResult: + return _result(_score("t1", "a", 1.0), _score("t1", "b", 0.0), _score("t2", "a", 1.0)) + + +def _run_with_failures() -> AgentEvalResult: + return _result( + _score("t1", "a", 1.0), + _score("t2", "b", status=AgentEvalScoreStatus.FAILED, diagnostics=(_TRIAL_FAILURE,)), + _score("t3", "a", status=AgentEvalScoreStatus.FAILED, diagnostics=(_METRIC_FAILURE,)), + ) + + +def test_row_records_keep_the_task_grouping_that_pass_at_k_depends_on() -> None: + # The fan-out is load-bearing: flattening trials into anonymous rows would make it impossible to + # regroup by task, so task_id and trial_id have to survive as columns. + records = _passing_run().to_records() + + assert [record["task_id"] for record in records] == ["t1", "t1", "t2"] + assert [record["trial_id"] for record in records] == ["a", "b", "a"] + assert records[0]["output.reward"] == 1.0 + + +def test_row_records_carry_score_identity_for_joining_and_stacking() -> None: + # An export is joined, concatenated, and kept: `id` addresses a row, and `run_id` keeps a frame + # self-describing once several runs are stacked together. + record = _passing_run().to_records()[0] + + assert record["id"] == "t1-a" + assert record["run_id"] == "run-1" + + +def test_row_records_flatten_caller_supplied_metadata() -> None: + # Metadata is the caller's, not the SDK's -- dropping it from the export discards data nothing + # else records. Flattened into dotted columns so the export stays tabular. + scored = _score("t1", "a", 1.0) + scored.metadata = {"backend": "gym", "attempt": {"index": 2}} + + record = _result(scored).to_records()[0] + + assert record["metadata.backend"] == "gym" + assert record["metadata.attempt.index"] == 2 + + +def test_summary_preview_stays_narrow_and_omits_export_only_columns() -> None: + # The preview is read on a terminal; identity and metadata columns belong in the export, which + # is the same split the dataset path makes between to_records and summary_row_base_record. + summary = _passing_run().format_summary() + + assert "run_id" not in summary + assert "metadata" not in summary + + +def test_summary_tolerates_max_rows_larger_than_the_run() -> None: + # A slice past the end is the whole list, so an over-large preview limit is not an error. + summary = _passing_run().format_summary(max_rows=1000) + + assert "Score preview (first 3 of 3)" in summary + + +def test_a_negative_error_limit_shows_no_failures_rather_than_all_but_some() -> None: + # Slicing would read a negative limit as an offset from the end -- failed[:-1] would show all + # but the last failure, which is not what "show at most -1" can sensibly mean. + result = _result( + *( + _score(f"t{index}", "a", status=AgentEvalScoreStatus.FAILED, diagnostics=(_METRIC_FAILURE,)) + for index in range(3) + ) + ) + + summary = result.format_summary(max_error_rows=-1) + + assert "Error details (0 of 3 failed scores)" in summary + assert "3 more failed scores omitted" in summary + + +def test_row_records_carry_error_text_and_diagnostics_only_for_failures() -> None: + records = _run_with_failures().to_records() + + assert "error" not in records[0] + assert records[1]["error"] == "agent crashed" + assert "diagnostics.reward" in records[1] + + +def test_aggregate_view_flattens_percentiles_like_the_dataset_path() -> None: + records = _passing_run().to_records(view="aggregate") + + assert records[0]["name"] == "reward.reward" + # Percentiles are flattened into columns rather than left nested, so the view stays tabular. + assert "percentiles.p50" in records[0] + assert "percentiles" not in records[0] + + +def test_unsupported_view_is_rejected_with_the_same_message_as_the_dataset_path() -> None: + with pytest.raises(ValueError, match=r"Unsupported view 'nope'. Expected 'rows' or 'aggregate'."): + _passing_run().to_records(view="nope") # ty: ignore[invalid-argument-type] + + +def test_to_table_keeps_columns_that_only_appear_on_later_records() -> None: + # pa.Table.from_pylist takes its schema from the first record. Error and diagnostics columns + # appear only on failures, so a run whose first score passed would silently export without them. + table = _run_with_failures().to_table() + + assert "error" in table.column_names + assert "diagnostics.reward" in table.column_names + + +def test_to_table_and_to_pandas_agree_on_columns() -> None: + result = _run_with_failures() + + assert set(result.to_table().column_names) == set(result.to_pandas().columns) + + +def test_to_table_handles_a_run_with_no_scores() -> None: + assert _result().to_table().num_rows == 0 + + +def test_summary_header_reports_the_units_a_run_actually_has() -> None: + header = _run_with_failures().format_summary().splitlines()[0] + + assert header.startswith("AgentEvalResult(") + assert "scores=3" in header + assert "completed=1" in header + assert "failed=2" in header + + +def test_summary_header_lists_only_the_statuses_the_run_produced() -> None: + # Matches the dataset path's header, which leaves out statuses with nothing behind them. + header = _passing_run().format_summary().splitlines()[0] + + assert "completed=3" in header + assert "failed" not in header + assert "partial" not in header + + +def test_summary_separates_a_failed_trial_from_a_failed_metric() -> None: + # Both surface as FAILED but mean different things: a failed trial is an attempt the agent is + # answerable for, a failed metric is a measurement that never happened. Each label is asserted + # against the score it describes -- checking only that both strings appear would still pass if + # the two were swapped. + summary = _run_with_failures().format_summary() + + assert "[t2 / b / reward] failed trial\nagent crashed" in summary + assert "[t3 / a / reward] failed metric\njudge timed out" in summary + + +def test_summary_has_no_error_section_when_nothing_failed() -> None: + assert "Error details" not in _passing_run().format_summary() + + +def test_summary_preview_is_capped_and_says_what_it_is_showing() -> None: + result = _result(*(_score(f"t{index}", "a", 1.0) for index in range(8))) + + summary = result.format_summary(max_rows=3) + + assert "Score preview (first 3 of 8)" in summary + assert "t7" not in summary + + +def test_error_section_reports_how_many_failures_it_omitted() -> None: + result = _result( + *( + _score(f"t{index}", "a", status=AgentEvalScoreStatus.FAILED, diagnostics=(_METRIC_FAILURE,)) + for index in range(5) + ) + ) + + summary = result.format_summary(max_error_rows=2) + + assert "Error details (2 of 5 failed scores)" in summary + assert "3 more failed scores omitted" in summary + + +def test_max_error_rows_follows_max_rows_when_not_given() -> None: + # The documented default. Without this, tightening the preview would silently leave the error + # section at its own limit, and the two sections would disagree about how much they are showing. + result = _result( + *( + _score(f"t{index}", "a", status=AgentEvalScoreStatus.FAILED, diagnostics=(_METRIC_FAILURE,)) + for index in range(3) + ) + ) + + assert "Error details (1 of 3 failed scores)" in result.format_summary(max_rows=1) + + +def test_str_renders_the_compact_summary_capped_at_five_rows() -> None: + # Matches EvaluationResult.__str__, which previews five rows. + result = _result(*(_score(f"t{index}", "a", 1.0) for index in range(8))) + + rendered = str(result) + + assert rendered.startswith("AgentEvalResult(") + assert "Score preview (first 5 of 8)" in rendered + + +def test_print_summary_writes_format_summary_to_stdout(capsys: pytest.CaptureFixture[str]) -> None: + result = _passing_run() + + result.print_summary() + + assert capsys.readouterr().out == f"{result.format_summary()}\n" diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py index cddf78e2ef..b97acdd823 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py @@ -5,19 +5,35 @@ from __future__ import annotations +import json import math from collections.abc import Sequence from datetime import datetime from pathlib import Path - -from nemo_platform.beta.evaluator.agent_eval.scores import AgentEvalScoreStatus, AgentEvalTaskScore, is_trial_failure +from typing import Any + +from nemo_platform.beta.evaluator.agent_eval.scores import ( + AgentEvalDiagnosticSeverity, + AgentEvalScoreStatus, + AgentEvalTaskScore, + is_trial_failure, +) from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalTask, SemanticReducer, ViewSignal from nemo_platform.beta.evaluator.agent_eval.trials import AgentEvalTrial, RunnerInfo from nemo_platform.beta.evaluator.metrics.aggregation import compute_percentiles from nemo_platform.beta.evaluator.metrics.protocol import MetricOutput from nemo_platform.beta.evaluator.metrics.utils import metric_type_name from nemo_platform.beta.evaluator.values.protocol import BooleanValue, ContinuousScore -from nemo_platform.beta.evaluator.values.results import AggregatedMetricResult, AggregateRangeScore, AggregateScore +from nemo_platform.beta.evaluator.values.results import ( + AggregatedMetricResult, + AggregateRangeScore, + AggregateScore, + ResultView, + flatten_dict, + format_table, + serialize_value, + summary_aggregate_record, +) from pydantic import BaseModel, ConfigDict, Field #: Metric-output value schemas eligible for pass@k (a per-attempt "did it pass?" signal). Labels, @@ -179,6 +195,248 @@ def persist(self, output_dir: str | Path | None = None, *, write_dashboard: bool ) return persist_run(self, target, write_html_dashboard=write_dashboard) + def to_records(self, view: ResultView = "rows") -> list[dict[str, Any]]: + """Convert this run into flat dictionaries for export or inspection. + + ``view="rows"`` yields one record per metric score — the agent-eval analogue of the dataset + path's row. The fan-out is preserved rather than collapsed: ``task_id`` and ``trial_id`` are + columns, so a consumer can still group by task, which is what pass@k depends on. + + ``view="aggregate"`` matches the dataset path exactly — percentiles flattened, histograms + kept as JSON strings so the view stays tabular. + + Args: + view: Output projection, either ``"rows"`` or ``"aggregate"``. + + Returns: + Flat record dictionaries for downstream table/dataframe conversion. + + Raises: + ValueError: If ``view`` is unsupported. + """ + if view == "rows": + return [_score_record(score) for score in self.scores] + + if view == "aggregate": + records: list[dict[str, Any]] = [] + for score in self.summary.scores.scores: + record: dict[str, Any] = {} + for key, value in score.model_dump(mode="json").items(): + if key == "percentiles" and isinstance(value, dict): + flatten_dict("percentiles", value, record) + elif key == "histogram" and value is not None: + # Histograms stay as JSON strings so aggregate views remain tabular instead + # of expanding variable-width nested columns. + record[key] = json.dumps(value, sort_keys=True) + else: + record[key] = value + records.append(record) + return records + + raise ValueError(f"Unsupported view {view!r}. Expected 'rows' or 'aggregate'.") + + def to_table(self, view: ResultView = "rows"): + """Convert records into a ``pyarrow.Table``. + + Args: + view: Output projection, either ``"rows"`` or ``"aggregate"``. + + Columns are unioned across every record before the table is built. ``pa.Table.from_pylist`` + takes its schema from the first record alone, and in a row view ``error`` and + ``diagnostics.*`` appear only on failed scores — so a run whose first score succeeded would + otherwise export a table with the failure columns silently missing. ``to_pandas`` already + unions keys, and the two should not disagree about what a run contains. + + Args: + view: Output projection, either ``"rows"`` or ``"aggregate"``. + + Returns: + Table built from ``to_records(view=view)``. + """ + import pyarrow as pa + + records = self.to_records(view=view) + # dict-of-None preserves first-appearance order, matching how format_table derives columns. + columns = {key: None for record in records for key in record} + return pa.Table.from_pylist([{key: record.get(key) for key in columns} for record in records]) + + def to_pandas(self, view: ResultView = "rows"): + """Convert records into a pandas ``DataFrame``. + + Args: + view: Output projection, either ``"rows"`` or ``"aggregate"``. + + Returns: + DataFrame built from ``to_records(view=view)``. + """ + import pandas as pd + + return pd.DataFrame.from_records(self.to_records(view=view)) + + def format_summary(self, max_rows: int = 10, *, max_error_rows: int | None = None) -> str: + """Render a human-readable summary with aggregates and a score preview. + + Args: + max_rows: Maximum number of score records included in the preview. + max_error_rows: Maximum number of failed scores included in the error-details section. + Defaults to ``max_rows``. + + Returns: + Multi-line summary string suitable for terminal/notebook display. + """ + if max_error_rows is None: + max_error_rows = max_rows + aggregate_records = [summary_aggregate_record(score) for score in self.summary.scores.scores] + preview = [_score_preview_record(score) for score in self.scores[:max_rows]] + parts = [ + _agent_eval_summary_header(self), + "", + "Aggregate scores", + format_table(aggregate_records), + ] + if preview: + parts.extend( + [ + "", + f"Score preview (first {len(preview)} of {len(self.scores)})", + format_table(preview), + ] + ) + parts.extend(_format_score_errors(self.scores, max_error_rows=max_error_rows)) + return "\n".join(parts) + + def print_summary(self, max_rows: int = 10, *, max_error_rows: int | None = None) -> None: + """Print ``format_summary`` output. + + Args: + max_rows: Maximum number of score records included in the preview. + max_error_rows: Maximum number of failed scores included in the error-details section. + Defaults to ``max_rows``. + """ + print(self.format_summary(max_rows=max_rows, max_error_rows=max_error_rows)) + + def __str__(self) -> str: + """Return the default compact summary representation. + + Returns: + Summary string with up to five preview scores. + """ + return self.format_summary(max_rows=5) + + +def _score_error_text(score: AgentEvalTaskScore) -> str | None: + """Join the error-severity diagnostic messages for a score, or None when it has none.""" + messages = [ + diagnostic.message + for diagnostic in score.diagnostics + if diagnostic.severity is AgentEvalDiagnosticSeverity.ERROR + ] + return "; ".join(messages) if messages else None + + +def _score_diagnostics_columns(score: AgentEvalTaskScore) -> dict[str, str]: + """JSON-encoded diagnostic columns, keyed ``diagnostics.``. + + Encoded as compact JSON for the same reason the dataset path does it: diagnostics have a + metric-defined shape, and exports stay flat only if that shape is a string. + """ + if not score.diagnostics: + return {} + return { + f"diagnostics.{score.metric_type}": json.dumps( + [serialize_value(diagnostic) for diagnostic in score.diagnostics], sort_keys=True + ) + } + + +def _score_preview_record(score: AgentEvalTaskScore) -> dict[str, Any]: + """Identity and status columns shared by the row export and the summary preview.""" + record: dict[str, Any] = { + "task_id": score.task_id, + "trial_id": score.trial_id, + "metric_type": score.metric_type, + "status": score.status.value, + } + for output in score.outputs: + record[f"output.{output.name}"] = serialize_value(output.value) + return record + + +def _score_record(score: AgentEvalTaskScore) -> dict[str, Any]: + """Full export record for one score: identity, preview columns, error text, and diagnostics. + + Carries ``id``, ``run_id``, and ``metadata`` that the summary preview leaves out. An export is + the thing a caller joins, concatenates, and keeps: ``id`` is what a row is addressable by, + ``run_id`` keeps a frame self-describing once several runs are stacked into one, and + ``metadata`` is caller-supplied — dropping it silently discards data the SDK never owned. The + preview stays narrow because it is read on a terminal, the same split the dataset path makes + between ``to_records`` and ``summary_row_base_record``. + """ + record: dict[str, Any] = {"id": score.id, "run_id": score.run_id} + record.update(_score_preview_record(score)) + if error_text := _score_error_text(score): + record["error"] = error_text + record.update(_score_diagnostics_columns(score)) + # Flattened rather than JSON-encoded: metadata is free-form but usually shallow and scalar, so + # dotted columns keep it queryable. Diagnostics get the JSON treatment instead because their + # shape is metric-defined and variable-width. + flatten_dict("metadata", serialize_value(score.metadata), record) + return record + + +def _agent_eval_summary_header(result: AgentEvalResult) -> str: + """Build the header line, mirroring the shape :func:`summary_header` produces for row results. + + The counts differ because the units do — a run has tasks, trials, and scores where the dataset + path has rows — but the ``Name(field=value, ...)`` shape is the same, and a status the run never + produced is left out, matching what that header does with its zero counts. + + Statuses are counted by tallying the scores present, so an absent status simply never becomes a + key; there is no zero to filter out. + """ + status_counts: dict[str, int] = {} + for score in result.scores: + status_counts[score.status.value] = status_counts.get(score.status.value, 0) + 1 + fields = [ + f"tasks={len(result.tasks)}", + f"trials={len(result.trials)}", + f"scores={len(result.scores)}", + f"aggregate_scores={len(result.summary.scores.scores)}", + ] + fields.extend(f"{status}={count}" for status, count in sorted(status_counts.items())) + return f"AgentEvalResult({', '.join(fields)})" + + +def _format_score_errors( + scores: Sequence[AgentEvalTaskScore], + *, + max_error_rows: int | None, +) -> list[str]: + """Render the failed-score detail section, separating a failed trial from a failed metric. + + Both arrive as ``FAILED``, but they mean different things to a reader: a failed trial is an + attempt the agent is answerable for, a failed metric is a measurement that never happened. The + dataset path has no equivalent distinction to make, so this section is agent-eval's own rather + than a reuse of :func:`format_error_details`. + """ + failed = [score for score in scores if score.status is AgentEvalScoreStatus.FAILED] + if not failed: + return [] + + # max(0, ...) guards a negative limit, which slicing would otherwise read as an offset from the + # end: failed[:-2] shows all but the last two rather than none. An over-large limit needs no + # guard, since a slice past the end is simply the whole list. Mirrors format_error_details. + shown_limit = len(failed) if max_error_rows is None else max(0, max_error_rows) + shown = failed[:shown_limit] + parts = ["", f"Error details ({len(shown)} of {len(failed)} failed scores)"] + for score in shown: + kind = "failed trial" if is_trial_failure(score) else "failed metric" + parts.extend(["", f"[{score.task_id} / {score.trial_id} / {score.metric_type}] {kind}"]) + parts.append(_score_error_text(score) or "(no error-severity diagnostic recorded)") + if len(shown) < len(failed): + parts.extend(["", f"... {len(failed) - len(shown)} more failed scores omitted"]) + return parts + def _aggregate_scores( scores: Sequence[AgentEvalTaskScore],