From 753c54f65317c9ace26c4ad8159ddcbebf7d7517 Mon Sep 17 00:00:00 2001
From: Sandy Chapman <schapman@nvidia.com>
Date: Mon, 10 Aug 2026 10:58:50 -0300
Subject: [PATCH 1/3] feat(evaluator): add by-name aggregate lookup to
 AgentEvalSummary

Reading one aggregate from a run meant a hand-written scan over a doubly-nested
attribute:

    next(s for s in result.summary.scores.scores if s.name == "...").mean

Seven sites did this -- six in tests (one via a private `_score` helper) and one
in the user-facing Hermes example, which is precisely the audience an ergonomics
gap costs most.

Adds `score(name)` and a `scores_by_name` mapping view to AggregatedMetricResult,
so the dataset path (`EvaluationResult.aggregate_scores`) gains the same surface,
with one-hop delegates on AgentEvalSummary.

A miss raises KeyError rather than returning None: an unknown name is nearly
always a typo or a metric that did not run, and failing where the name is in hand
beats an AttributeError on `.mean` downstream. `scores_by_name.get()` covers
legitimate absence.

The miss message leads with close matches rather than enumerating everything --
a run with several metrics times pass@k carries dozens of names, and a wall of
them buries the answer. It reports how many others exist so a wrong suggestion is
not a dead end, and truncates the fallback listing.

`score` is a method and `scores_by_name` a property, so neither enters the JSON
schema; verified against AggregatedMetricResult, AgentEvalSummary, and
EvaluationResult. The committed OpenAPI spec is unchanged.

Left alone deliberately: harbor_runtime and the dashboard iterate rather than
look up, and plugin_examples filters on two candidate names -- converting those
would change behaviour, not just style.

Signed-off-by: Sandy Chapman <schapman@nvidia.com>
---
 .../examples/hermes/example.py                |   2 +-
 .../nemo_evaluator_sdk/agent_eval/results.py  |  15 ++-
 .../src/nemo_evaluator_sdk/values/results.py  |  62 +++++++++
 .../tests/agent_eval/test_evaluator.py        |  22 +---
 .../agent_eval/test_gym_aggregate_scores.py   |   2 +-
 .../tests/agent_eval/test_hermes_example.py   |   2 +-
 .../tests/agent_eval/test_pass_at_k.py        |   8 +-
 .../agent_eval/test_summary_accessors.py      | 123 ++++++++++++++++++
 8 files changed, 211 insertions(+), 25 deletions(-)
 create mode 100644 packages/nemo_evaluator_sdk/tests/agent_eval/test_summary_accessors.py

diff --git a/packages/nemo_evaluator_sdk/examples/hermes/example.py b/packages/nemo_evaluator_sdk/examples/hermes/example.py
index eee0476b39..e216130706 100644
--- a/packages/nemo_evaluator_sdk/examples/hermes/example.py
+++ b/packages/nemo_evaluator_sdk/examples/hermes/example.py
@@ -219,7 +219,7 @@ async def main() -> None:
     result = await evaluate()
 
     answer = result.trials[0].output.output_text if result.trials[0].output else None
-    aggregate = next(score for score in result.summary.scores.scores if score.name == "keyword_match.score")
+    aggregate = result.summary.score("keyword_match.score")
 
     print(f"response: {answer}")
     print(f"{aggregate.name}: {aggregate.mean}")
diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py
index b31d1b5078..211610da55 100644
--- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py
+++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py
@@ -6,7 +6,7 @@
 from __future__ import annotations
 
 import math
-from collections.abc import Sequence
+from collections.abc import Mapping, Sequence
 from datetime import datetime
 from pathlib import Path
 
@@ -63,6 +63,19 @@ class AgentEvalSummary(BaseModel):
     trial_count: int = Field(default=0, description="Number of distinct trials scored.")
     score_count: int = Field(default=0, description="Total number of metric scores.")
 
+    @property
+    def scores_by_name(self) -> Mapping[str, AggregateScore]:
+        """Aggregates keyed by name — see :attr:`AggregatedMetricResult.scores_by_name`."""
+        return self.scores.scores_by_name
+
+    def score(self, name: str) -> AggregateScore:
+        """Return the aggregate named ``name`` — see :meth:`AggregatedMetricResult.score`.
+
+        Exists so callers needn't know the aggregates sit one level down, behind a field whose name
+        differs from the summary's own accessor by a single character.
+        """
+        return self.scores.score(name)
+
     @staticmethod
     def from_scores(
         scores: Sequence[AgentEvalTaskScore],
diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/results.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/results.py
index c52c31d71d..2ab78c065e 100644
--- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/results.py
+++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/results.py
@@ -7,6 +7,8 @@
 
 import json
 import math
+from collections.abc import Mapping
+from difflib import get_close_matches
 from typing import Any, Literal, Self
 
 import pyarrow as pa
@@ -417,12 +419,72 @@ def _serialize(self, handler):
 AggregateScore = AggregateRangeScore | AggregateRubricScore | AggregateScalarScore
 
 
+#: Names listed in a lookup-miss message when nothing resembles what was asked for. Enough to spot a
+#: naming-convention mistake, few enough to read; a run with several metrics times pass@k can carry
+#: dozens, and a wall of them buries the answer rather than giving it.
+_MISS_NAME_LIMIT = 10
+
+
 class AggregatedMetricResult(BaseModel):
     """Result of aggregating metric scores with full statistics."""
 
     model_config = ConfigDict(extra="forbid")
     scores: list[AggregateScore] = Field(description="The list of aggregated scores.")
 
+    @property
+    def scores_by_name(self) -> Mapping[str, AggregateScore]:
+        """Aggregates keyed by :attr:`AggregateScoreBase.name`, for ``in``, ``.get()``, and iteration.
+
+        Reach for this when a score's absence is a legitimate outcome ("did this metric run?"); use
+        :meth:`score` when it isn't. Names are expected unique, but runner-contributed extras are
+        appended as-is, so a collision is possible: the first wins, matching the ``next(...)`` scans
+        this replaces.
+        """
+        by_name: dict[str, AggregateScore] = {}
+        for score in self.scores:
+            by_name.setdefault(score.name, score)
+        return by_name
+
+    def score(self, name: str) -> AggregateScore:
+        """Return the aggregate named ``name``, raising :class:`KeyError` if there isn't one.
+
+        Raises rather than returning ``None`` because an unknown name is nearly always a typo or a
+        metric that didn't run. Both are bugs worth surfacing at the lookup, where the name is in
+        hand, instead of as an ``AttributeError`` on ``.mean`` further downstream. When absence is a
+        real possibility, use ``scores_by_name.get(...)``.
+        """
+        # A direct scan rather than a lookup into `scores_by_name`: building the whole mapping to
+        # return one element allocates a dict per call, and the score list is short enough that the
+        # scan wins outright.
+        for score in self.scores:
+            if score.name == name:
+                return score
+        raise KeyError(self._unknown_score_message(name))
+
+    def _unknown_score_message(self, name: str) -> str:
+        """Explain a lookup miss, leading with near-misses when the name looks like a typo."""
+        available = sorted(score.name for score in self.scores)
+        if not available:
+            return f"no aggregate score named {name!r}: this result has no aggregates at all"
+        # Suggestions beat enumeration for the common case (a typo, or the wrong pass@k), and stay
+        # useful when a run carries dozens of names.
+        close = get_close_matches(name, available, n=3)
+        if close:
+            suggestions = ", ".join(repr(match) for match in close)
+            message = f"no aggregate score named {name!r}; did you mean {suggestions}?"
+            # Say how many others there are, so a wrong guess isn't a dead end: without this the
+            # caller can't tell whether the suggestions are the whole set or three of forty.
+            others = len(available) - len(close)
+            if others == 0:
+                return message
+            noun = "aggregate" if others == 1 else "aggregates"
+            return f"{message} ({others} other {noun} in this result)"
+        shown = ", ".join(repr(score_name) for score_name in available[:_MISS_NAME_LIMIT])
+        remainder = len(available) - _MISS_NAME_LIMIT
+        if remainder > 0:
+            shown = f"{shown}, ... ({remainder} more)"
+        return f"no aggregate score named {name!r}; available: {shown}"
+
 
 class RowScore(BaseModel):
     """Normalized row-level score payload for metric/benchmark job results."""
diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_evaluator.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_evaluator.py
index 94d4efec56..8274ddb20d 100644
--- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_evaluator.py
+++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_evaluator.py
@@ -47,7 +47,6 @@
     RunConfigOnlineModel,
 )
 from nemo_evaluator_sdk.values.evidence import CandidateEvidence, EvidenceDescriptor
-from nemo_evaluator_sdk.values.results import AggregateScore
 
 
 def test_trial_from_sample_falls_back_to_reasoning_content() -> None:
@@ -147,13 +146,6 @@ def test_metric_row_exposes_reference_but_task_row_hides_it() -> None:
     assert "reference" not in task_row
 
 
-def _score(summary: AgentEvalSummary, name: str) -> AggregateScore:
-    for aggregate in summary.scores.scores:
-        if aggregate.name == name:
-            return aggregate
-    raise KeyError(name)
-
-
 class _ConstantMetric:
     @property
     def type(self) -> str:
@@ -327,7 +319,7 @@ async def test_scores_imported_trials_with_metric_and_persists_bundle(tmp_path:
     # run() no longer writes anything; persisting is the caller's call and defaults to the work_dir.
     location = result.persist()
 
-    assert _score(result.summary, "constant_metric.score").mean == 0.75
+    assert result.summary.score("constant_metric.score").mean == 0.75
     assert location.output_dir == tmp_path
     assert location.dashboard_path == tmp_path / "report.html"
     assert (tmp_path / "run.json").exists()
@@ -371,7 +363,7 @@ async def test_scores_partial_trials() -> None:
         ],
     )
 
-    assert _score(result.summary, "constant_metric.score").mean == 0.75
+    assert result.summary.score("constant_metric.score").mean == 0.75
 
 
 @pytest.mark.asyncio
@@ -385,7 +377,7 @@ async def test_target_runtime_produces_trials_before_scoring() -> None:
     assert result.trials[0].id == "task-1:runtime"
     assert runtime.config is not None
     assert runtime.config.run_id == result.run_id
-    assert _score(result.summary, "constant_metric.score").mean == 0.75
+    assert result.summary.score("constant_metric.score").mean == 0.75
 
 
 @pytest.mark.asyncio
@@ -411,7 +403,7 @@ async def fake_model_inference(
     assert result.trials[0].metadata["model_id"] == "target-model"
     assert result.trials[0].output is not None
     assert result.trials[0].output.output_text == "Generated model answer"
-    assert _score(result.summary, "constant_metric.score").mean == 0.75
+    assert result.summary.score("constant_metric.score").mean == 0.75
 
 
 @pytest.mark.asyncio
@@ -524,9 +516,9 @@ def test_summary_reports_coverage_and_merges_views_into_scores() -> None:
 
     summary = AgentEvalSummary.from_scores(scores, tasks=[task])
 
-    assert _score(summary, "constant_metric.score").mean == 1.0
-    assert _score(summary, "other_metric.quality").mean == 0.0
-    assert _score(summary, "view.outcome_correctness").mean == 0.5
+    assert summary.score("constant_metric.score").mean == 1.0
+    assert summary.score("other_metric.quality").mean == 0.0
+    assert summary.score("view.outcome_correctness").mean == 0.5
     assert summary.metric_coverage["constant_metric"]["score"].total == 1
     assert summary.metric_coverage["constant_metric"]["score"].scored == 1
 
diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_gym_aggregate_scores.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_gym_aggregate_scores.py
index df5c60166a..cbbbf63ebe 100644
--- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_gym_aggregate_scores.py
+++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_gym_aggregate_scores.py
@@ -223,7 +223,7 @@ def test_median_matches_p50_wherever_both_are_reported() -> None:
     ]
 
     summary = AgentEvalSummary.from_scores(scores)
-    aggregate = next(s for s in summary.scores.scores if s.name == "m.score")
+    aggregate = summary.score("m.score")
 
     assert isinstance(aggregate, AggregateRangeScore)
     assert aggregate.percentiles is not None
diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_hermes_example.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_hermes_example.py
index 5e2df89c61..12e04f1b45 100644
--- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_hermes_example.py
+++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_hermes_example.py
@@ -76,7 +76,7 @@ def handle(request: httpx.Request) -> httpx.Response:
     assert trial.output is not None
     assert trial.output.output_text == "streaming works"
 
-    aggregate = next(score for score in result.summary.scores.scores if score.name == "keyword_match.score")
+    aggregate = result.summary.score("keyword_match.score")
     assert aggregate.mean == 1.0
 
     assert trial.evidence is not None
diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_pass_at_k.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_pass_at_k.py
index 30df07e1c0..00d3bf2625 100644
--- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_pass_at_k.py
+++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_pass_at_k.py
@@ -136,9 +136,7 @@ def test_population_and_sample_stats_are_both_reported() -> None:
     values = [1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0]  # n=10, mean=0.6, sum_sq_dev=2.4
     scores = [_score(f"t{1 + index % 2}", f"a{index}", "reward", "reward", value) for index, value in enumerate(values)]
 
-    aggregate = next(
-        s for s in AgentEvalSummary.from_scores(scores, tasks=tasks).scores.scores if s.name == "reward.reward"
-    )
+    aggregate = AgentEvalSummary.from_scores(scores, tasks=tasks).score("reward.reward")
 
     assert aggregate.count == 10
     assert aggregate.mean == pytest.approx(0.6)
@@ -153,9 +151,7 @@ def test_sample_stats_undefined_for_a_single_value() -> None:
     tasks = [_task("t1", _ScoreMetric("reward"))]
     scores = [_score("t1", "a0", "reward", "reward", 1.0)]
 
-    aggregate = next(
-        s for s in AgentEvalSummary.from_scores(scores, tasks=tasks).scores.scores if s.name == "reward.reward"
-    )
+    aggregate = AgentEvalSummary.from_scores(scores, tasks=tasks).score("reward.reward")
 
     assert aggregate.std_dev == 0.0  # population is well-defined for one value
     assert aggregate.sample_std_dev is None
diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_summary_accessors.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_summary_accessors.py
new file mode 100644
index 0000000000..176faced6f
--- /dev/null
+++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_summary_accessors.py
@@ -0,0 +1,123 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""By-name aggregate lookup on AgentEvalSummary and the AggregatedMetricResult it delegates to."""
+
+from __future__ import annotations
+
+import pytest
+from nemo_evaluator_sdk.agent_eval.results import AgentEvalSummary
+from nemo_evaluator_sdk.values.results import AggregatedMetricResult, AggregateRangeScore
+
+
+def _aggregates(*names: str) -> AggregatedMetricResult:
+    return AggregatedMetricResult(
+        scores=[AggregateRangeScore(name=name, count=2, nan_count=0, mean=0.5) for name in names]
+    )
+
+
+def _summary(*names: str) -> AgentEvalSummary:
+    return AgentEvalSummary(scores=_aggregates(*names))
+
+
+def test_score_returns_the_aggregate_with_that_name() -> None:
+    summary = _summary("gym_reward.reward", "gym_reward.reward.pass@2")
+
+    assert summary.score("gym_reward.reward.pass@2").mean == 0.5
+
+
+def test_score_finds_a_name_that_is_not_the_first_in_the_list() -> None:
+    # Guards the scan itself: returning self.scores[0] regardless of name would satisfy a
+    # single-aggregate test but is plainly wrong.
+    summary = _summary("a.first", "b.second", "c.third")
+
+    assert summary.score("c.third").name == "c.third"
+
+
+def test_score_suggests_the_intended_name_when_the_lookup_looks_like_a_typo() -> None:
+    summary = _summary("gym_reward.reward", "view.solved")
+
+    with pytest.raises(KeyError, match="did you mean") as excinfo:
+        summary.score("gym_reward.rewrad")
+
+    # The suggestion is the whole point: a transposed name should be fixable from the message alone,
+    # without going back to the aggregation code to find out what was produced.
+    assert "gym_reward.reward" in str(excinfo.value)
+
+
+@pytest.mark.parametrize(
+    ("extra_names", "expected"),
+    [
+        # A suggestion that misses shouldn't be a dead end: the count tells the caller there is more
+        # to look at, rather than implying the offered names are the whole set.
+        (20, "(20 other aggregates in this result)"),
+        (1, "(1 other aggregate in this result)"),
+        (0, None),
+    ],
+)
+def test_score_reports_how_many_other_names_exist_alongside_a_suggestion(
+    extra_names: int, expected: str | None
+) -> None:
+    aggregates = _aggregates("reward.reward", *(f"metric_{index:02d}.zzz" for index in range(extra_names)))
+
+    with pytest.raises(KeyError) as excinfo:
+        aggregates.score("reward.rewrad")
+
+    message = str(excinfo.value)
+    assert "'reward.reward'" in message
+    if expected is None:
+        assert "in this result" not in message
+    else:
+        assert expected in message
+
+
+def test_score_lists_available_names_when_nothing_is_close() -> None:
+    summary = _summary("gym_reward.reward", "view.solved")
+
+    with pytest.raises(KeyError) as excinfo:
+        summary.score("totally_unrelated")
+
+    message = str(excinfo.value)
+    assert "did you mean" not in message
+    assert "gym_reward.reward" in message
+    assert "view.solved" in message
+
+
+def test_score_truncates_a_long_name_list_rather_than_dumping_all_of_them() -> None:
+    # A real run carries several metrics times pass@k values; an untruncated dump buries the answer.
+    aggregates = _aggregates(*(f"metric_{index:02d}.zzz" for index in range(25)))
+
+    with pytest.raises(KeyError) as excinfo:
+        aggregates.score("qqq")
+
+    message = str(excinfo.value)
+    assert "(15 more)" in message
+    assert "metric_00.zzz" in message
+    assert "metric_24.zzz" not in message
+
+
+def test_score_says_so_when_the_result_has_no_aggregates_at_all() -> None:
+    # Distinct from a typo: nothing was produced, so no name would have worked.
+    with pytest.raises(KeyError, match="no aggregates at all"):
+        _summary().score("anything")
+
+
+def test_scores_by_name_supports_membership_and_get_for_optional_aggregates() -> None:
+    summary = _summary("gym_reward.reward")
+
+    assert "gym_reward.reward" in summary.scores_by_name
+    assert summary.scores_by_name.get("never_ran") is None
+
+
+def test_scores_by_name_keeps_the_first_of_a_repeated_name() -> None:
+    # Names are expected unique, but runner-contributed extras are appended as-is. First-wins matches
+    # the `next(...)` scans this replaced, so a collision behaves as it did before.
+    duplicated = AggregatedMetricResult(
+        scores=[
+            AggregateRangeScore(name="m.score", count=2, nan_count=0, mean=0.25),
+            AggregateRangeScore(name="m.score", count=2, nan_count=0, mean=0.75),
+        ]
+    )
+
+    assert duplicated.score("m.score").mean == 0.25
+    assert duplicated.scores_by_name["m.score"].mean == 0.25

From 690ab863fc5f7e089cd220c2416da852b9971087 Mon Sep 17 00:00:00 2001
From: Sandy Chapman <schapman@nvidia.com>
Date: Mon, 10 Aug 2026 12:56:01 -0300
Subject: [PATCH 2/3] fix(evaluator): dedupe aggregate names in miss
 diagnostics and sync the vendored SDK

Two follow-ups on the by-name lookup.

**Duplicate names counted twice.** `available` was built from a generator, so a
repeated aggregate name was suggested twice, listed twice in the fallback
enumeration, and counted twice in the "N other aggregates" tally -- making one
collision read as two distinct near-misses. `scores_by_name` already collapsed
duplicates first-wins; the diagnostic path did not, and the two disagreed.
Deduplicated, with tests for both the suggestion and enumeration branches.

Reported by CodeRabbit on #1198.

**Vendored SDK out of sync.** `nemo_evaluator_sdk` is vendored into
`sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/`, so any change to
its source needs `make vendor` re-run and the result committed. Without it
`lint-sdk-vendored` fails, and `lint-cli` fails after it because the vendor step
leaves `sdk/python/` dirty and lint-cli diffs that same path.

Verifying the JSON schema was unchanged was necessary but not sufficient: the
vendor copies source, not schema.

Signed-off-by: Sandy Chapman <schapman@nvidia.com>
---
 .../src/nemo_evaluator_sdk/values/results.py  |  5 +-
 .../agent_eval/test_summary_accessors.py      | 23 +++++++
 .../beta/evaluator/agent_eval/results.py      | 15 ++++-
 .../beta/evaluator/values/results.py          | 65 +++++++++++++++++++
 4 files changed, 106 insertions(+), 2 deletions(-)

diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/results.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/results.py
index 2ab78c065e..50234e2bda 100644
--- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/results.py
+++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/results.py
@@ -463,7 +463,10 @@ def score(self, name: str) -> AggregateScore:
 
     def _unknown_score_message(self, name: str) -> str:
         """Explain a lookup miss, leading with near-misses when the name looks like a typo."""
-        available = sorted(score.name for score in self.scores)
+        # Deduplicated: names are expected unique, but runner-contributed extras are appended as-is,
+        # and a repeat would otherwise be suggested twice, listed twice, and counted twice in the
+        # "N other aggregates" tally -- making a collision look like two distinct near-misses.
+        available = sorted({score.name for score in self.scores})
         if not available:
             return f"no aggregate score named {name!r}: this result has no aggregates at all"
         # Suggestions beat enumeration for the common case (a typo, or the wrong pass@k), and stay
diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_summary_accessors.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_summary_accessors.py
index 176faced6f..4a0e47c48e 100644
--- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_summary_accessors.py
+++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_summary_accessors.py
@@ -71,6 +71,29 @@ def test_score_reports_how_many_other_names_exist_alongside_a_suggestion(
         assert expected in message
 
 
+def test_miss_message_names_a_repeated_aggregate_only_once() -> None:
+    # A duplicate name is one name, not two near-misses: suggesting it twice, listing it twice, or
+    # counting it twice in the "other aggregates" tally all misrepresent what the result holds.
+    duplicated = _aggregates("reward.reward", "reward.reward", "view.solved")
+
+    with pytest.raises(KeyError) as excinfo:
+        duplicated.score("reward.rewrad")
+
+    message = str(excinfo.value)
+    assert message.count("'reward.reward'") == 1
+    assert "(1 other aggregate in this result)" in message
+
+
+def test_miss_message_does_not_repeat_a_duplicate_in_the_fallback_listing() -> None:
+    duplicated = _aggregates("alpha.one", "alpha.one", "beta.two")
+
+    with pytest.raises(KeyError) as excinfo:
+        duplicated.score("zzzzzz")
+
+    message = str(excinfo.value)
+    assert message.count("'alpha.one'") == 1
+
+
 def test_score_lists_available_names_when_nothing_is_close() -> None:
     summary = _summary("gym_reward.reward", "view.solved")
 
diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py
index cddf78e2ef..20c38c695d 100644
--- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py
+++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py
@@ -6,7 +6,7 @@
 from __future__ import annotations
 
 import math
-from collections.abc import Sequence
+from collections.abc import Mapping, Sequence
 from datetime import datetime
 from pathlib import Path
 
@@ -63,6 +63,19 @@ class AgentEvalSummary(BaseModel):
     trial_count: int = Field(default=0, description="Number of distinct trials scored.")
     score_count: int = Field(default=0, description="Total number of metric scores.")
 
+    @property
+    def scores_by_name(self) -> Mapping[str, AggregateScore]:
+        """Aggregates keyed by name — see :attr:`AggregatedMetricResult.scores_by_name`."""
+        return self.scores.scores_by_name
+
+    def score(self, name: str) -> AggregateScore:
+        """Return the aggregate named ``name`` — see :meth:`AggregatedMetricResult.score`.
+
+        Exists so callers needn't know the aggregates sit one level down, behind a field whose name
+        differs from the summary's own accessor by a single character.
+        """
+        return self.scores.score(name)
+
     @staticmethod
     def from_scores(
         scores: Sequence[AgentEvalTaskScore],
diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/results.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/results.py
index 9cf87e2674..4caa3a6b26 100644
--- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/results.py
+++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/results.py
@@ -7,6 +7,8 @@
 
 import json
 import math
+from collections.abc import Mapping
+from difflib import get_close_matches
 from typing import Any, Literal, Self
 
 import pyarrow as pa
@@ -417,12 +419,75 @@ def _serialize(self, handler):
 AggregateScore = AggregateRangeScore | AggregateRubricScore | AggregateScalarScore
 
 
+#: Names listed in a lookup-miss message when nothing resembles what was asked for. Enough to spot a
+#: naming-convention mistake, few enough to read; a run with several metrics times pass@k can carry
+#: dozens, and a wall of them buries the answer rather than giving it.
+_MISS_NAME_LIMIT = 10
+
+
 class AggregatedMetricResult(BaseModel):
     """Result of aggregating metric scores with full statistics."""
 
     model_config = ConfigDict(extra="forbid")
     scores: list[AggregateScore] = Field(description="The list of aggregated scores.")
 
+    @property
+    def scores_by_name(self) -> Mapping[str, AggregateScore]:
+        """Aggregates keyed by :attr:`AggregateScoreBase.name`, for ``in``, ``.get()``, and iteration.
+
+        Reach for this when a score's absence is a legitimate outcome ("did this metric run?"); use
+        :meth:`score` when it isn't. Names are expected unique, but runner-contributed extras are
+        appended as-is, so a collision is possible: the first wins, matching the ``next(...)`` scans
+        this replaces.
+        """
+        by_name: dict[str, AggregateScore] = {}
+        for score in self.scores:
+            by_name.setdefault(score.name, score)
+        return by_name
+
+    def score(self, name: str) -> AggregateScore:
+        """Return the aggregate named ``name``, raising :class:`KeyError` if there isn't one.
+
+        Raises rather than returning ``None`` because an unknown name is nearly always a typo or a
+        metric that didn't run. Both are bugs worth surfacing at the lookup, where the name is in
+        hand, instead of as an ``AttributeError`` on ``.mean`` further downstream. When absence is a
+        real possibility, use ``scores_by_name.get(...)``.
+        """
+        # A direct scan rather than a lookup into `scores_by_name`: building the whole mapping to
+        # return one element allocates a dict per call, and the score list is short enough that the
+        # scan wins outright.
+        for score in self.scores:
+            if score.name == name:
+                return score
+        raise KeyError(self._unknown_score_message(name))
+
+    def _unknown_score_message(self, name: str) -> str:
+        """Explain a lookup miss, leading with near-misses when the name looks like a typo."""
+        # Deduplicated: names are expected unique, but runner-contributed extras are appended as-is,
+        # and a repeat would otherwise be suggested twice, listed twice, and counted twice in the
+        # "N other aggregates" tally -- making a collision look like two distinct near-misses.
+        available = sorted({score.name for score in self.scores})
+        if not available:
+            return f"no aggregate score named {name!r}: this result has no aggregates at all"
+        # Suggestions beat enumeration for the common case (a typo, or the wrong pass@k), and stay
+        # useful when a run carries dozens of names.
+        close = get_close_matches(name, available, n=3)
+        if close:
+            suggestions = ", ".join(repr(match) for match in close)
+            message = f"no aggregate score named {name!r}; did you mean {suggestions}?"
+            # Say how many others there are, so a wrong guess isn't a dead end: without this the
+            # caller can't tell whether the suggestions are the whole set or three of forty.
+            others = len(available) - len(close)
+            if others == 0:
+                return message
+            noun = "aggregate" if others == 1 else "aggregates"
+            return f"{message} ({others} other {noun} in this result)"
+        shown = ", ".join(repr(score_name) for score_name in available[:_MISS_NAME_LIMIT])
+        remainder = len(available) - _MISS_NAME_LIMIT
+        if remainder > 0:
+            shown = f"{shown}, ... ({remainder} more)"
+        return f"no aggregate score named {name!r}; available: {shown}"
+
 
 class RowScore(BaseModel):
     """Normalized row-level score payload for metric/benchmark job results."""

From abd342a559461e04e77736485297dad92c2e5744 Mon Sep 17 00:00:00 2001
From: Sandy Chapman <schapman@nvidia.com>
Date: Mon, 10 Aug 2026 14:07:33 -0300
Subject: [PATCH 3/3] docs(evaluator): say what "close" means in an
 aggregate-name miss message

Review question on #1198: what counts as a close match, and would listing the
names sorted be simpler?

Documents the answer where the code is rather than in a review thread. "Close"
is difflib.get_close_matches -- SequenceMatcher (Ratcliff/Obershelp) similarity
of at least 0.6, best three first -- which is a subsequence-overlap ratio, not
an edit distance.

Also records why the sorted list is not enough on its own: it is the better
answer left whole, but truncating one breaks it, because the name a caller
meant is not reliably in the first N. A typo'd `view.solved` sits behind a page
of `gym_reward.*` in a run carrying pass@1..8 for two metrics.

_MISS_NAME_LIMIT now says 10 is a judgement call rather than implying a measured
optimum, which is what the comment read like.

No behaviour change; the vendored SDK copy is re-synced.

Signed-off-by: Sandy Chapman <schapman@nvidia.com>
---
 .../src/nemo_evaluator_sdk/values/results.py  | 21 +++++++++++++------
 .../beta/evaluator/values/results.py          | 21 +++++++++++++------
 2 files changed, 30 insertions(+), 12 deletions(-)

diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/results.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/results.py
index 50234e2bda..02eef50977 100644
--- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/results.py
+++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/results.py
@@ -419,9 +419,11 @@ def _serialize(self, handler):
 AggregateScore = AggregateRangeScore | AggregateRubricScore | AggregateScalarScore
 
 
-#: Names listed in a lookup-miss message when nothing resembles what was asked for. Enough to spot a
-#: naming-convention mistake, few enough to read; a run with several metrics times pass@k can carry
-#: dozens, and a wall of them buries the answer rather than giving it.
+#: How many names a lookup-miss message lists when nothing resembles what was asked for. A judgement
+#: call rather than a measured optimum -- enough to show the naming convention, few enough to stay
+#: readable, since a run with several metrics times pass@k can carry dozens. Only this fallback is
+#: truncated; a near-miss is surfaced by similarity, so finding the name you meant never depends on
+#: where it happens to fall alphabetically.
 _MISS_NAME_LIMIT = 10
 
 
@@ -462,15 +464,22 @@ def score(self, name: str) -> AggregateScore:
         raise KeyError(self._unknown_score_message(name))
 
     def _unknown_score_message(self, name: str) -> str:
-        """Explain a lookup miss, leading with near-misses when the name looks like a typo."""
+        """Explain a lookup miss, leading with near-misses when the name looks like a typo.
+
+        "Close" is :func:`difflib.get_close_matches`: SequenceMatcher (Ratcliff/Obershelp) similarity
+        of at least 0.6, best three first. That is a subsequence-overlap ratio, not an edit distance.
+
+        Listing every name alphabetically would be simpler, and is the better answer if the list is
+        left whole. Truncating one is what breaks it -- the name a caller meant is not reliably in
+        the first :data:`_MISS_NAME_LIMIT`, since a typo'd ``view.solved`` sits behind a page of
+        ``gym_reward.*`` in a run carrying pass@1..8 for two metrics.
+        """
         # Deduplicated: names are expected unique, but runner-contributed extras are appended as-is,
         # and a repeat would otherwise be suggested twice, listed twice, and counted twice in the
         # "N other aggregates" tally -- making a collision look like two distinct near-misses.
         available = sorted({score.name for score in self.scores})
         if not available:
             return f"no aggregate score named {name!r}: this result has no aggregates at all"
-        # Suggestions beat enumeration for the common case (a typo, or the wrong pass@k), and stay
-        # useful when a run carries dozens of names.
         close = get_close_matches(name, available, n=3)
         if close:
             suggestions = ", ".join(repr(match) for match in close)
diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/results.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/results.py
index 4caa3a6b26..52551f88c8 100644
--- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/results.py
+++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/results.py
@@ -419,9 +419,11 @@ def _serialize(self, handler):
 AggregateScore = AggregateRangeScore | AggregateRubricScore | AggregateScalarScore
 
 
-#: Names listed in a lookup-miss message when nothing resembles what was asked for. Enough to spot a
-#: naming-convention mistake, few enough to read; a run with several metrics times pass@k can carry
-#: dozens, and a wall of them buries the answer rather than giving it.
+#: How many names a lookup-miss message lists when nothing resembles what was asked for. A judgement
+#: call rather than a measured optimum -- enough to show the naming convention, few enough to stay
+#: readable, since a run with several metrics times pass@k can carry dozens. Only this fallback is
+#: truncated; a near-miss is surfaced by similarity, so finding the name you meant never depends on
+#: where it happens to fall alphabetically.
 _MISS_NAME_LIMIT = 10
 
 
@@ -462,15 +464,22 @@ def score(self, name: str) -> AggregateScore:
         raise KeyError(self._unknown_score_message(name))
 
     def _unknown_score_message(self, name: str) -> str:
-        """Explain a lookup miss, leading with near-misses when the name looks like a typo."""
+        """Explain a lookup miss, leading with near-misses when the name looks like a typo.
+
+        "Close" is :func:`difflib.get_close_matches`: SequenceMatcher (Ratcliff/Obershelp) similarity
+        of at least 0.6, best three first. That is a subsequence-overlap ratio, not an edit distance.
+
+        Listing every name alphabetically would be simpler, and is the better answer if the list is
+        left whole. Truncating one is what breaks it -- the name a caller meant is not reliably in
+        the first :data:`_MISS_NAME_LIMIT`, since a typo'd ``view.solved`` sits behind a page of
+        ``gym_reward.*`` in a run carrying pass@1..8 for two metrics.
+        """
         # Deduplicated: names are expected unique, but runner-contributed extras are appended as-is,
         # and a repeat would otherwise be suggested twice, listed twice, and counted twice in the
         # "N other aggregates" tally -- making a collision look like two distinct near-misses.
         available = sorted({score.name for score in self.scores})
         if not available:
             return f"no aggregate score named {name!r}: this result has no aggregates at all"
-        # Suggestions beat enumeration for the common case (a typo, or the wrong pass@k), and stay
-        # useful when a run carries dozens of names.
         close = get_close_matches(name, available, n=3)
         if close:
             suggestions = ", ".join(repr(match) for match in close)