NVIDIA-NeMo · sklinglernv · Aug 17, 2026 · Aug 3, 2026 · Aug 3, 2026 · Aug 3, 2026
@@ -459,6 +459,13 @@ async def _run(
                 total=len(trials),
                 unit="trace",
             )
+
+        async def load_trace(reference: ResourceRef) -> TraceExplorer:
+            """Resolve a trace ref with this run's client, matching the Experimentalist's
+            `ctx.load_trace`: the analyzer takes a loader rather than a platform client,
+            so its signature names no platform type."""
+            return await TraceExplorer.from_ref(reference, client, insight.workspace)
+
         raw_diagnostics: list[Diagnostic | BaseException] = list(
             await asyncio.gather(
                 *[
@@ -467,8 +474,7 @@ async def _run(
                         task=task,
                         agent_path=resolved_agent,
                         insight=insight,
-                        client=client,
-                        workspace=insight.workspace,
+                        load_trace=load_trace,
                     )
                     for analyzer, trial, task in zip(analyzers, trials, tasks, strict=True)
                 ],

@@ -10,7 +10,7 @@
 workspace: default
 base_url: http://localhost:8080
 mode: local
-evaluator_type: harbor_native
+evaluator_type: harbor-native
 
 # Required per run.
 insight: ""

@@ -51,7 +51,7 @@ async def run_eval_author(
     base_url: str | None,
     config: EvalAuthorConfig,
     agent: Path | str | None = None,
-    evaluator_type: EvaluatorType = "harbor_native",
+    evaluator_type: EvaluatorType = "harbor-native",
     model_refs: ConfiguredModelRefs | None = None,
 ) -> EvalAuthorResult:
     """Stage evaluation inputs, resolve one Insight, then run Eval Author.

@@ -401,11 +401,17 @@ async def analyze_trace(
         trace=ResourceRef(uri=f"intake://{trace_id}", description="Production trace read from Intake."),
         metadata={"source": "intake"},
     )
+
+    async def load_trace(reference: ResourceRef) -> TraceExplorer:
+        """Resolve a trace ref with this call's client, matching the Experimentalist's
+        `ctx.load_trace`: the analyzer takes a loader rather than a platform client, so
+        its signature names no platform type."""
+        return await TraceExplorer.from_ref(reference, client, workspace)
+
     return await TraceAnalyzer(experiment_dir=experiment_dir).run(
         trial=trial,
         task=task,
         agent_path=agent_path,
         insight=None,
-        client=client,
-        workspace=workspace,
+        load_trace=load_trace,
     )
@@ -186,10 +186,9 @@ async def run(
             task: Task,
             agent_path: Path,
             insight: Insight,
-            client: Any,
-            workspace: str,
+            load_trace: Any,
         ) -> Diagnostic:
-            del task, agent_path, insight, client, workspace
+            del task, agent_path, insight, load_trace
             ref = cast(str, trial.metadata["trace_ref"])
             calls.analyzed_refs.append(ref)
             outcome = outcomes[self.index]

@@ -15,12 +15,10 @@
 from nemo_eval_author_plugin.eval_author.models import EvalAuthorConfig
 from nemo_experimentalist_plugin.client import make_client
 from nemo_experimentalist_plugin.entities import DatasetValidationError, local_path_from_uri
-from nemo_experimentalist_plugin.experimentalist.components.evaluator.harbor import (
-    HarborDataset,
-)
 from nemo_experimentalist_plugin.experimentalist.components.evaluator.harbor_native import (
-    HarborEvaluator,
+    HarborDataset,
     HarborEvaluatorConfig,
+    HarborNativeOutcomeEvaluator,
 )
 from nemo_experimentalist_plugin.experimentalist.components.trace_analyzer import Diagnostic
 from nemo_insights_plugin.entities import Insight
@@ -401,7 +399,7 @@ async def test_eval_author_metric_scores_known_failing_harbor_baseline_low(
 
     assert summary.summary
     await insight_suite.validate()
-    evaluator = HarborEvaluator(experiment_dir=tmp_path)
+    evaluator = HarborNativeOutcomeEvaluator(experiment_dir=tmp_path)
     result = await asyncio.wait_for(
         evaluator.run(
             agent=agent_dir,
@@ -530,7 +528,7 @@ async def test_eval_author_metric_discriminates_controlled_harbor_tool_evidence(
 
     assert summary.summary
     await insight_suite.validate()
-    evaluator = HarborEvaluator(experiment_dir=tmp_path)
+    evaluator = HarborNativeOutcomeEvaluator(experiment_dir=tmp_path)
 
     async def run_agent(agent_dir: Path, job_name: str):
         return await asyncio.wait_for(

@@ -158,7 +158,7 @@ def missing_model_refs() -> eval_author_run.ConfiguredModelRefs:
     make_client.assert_not_called()
 
 
-@pytest.mark.parametrize("evaluator_type", ["harbor_native", "harbor_evaluator"])
+@pytest.mark.parametrize("evaluator_type", ["harbor-native", "harbor-runner"])
 @pytest.mark.asyncio
 async def test_run_eval_author_resolves_inputs_and_returns_datasets(
     monkeypatch: pytest.MonkeyPatch,
@@ -315,7 +315,7 @@ async def resolve_model_clients(*_: object) -> ClosingModelClients:
     results: list[EvalAuthorResult] = []
     calls: list[tuple[Insight, Path, Task, Dataset, Dataset, ClosingClient]] = []
 
-    for evaluator_type in ("harbor_native", "harbor_evaluator"):
+    for evaluator_type in ("harbor-native", "harbor-runner"):
         results.append(
             await eval_author_run.run_eval_author(
                 insight="insight-remote-123",
@@ -354,7 +354,7 @@ async def resolve_model_clients(*_: object) -> ClosingModelClients:
     assert all(model_clients.closed for model_clients in model_client_sets)
 
 
-@pytest.mark.parametrize("evaluator_type", ["harbor_native", "harbor_evaluator"])
+@pytest.mark.parametrize("evaluator_type", ["harbor-native", "harbor-runner"])
 @pytest.mark.asyncio
 async def test_run_eval_author_hydrates_fileset_task_template(
     monkeypatch: pytest.MonkeyPatch,

@@ -70,17 +70,17 @@ breaking rename with no compatibility aliases:
 Two names deliberately did **not** change. `optimizer.yaml` and the
 `.nemo-optimizer/` state directory are a shared contract with
 `nemo-insights-plugin`: `PROFILE_FILENAME` and `discover_profile()` live in
-`nemo_insights_plugin.contracts.profile`, and `nemo agents analyst run` can
-mirror the Platform rows it wrote into `<profile-dir>/.nemo-optimizer/insights.yaml`
-via `--insights-file-output`, which this plugin reads as the default insight when
-the file exists. Rename them only in lockstep with a Platform change to that
-contract. `EvolutionaryOptimizer` and `EvolutionaryOptimizerConfig` also keep
-their names — they describe the optimization algorithm, not the product.
-
-At the time of this rename the command group was top-level (`nemo
-experimentalist`), because the platform's `nemo.cli` entry-point group was flat
-and nesting under `nemo agents` needed a Platform-side change first. That change
-has since landed — see the `nemo agents` entry above for the current path.
+`nemo_insights_plugin.contracts.profile`, and `nemo agents analyst run` writes
+`<profile-dir>/.nemo-optimizer/insights.yaml`, which this plugin reads as the
+default insight. Rename them only in lockstep with a Platform change to that
+contract.
+
+`EvolutionaryOptimizer` is now `EvolutionaryStrategy`, in
+`experimentalist/strategies/evolutionary.py`, and is resolved by name like any other
+component (`strategy: evolutionary`).
+
+Run `nemo agents experimentalist components` to see everything this install can resolve,
+including components registered by a separately installed package.
 
 ### 2026-07-21: Curator renamed to Eval Author
 

@@ -14,9 +14,9 @@ optimizer:
   max_trajectory_tasks: 8
   max_train_batch_tasks: 16
   train_batch_seed: 20260727
-  disable_trajectory_scoring: false
-  disable_convergence_check: false
-  evaluator:
+  trajectory_scorer: goal-tree
+  terminator: convergence
+  outcome_evaluator_config:
     n_attempts: 2
     # Keep at 3 or lower: each task requests 8192 MB and parallel image builds
     # have triggered Docker Hub rate limiting and DNS failures.

@@ -14,9 +14,9 @@ optimizer:
   max_trajectory_tasks: 2
   max_train_batch_tasks: 4
   train_batch_seed: 20260727
-  disable_trajectory_scoring: true
-  disable_convergence_check: true
-  evaluator:
+  trajectory_scorer: null
+  terminator: null
+  outcome_evaluator_config:
     n_attempts: 1
     n_concurrent_trials: 1
     quiet: true

@@ -13,8 +13,8 @@ optimizer:
   max_trajectory_tasks: 6
   max_train_batch_tasks: 12
   train_batch_seed: 20260722
-  disable_trajectory_scoring: true
-  evaluator:
+  trajectory_scorer: null
+  outcome_evaluator_config:
     n_attempts: 2
     n_concurrent_trials: 2
     quiet: true

@@ -13,9 +13,9 @@ optimizer:
   max_trajectory_tasks: 2
   max_train_batch_tasks: 4
   train_batch_seed: 20260722
-  disable_trajectory_scoring: true
-  disable_convergence_check: true
-  evaluator:
+  trajectory_scorer: null
+  terminator: null
+  outcome_evaluator_config:
     n_attempts: 1
     n_concurrent_trials: 1
     quiet: true

@@ -14,6 +14,7 @@
 from typing import Any, Literal, Self
 
 import yaml
+from harbor.models.task.id import GitTaskId, LocalTaskId, PackageTaskId
 from harbor.registry.client.package import PackageDatasetClient
 from nemo_experimentalist_plugin.config import EvolutionaryOptimizerConfig
 from nemo_experimentalist_plugin.entities import (
@@ -24,9 +25,10 @@
 )
 from nemo_experimentalist_plugin.experimentalist.components.evaluator.harbor import HarborDataset
 from nemo_experimentalist_plugin.experimentalist.components.evaluator.harbor_native import (
-    HarborEvaluator,
     HarborEvaluatorConfig,
+    HarborNativeOutcomeEvaluator,
 )
+from nemo_experimentalist_plugin.experimentalist.experimentalist_backend import load_winner
 from nemo_experimentalist_plugin.resolve import resolve_dataset
 from pydantic import BaseModel, Field, model_validator
 
@@ -151,6 +153,24 @@ def load_benchmark_config(path: Path) -> BenchmarkConfig:
     return BenchmarkConfig.model_validate(_load_yaml(path))
 
 
+def _canonical_task_id(task: GitTaskId | LocalTaskId | PackageTaskId) -> str:
+    """The suite-facing id of one Harbor task.
+
+    Harbor's task ids are a union, and only ``PackageTaskId`` carries a ``name``. The
+    others do have a ``get_name()``, but it means something different — ``hello-world``
+    for a git or local task versus ``org/hello-world`` for a package one — so there is no
+    accessor that yields the same string across the union. Benchmark suites are pinned to
+    a published package, so anything else is a suite that was authored wrong, and saying
+    so beats silently substituting a value.
+    """
+    if not isinstance(task, PackageTaskId):
+        raise RuntimeError(
+            f"Benchmark suites address tasks by package name, but this dataset yielded a "
+            f"{type(task).__name__}. Point the suite at a published package dataset."
+        )
+    return task.name
+
+
 def validate_canonical_suite(
     suite: SuiteSpec,
     *,
@@ -376,7 +396,7 @@ async def _evaluate_heldout(
         environment_build_timeout_multiplier=2.0,
     )
     started = time.monotonic()
-    result = await HarborEvaluator(experiment_dir=run_dir).run(
+    result = await HarborNativeOutcomeEvaluator(experiment_dir=run_dir).run(
         agent=agent_dir,
         dataset=dataset,
         options=options,
@@ -467,11 +487,9 @@ async def run_benchmark(args: argparse.Namespace) -> Path:
         framework_skills_dirs=framework_skills_dirs,
         model_refs=optimizer_model_refs,
     )
-    run_document = json.loads((experimentalist_dir / "eval-and-optimize" / "run.json").read_text(encoding="utf-8"))
-    winner_label = run_document.get("winner_agent")
-    if not isinstance(winner_label, str) or not winner_label:
-        raise RuntimeError("Experimentalist completed without a selected winner")
-    winner_dir = experimentalist_dir / "eval-and-optimize" / "agents" / winner_label
+    winner_candidate = load_winner(experimentalist_dir / "eval-and-optimize")
+    winner_label = winner_candidate.label
+    winner_dir = local_path_from_uri(winner_candidate.artifact.uri, context="Winner artifact")
     winner = await _evaluate_heldout(
         label="winner",
         agent_dir=winner_dir,

@@ -0,0 +1,106 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""A strategy written outside the Experimentalist repository.
+
+Everything it uses is public: the entities, the `Strategy` role, and the context
+Protocol. It never imports a private module, and it is selected with
+`strategy: random-search` in the run config.
+
+The search itself is deliberately trivial. What it demonstrates is that a package
+installed beside the plugin can fill the strategy role, reach the platform only through
+the context, and land results in Studio like any other run.
+"""
+
+import random
+from pathlib import Path
+
+from nemo_experimentalist_plugin.config import EvolutionaryOptimizerConfig
+from nemo_experimentalist_plugin.entities import Candidate, Proposal
+from nemo_experimentalist_plugin.experimentalist.roles import Strategy
+from nemo_experimentalist_plugin.experimentalist.seam import StrategyContext
+
+#: Reuses the code-change Proposal kind, so the built-in Coder can build what this emits.
+CODE_CHANGE = "code-change"
+
+_IDEAS = [
+    "add a retry around the tool call that fails most often",
+    "split the single prompt into a plan step and an execute step",
+    "cache the lookup the agent repeats within one task",
+]
+
+
+def _validation_score(candidate: Candidate) -> float:
+    """Mean of whatever the evaluator measured on the validation channel.
+
+    Reads `metrics`, not `summary`: `summary` is an optional scalar rollup that nothing
+    currently writes, so ranking on it scores every candidate zero.
+    """
+    metrics = candidate.rewards["validation"].metrics
+    return sum(metrics.values()) / len(metrics) if metrics else 0.0
+
+
+class RandomSearch(Strategy):
+    """Propose a random change each round and keep whatever scores best."""
+
+    name = "random-search"
+    supports_resume = True
+
+    def __init__(
+        self,
+        working_dir: Path,
+        config: EvolutionaryOptimizerConfig | None = None,
+        framework_skills_dirs: list[Path] | None = None,
+        **_: object,
+    ) -> None:
+        """The arguments the runner constructs a strategy with; the rest are ignored.
+
+        Spelled out rather than left untyped because this package is the worked example a
+        third party copies: the signature is the contract, so it should show it.
+        """
+        del working_dir, framework_skills_dirs
+        settings = config or EvolutionaryOptimizerConfig()
+        self._rounds = settings.max_rounds
+        self._per_round = settings.max_candidates
+        self._builder = settings.builder
+
+    async def run(self, ctx: StrategyContext) -> Candidate | None:
+        """Import the agent, then build and score random variants of it."""
+        population = await ctx.candidates()
+        if not population:
+            population = [await self._import_baseline(ctx)]
+
+        rng = random.Random(0)
+        # Resume where the store left off. `supports_resume` is true, so `ctx.candidates()`
+        # can return work from an earlier attempt; restarting at 1 would rebuild it and
+        # report progress that goes backwards.
+        done = max((c.generation for c in population), default=0)
+        if done >= self._rounds:
+            return self._best(population)
+        for generation in range(done + 1, self._rounds + 1):
+            parent = rng.choice(population)
+            for _ in range(self._per_round):
+                proposal = Proposal(
+                    ancestor=parent.id,
+                    description=rng.choice(_IDEAS),
+                    kind=CODE_CHANGE,
+                    payload={"root_cause": "chosen at random", "optimization_type": "add_method", "task_ids": []},
+                )
+                builder = ctx.component("builder", self._builder)
+                candidate = await builder.build(ctx, proposal, generation=generation)
+                result = await ctx.evaluate(candidate)
+                await ctx.record_reward(candidate, channel="validation", result=result)
+                population.append(candidate)
+            await ctx.report_progress(completed=generation, total=self._rounds, unit="round")
+
+        return self._best(population)
+
+    @staticmethod
+    def _best(population: list[Candidate]) -> Candidate | None:
+        """Highest validation score, or None when the run produced nothing."""
+        return max(population, key=_validation_score) if population else None
+
+    async def _import_baseline(self, ctx: StrategyContext) -> Candidate:
+        """Commit the agent under test unchanged, using the built-in import Builder."""
+        proposal = Proposal(ancestor=None, description="the agent under test", kind="import", payload={})
+        return await ctx.component("builder", "import").build(ctx, proposal, generation=0)