diff --git a/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/agent.py b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/agent.py index a799618e6a..3d5e51a08d 100644 --- a/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/agent.py +++ b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/agent.py @@ -459,6 +459,13 @@ async def _run( total=len(trials), unit="trace", ) + + async def load_trace(reference: ResourceRef) -> TraceExplorer: + """Resolve a trace ref with this run's client, matching the Experimentalist's + `ctx.load_trace`: the analyzer takes a loader rather than a platform client, + so its signature names no platform type.""" + return await TraceExplorer.from_ref(reference, client, insight.workspace) + raw_diagnostics: list[Diagnostic | BaseException] = list( await asyncio.gather( *[ @@ -467,8 +474,7 @@ async def _run( task=task, agent_path=resolved_agent, insight=insight, - client=client, - workspace=insight.workspace, + load_trace=load_trace, ) for analyzer, trial, task in zip(analyzers, trials, tasks, strict=True) ], diff --git a/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/config.yaml b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/config.yaml index 08cc95dcee..559b5e62c7 100644 --- a/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/config.yaml +++ b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/config.yaml @@ -10,7 +10,7 @@ workspace: default base_url: http://localhost:8080 mode: local -evaluator_type: harbor_native +evaluator_type: harbor-native # Required per run. insight: "" diff --git a/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/run.py b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/run.py index bd4340fbb7..08303119ad 100644 --- a/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/run.py +++ b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/run.py @@ -51,7 +51,7 @@ async def run_eval_author( base_url: str | None, config: EvalAuthorConfig, agent: Path | str | None = None, - evaluator_type: EvaluatorType = "harbor_native", + evaluator_type: EvaluatorType = "harbor-native", model_refs: ConfiguredModelRefs | None = None, ) -> EvalAuthorResult: """Stage evaluation inputs, resolve one Insight, then run Eval Author. diff --git a/plugins/nemo-eval-author/src/nemo_eval_author_plugin/traces.py b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/traces.py index 69fd1d98d2..0e314d7e5e 100644 --- a/plugins/nemo-eval-author/src/nemo_eval_author_plugin/traces.py +++ b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/traces.py @@ -401,11 +401,17 @@ async def analyze_trace( trace=ResourceRef(uri=f"intake://{trace_id}", description="Production trace read from Intake."), metadata={"source": "intake"}, ) + + async def load_trace(reference: ResourceRef) -> TraceExplorer: + """Resolve a trace ref with this call's client, matching the Experimentalist's + `ctx.load_trace`: the analyzer takes a loader rather than a platform client, so + its signature names no platform type.""" + return await TraceExplorer.from_ref(reference, client, workspace) + return await TraceAnalyzer(experiment_dir=experiment_dir).run( trial=trial, task=task, agent_path=agent_path, insight=None, - client=client, - workspace=workspace, + load_trace=load_trace, ) diff --git a/plugins/nemo-eval-author/tests/test_eval_author_agent.py b/plugins/nemo-eval-author/tests/test_eval_author_agent.py index c4329c64c6..cb16b44b95 100644 --- a/plugins/nemo-eval-author/tests/test_eval_author_agent.py +++ b/plugins/nemo-eval-author/tests/test_eval_author_agent.py @@ -186,10 +186,9 @@ async def run( task: Task, agent_path: Path, insight: Insight, - client: Any, - workspace: str, + load_trace: Any, ) -> Diagnostic: - del task, agent_path, insight, client, workspace + del task, agent_path, insight, load_trace ref = cast(str, trial.metadata["trace_ref"]) calls.analyzed_refs.append(ref) outcome = outcomes[self.index] diff --git a/plugins/nemo-eval-author/tests/test_eval_author_repair_e2e.py b/plugins/nemo-eval-author/tests/test_eval_author_repair_e2e.py index 0e79e446c7..332e76a79a 100644 --- a/plugins/nemo-eval-author/tests/test_eval_author_repair_e2e.py +++ b/plugins/nemo-eval-author/tests/test_eval_author_repair_e2e.py @@ -15,12 +15,10 @@ from nemo_eval_author_plugin.eval_author.models import EvalAuthorConfig from nemo_experimentalist_plugin.client import make_client from nemo_experimentalist_plugin.entities import DatasetValidationError, local_path_from_uri -from nemo_experimentalist_plugin.experimentalist.components.evaluator.harbor import ( - HarborDataset, -) from nemo_experimentalist_plugin.experimentalist.components.evaluator.harbor_native import ( - HarborEvaluator, + HarborDataset, HarborEvaluatorConfig, + HarborNativeOutcomeEvaluator, ) from nemo_experimentalist_plugin.experimentalist.components.trace_analyzer import Diagnostic from nemo_insights_plugin.entities import Insight @@ -401,7 +399,7 @@ async def test_eval_author_metric_scores_known_failing_harbor_baseline_low( assert summary.summary await insight_suite.validate() - evaluator = HarborEvaluator(experiment_dir=tmp_path) + evaluator = HarborNativeOutcomeEvaluator(experiment_dir=tmp_path) result = await asyncio.wait_for( evaluator.run( agent=agent_dir, @@ -530,7 +528,7 @@ async def test_eval_author_metric_discriminates_controlled_harbor_tool_evidence( assert summary.summary await insight_suite.validate() - evaluator = HarborEvaluator(experiment_dir=tmp_path) + evaluator = HarborNativeOutcomeEvaluator(experiment_dir=tmp_path) async def run_agent(agent_dir: Path, job_name: str): return await asyncio.wait_for( diff --git a/plugins/nemo-eval-author/tests/test_eval_author_run.py b/plugins/nemo-eval-author/tests/test_eval_author_run.py index 3ec5cfa33c..f0d86c96d5 100644 --- a/plugins/nemo-eval-author/tests/test_eval_author_run.py +++ b/plugins/nemo-eval-author/tests/test_eval_author_run.py @@ -158,7 +158,7 @@ def missing_model_refs() -> eval_author_run.ConfiguredModelRefs: make_client.assert_not_called() -@pytest.mark.parametrize("evaluator_type", ["harbor_native", "harbor_evaluator"]) +@pytest.mark.parametrize("evaluator_type", ["harbor-native", "harbor-runner"]) @pytest.mark.asyncio async def test_run_eval_author_resolves_inputs_and_returns_datasets( monkeypatch: pytest.MonkeyPatch, @@ -315,7 +315,7 @@ async def resolve_model_clients(*_: object) -> ClosingModelClients: results: list[EvalAuthorResult] = [] calls: list[tuple[Insight, Path, Task, Dataset, Dataset, ClosingClient]] = [] - for evaluator_type in ("harbor_native", "harbor_evaluator"): + for evaluator_type in ("harbor-native", "harbor-runner"): results.append( await eval_author_run.run_eval_author( insight="insight-remote-123", @@ -354,7 +354,7 @@ async def resolve_model_clients(*_: object) -> ClosingModelClients: assert all(model_clients.closed for model_clients in model_client_sets) -@pytest.mark.parametrize("evaluator_type", ["harbor_native", "harbor_evaluator"]) +@pytest.mark.parametrize("evaluator_type", ["harbor-native", "harbor-runner"]) @pytest.mark.asyncio async def test_run_eval_author_hydrates_fileset_task_template( monkeypatch: pytest.MonkeyPatch, diff --git a/plugins/nemo-experimentalist/AGENTS.md b/plugins/nemo-experimentalist/AGENTS.md index 94d57f6d77..62522c968e 100644 --- a/plugins/nemo-experimentalist/AGENTS.md +++ b/plugins/nemo-experimentalist/AGENTS.md @@ -70,17 +70,17 @@ breaking rename with no compatibility aliases: Two names deliberately did **not** change. `optimizer.yaml` and the `.nemo-optimizer/` state directory are a shared contract with `nemo-insights-plugin`: `PROFILE_FILENAME` and `discover_profile()` live in -`nemo_insights_plugin.contracts.profile`, and `nemo agents analyst run` can -mirror the Platform rows it wrote into `/.nemo-optimizer/insights.yaml` -via `--insights-file-output`, which this plugin reads as the default insight when -the file exists. Rename them only in lockstep with a Platform change to that -contract. `EvolutionaryOptimizer` and `EvolutionaryOptimizerConfig` also keep -their names — they describe the optimization algorithm, not the product. - -At the time of this rename the command group was top-level (`nemo -experimentalist`), because the platform's `nemo.cli` entry-point group was flat -and nesting under `nemo agents` needed a Platform-side change first. That change -has since landed — see the `nemo agents` entry above for the current path. +`nemo_insights_plugin.contracts.profile`, and `nemo agents analyst run` writes +`/.nemo-optimizer/insights.yaml`, which this plugin reads as the +default insight. Rename them only in lockstep with a Platform change to that +contract. + +`EvolutionaryOptimizer` is now `EvolutionaryStrategy`, in +`experimentalist/strategies/evolutionary.py`, and is resolved by name like any other +component (`strategy: evolutionary`). + +Run `nemo agents experimentalist components` to see everything this install can resolve, +including components registered by a separately installed package. ### 2026-07-21: Curator renamed to Eval Author diff --git a/plugins/nemo-experimentalist/benchmarks/configs/tau3-quality.yaml b/plugins/nemo-experimentalist/benchmarks/configs/tau3-quality.yaml index 08d6d09aae..0461d875b5 100644 --- a/plugins/nemo-experimentalist/benchmarks/configs/tau3-quality.yaml +++ b/plugins/nemo-experimentalist/benchmarks/configs/tau3-quality.yaml @@ -14,9 +14,9 @@ optimizer: max_trajectory_tasks: 8 max_train_batch_tasks: 16 train_batch_seed: 20260727 - disable_trajectory_scoring: false - disable_convergence_check: false - evaluator: + trajectory_scorer: goal-tree + terminator: convergence + outcome_evaluator_config: n_attempts: 2 # Keep at 3 or lower: each task requests 8192 MB and parallel image builds # have triggered Docker Hub rate limiting and DNS failures. diff --git a/plugins/nemo-experimentalist/benchmarks/configs/tau3-smoke.yaml b/plugins/nemo-experimentalist/benchmarks/configs/tau3-smoke.yaml index 9d2359f909..ddb518059d 100644 --- a/plugins/nemo-experimentalist/benchmarks/configs/tau3-smoke.yaml +++ b/plugins/nemo-experimentalist/benchmarks/configs/tau3-smoke.yaml @@ -14,9 +14,9 @@ optimizer: max_trajectory_tasks: 2 max_train_batch_tasks: 4 train_batch_seed: 20260727 - disable_trajectory_scoring: true - disable_convergence_check: true - evaluator: + trajectory_scorer: null + terminator: null + outcome_evaluator_config: n_attempts: 1 n_concurrent_trials: 1 quiet: true diff --git a/plugins/nemo-experimentalist/benchmarks/configs/terminal-bench-quality.yaml b/plugins/nemo-experimentalist/benchmarks/configs/terminal-bench-quality.yaml index 08e1270887..5f4e3d577c 100644 --- a/plugins/nemo-experimentalist/benchmarks/configs/terminal-bench-quality.yaml +++ b/plugins/nemo-experimentalist/benchmarks/configs/terminal-bench-quality.yaml @@ -13,8 +13,8 @@ optimizer: max_trajectory_tasks: 6 max_train_batch_tasks: 12 train_batch_seed: 20260722 - disable_trajectory_scoring: true - evaluator: + trajectory_scorer: null + outcome_evaluator_config: n_attempts: 2 n_concurrent_trials: 2 quiet: true diff --git a/plugins/nemo-experimentalist/benchmarks/configs/terminal-bench-smoke.yaml b/plugins/nemo-experimentalist/benchmarks/configs/terminal-bench-smoke.yaml index 1dec23bd5c..942e051505 100644 --- a/plugins/nemo-experimentalist/benchmarks/configs/terminal-bench-smoke.yaml +++ b/plugins/nemo-experimentalist/benchmarks/configs/terminal-bench-smoke.yaml @@ -13,9 +13,9 @@ optimizer: max_trajectory_tasks: 2 max_train_batch_tasks: 4 train_batch_seed: 20260722 - disable_trajectory_scoring: true - disable_convergence_check: true - evaluator: + trajectory_scorer: null + terminator: null + outcome_evaluator_config: n_attempts: 1 n_concurrent_trials: 1 quiet: true diff --git a/plugins/nemo-experimentalist/benchmarks/run.py b/plugins/nemo-experimentalist/benchmarks/run.py index 27362ea9a7..10653a45ff 100644 --- a/plugins/nemo-experimentalist/benchmarks/run.py +++ b/plugins/nemo-experimentalist/benchmarks/run.py @@ -14,6 +14,7 @@ from typing import Any, Literal, Self import yaml +from harbor.models.task.id import GitTaskId, LocalTaskId, PackageTaskId from harbor.registry.client.package import PackageDatasetClient from nemo_experimentalist_plugin.config import EvolutionaryOptimizerConfig from nemo_experimentalist_plugin.entities import ( @@ -24,9 +25,10 @@ ) from nemo_experimentalist_plugin.experimentalist.components.evaluator.harbor import HarborDataset from nemo_experimentalist_plugin.experimentalist.components.evaluator.harbor_native import ( - HarborEvaluator, HarborEvaluatorConfig, + HarborNativeOutcomeEvaluator, ) +from nemo_experimentalist_plugin.experimentalist.experimentalist_backend import load_winner from nemo_experimentalist_plugin.resolve import resolve_dataset from pydantic import BaseModel, Field, model_validator @@ -151,6 +153,24 @@ def load_benchmark_config(path: Path) -> BenchmarkConfig: return BenchmarkConfig.model_validate(_load_yaml(path)) +def _canonical_task_id(task: GitTaskId | LocalTaskId | PackageTaskId) -> str: + """The suite-facing id of one Harbor task. + + Harbor's task ids are a union, and only ``PackageTaskId`` carries a ``name``. The + others do have a ``get_name()``, but it means something different — ``hello-world`` + for a git or local task versus ``org/hello-world`` for a package one — so there is no + accessor that yields the same string across the union. Benchmark suites are pinned to + a published package, so anything else is a suite that was authored wrong, and saying + so beats silently substituting a value. + """ + if not isinstance(task, PackageTaskId): + raise RuntimeError( + f"Benchmark suites address tasks by package name, but this dataset yielded a " + f"{type(task).__name__}. Point the suite at a published package dataset." + ) + return task.name + + def validate_canonical_suite( suite: SuiteSpec, *, @@ -376,7 +396,7 @@ async def _evaluate_heldout( environment_build_timeout_multiplier=2.0, ) started = time.monotonic() - result = await HarborEvaluator(experiment_dir=run_dir).run( + result = await HarborNativeOutcomeEvaluator(experiment_dir=run_dir).run( agent=agent_dir, dataset=dataset, options=options, @@ -467,11 +487,9 @@ async def run_benchmark(args: argparse.Namespace) -> Path: framework_skills_dirs=framework_skills_dirs, model_refs=optimizer_model_refs, ) - run_document = json.loads((experimentalist_dir / "eval-and-optimize" / "run.json").read_text(encoding="utf-8")) - winner_label = run_document.get("winner_agent") - if not isinstance(winner_label, str) or not winner_label: - raise RuntimeError("Experimentalist completed without a selected winner") - winner_dir = experimentalist_dir / "eval-and-optimize" / "agents" / winner_label + winner_candidate = load_winner(experimentalist_dir / "eval-and-optimize") + winner_label = winner_candidate.label + winner_dir = local_path_from_uri(winner_candidate.artifact.uri, context="Winner artifact") winner = await _evaluate_heldout( label="winner", agent_dir=winner_dir, diff --git a/plugins/nemo-experimentalist/examples/acme-strategies/acme_strategies/random_search.py b/plugins/nemo-experimentalist/examples/acme-strategies/acme_strategies/random_search.py new file mode 100644 index 0000000000..ec6462c78e --- /dev/null +++ b/plugins/nemo-experimentalist/examples/acme-strategies/acme_strategies/random_search.py @@ -0,0 +1,106 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""A strategy written outside the Experimentalist repository. + +Everything it uses is public: the entities, the `Strategy` role, and the context +Protocol. It never imports a private module, and it is selected with +`strategy: random-search` in the run config. + +The search itself is deliberately trivial. What it demonstrates is that a package +installed beside the plugin can fill the strategy role, reach the platform only through +the context, and land results in Studio like any other run. +""" + +import random +from pathlib import Path + +from nemo_experimentalist_plugin.config import EvolutionaryOptimizerConfig +from nemo_experimentalist_plugin.entities import Candidate, Proposal +from nemo_experimentalist_plugin.experimentalist.roles import Strategy +from nemo_experimentalist_plugin.experimentalist.seam import StrategyContext + +#: Reuses the code-change Proposal kind, so the built-in Coder can build what this emits. +CODE_CHANGE = "code-change" + +_IDEAS = [ + "add a retry around the tool call that fails most often", + "split the single prompt into a plan step and an execute step", + "cache the lookup the agent repeats within one task", +] + + +def _validation_score(candidate: Candidate) -> float: + """Mean of whatever the evaluator measured on the validation channel. + + Reads `metrics`, not `summary`: `summary` is an optional scalar rollup that nothing + currently writes, so ranking on it scores every candidate zero. + """ + metrics = candidate.rewards["validation"].metrics + return sum(metrics.values()) / len(metrics) if metrics else 0.0 + + +class RandomSearch(Strategy): + """Propose a random change each round and keep whatever scores best.""" + + name = "random-search" + supports_resume = True + + def __init__( + self, + working_dir: Path, + config: EvolutionaryOptimizerConfig | None = None, + framework_skills_dirs: list[Path] | None = None, + **_: object, + ) -> None: + """The arguments the runner constructs a strategy with; the rest are ignored. + + Spelled out rather than left untyped because this package is the worked example a + third party copies: the signature is the contract, so it should show it. + """ + del working_dir, framework_skills_dirs + settings = config or EvolutionaryOptimizerConfig() + self._rounds = settings.max_rounds + self._per_round = settings.max_candidates + self._builder = settings.builder + + async def run(self, ctx: StrategyContext) -> Candidate | None: + """Import the agent, then build and score random variants of it.""" + population = await ctx.candidates() + if not population: + population = [await self._import_baseline(ctx)] + + rng = random.Random(0) + # Resume where the store left off. `supports_resume` is true, so `ctx.candidates()` + # can return work from an earlier attempt; restarting at 1 would rebuild it and + # report progress that goes backwards. + done = max((c.generation for c in population), default=0) + if done >= self._rounds: + return self._best(population) + for generation in range(done + 1, self._rounds + 1): + parent = rng.choice(population) + for _ in range(self._per_round): + proposal = Proposal( + ancestor=parent.id, + description=rng.choice(_IDEAS), + kind=CODE_CHANGE, + payload={"root_cause": "chosen at random", "optimization_type": "add_method", "task_ids": []}, + ) + builder = ctx.component("builder", self._builder) + candidate = await builder.build(ctx, proposal, generation=generation) + result = await ctx.evaluate(candidate) + await ctx.record_reward(candidate, channel="validation", result=result) + population.append(candidate) + await ctx.report_progress(completed=generation, total=self._rounds, unit="round") + + return self._best(population) + + @staticmethod + def _best(population: list[Candidate]) -> Candidate | None: + """Highest validation score, or None when the run produced nothing.""" + return max(population, key=_validation_score) if population else None + + async def _import_baseline(self, ctx: StrategyContext) -> Candidate: + """Commit the agent under test unchanged, using the built-in import Builder.""" + proposal = Proposal(ancestor=None, description="the agent under test", kind="import", payload={}) + return await ctx.component("builder", "import").build(ctx, proposal, generation=0) diff --git a/plugins/nemo-experimentalist/examples/acme-strategies/pyproject.toml b/plugins/nemo-experimentalist/examples/acme-strategies/pyproject.toml new file mode 100644 index 0000000000..22b5f70a48 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/acme-strategies/pyproject.toml @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# A component package written the way someone outside this repository would write one. +# It depends on the plugin, not on the monorepo, and ships one entry point. +[project] +name = "acme-strategies" +version = "0.1.0" +description = "An out-of-tree Experimentalist strategy, used to prove the plugin mechanism." +requires-python = ">=3.12,<3.14" +dependencies = ["nemo-experimentalist-plugin"] + +# The entry point's only job is to import the module, so the class registers itself. +[project.entry-points."nemo.experimentalist.components"] +"strategy.random-search" = "acme_strategies.random_search" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["acme_strategies"] diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/configs/full.yaml b/plugins/nemo-experimentalist/examples/smoke-agent/configs/full.yaml index 308ec32876..9c1f531a68 100644 --- a/plugins/nemo-experimentalist/examples/smoke-agent/configs/full.yaml +++ b/plugins/nemo-experimentalist/examples/smoke-agent/configs/full.yaml @@ -80,12 +80,13 @@ max_train_batch_tasks: null # scenario. Multiple rounds, survivors, ranking, and the convergence check are. # Turning it on is worthwhile once it is dependable -- build_all_group.py holds # g5 out of the combined set for the same reason. -disable_trajectory_scoring: true +trajectory_scorer: null # The one that matters here: the terminator deciding when to stop is the point. -disable_convergence_check: false +# The terminator deciding when to stop is the point of this scenario, so it keeps +# its default ('convergence') rather than being turned off. -evaluator: +outcome_evaluator_config: n_attempts: 1 n_concurrent_trials: 5 quiet: true diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/configs/short.yaml b/plugins/nemo-experimentalist/examples/smoke-agent/configs/short.yaml index c614a994c3..b55362393f 100644 --- a/plugins/nemo-experimentalist/examples/smoke-agent/configs/short.yaml +++ b/plugins/nemo-experimentalist/examples/smoke-agent/configs/short.yaml @@ -60,7 +60,7 @@ objective_function: direction: maximize target: 1.0 -disable_trajectory_scoring: true +trajectory_scorer: null # Off, and it should stay off at this depth. The terminator only tests for # *stagnation* -- whether any new candidate reached the Pareto front, then an LLM @@ -82,9 +82,9 @@ disable_trajectory_scoring: true # # `full.yaml` is where the terminator earns its place: five rounds, three # candidates, and stopping when the front genuinely stops moving. -disable_convergence_check: true +terminator: null -evaluator: +outcome_evaluator_config: n_attempts: 1 n_concurrent_trials: 3 quiet: true diff --git a/plugins/nemo-experimentalist/examples/smoke-agent/scripts/record_traces.py b/plugins/nemo-experimentalist/examples/smoke-agent/scripts/record_traces.py index 018e014d60..355d453fe7 100644 --- a/plugins/nemo-experimentalist/examples/smoke-agent/scripts/record_traces.py +++ b/plugins/nemo-experimentalist/examples/smoke-agent/scripts/record_traces.py @@ -20,10 +20,10 @@ from nemo_experimentalist_plugin.client import make_client from nemo_experimentalist_plugin.entities import TrialResult, local_path_from_uri -from nemo_experimentalist_plugin.experimentalist.components.evaluator.harbor import ( +from nemo_experimentalist_plugin.experimentalist.components.evaluator.harbor_native import ( HarborDataset, - HarborEvaluator, HarborEvaluatorConfig, + HarborNativeOutcomeEvaluator, ) from nemo_experimentalist_plugin.experimentalist.otlp import jsonl_to_protobuf, read_trace_id from nemo_platform import AsyncNeMoPlatform, NotFoundError @@ -121,7 +121,7 @@ async def run(args: argparse.Namespace) -> dict[str, str]: n_concurrent_trials=args.concurrency, quiet=True, ) - result = await HarborEvaluator(experiment_dir=run_dir).run( + result = await HarborNativeOutcomeEvaluator(experiment_dir=run_dir).run( agent=agent_path, dataset=dataset, options=options, diff --git a/plugins/nemo-experimentalist/examples/tau3-nooa-agent/experimentalist-smoke.yaml b/plugins/nemo-experimentalist/examples/tau3-nooa-agent/experimentalist-smoke.yaml index c4b00ce3fc..4b6c043500 100644 --- a/plugins/nemo-experimentalist/examples/tau3-nooa-agent/experimentalist-smoke.yaml +++ b/plugins/nemo-experimentalist/examples/tau3-nooa-agent/experimentalist-smoke.yaml @@ -7,11 +7,11 @@ max_survivors: 1 max_candidates: 1 max_trajectory_tasks: 2 train_batch_seed: 20260727 -disable_trajectory_scoring: true -disable_convergence_check: true +trajectory_scorer: null +terminator: null storage: archive_candidates: true -evaluator: +outcome_evaluator_config: n_attempts: 1 quiet: true agent_setup_timeout_multiplier: 2.0 diff --git a/plugins/nemo-experimentalist/examples/tau3-nooa-agent/record_tau_airline_traces.py b/plugins/nemo-experimentalist/examples/tau3-nooa-agent/record_tau_airline_traces.py index 9558a74b9e..e153b2f8fe 100644 --- a/plugins/nemo-experimentalist/examples/tau3-nooa-agent/record_tau_airline_traces.py +++ b/plugins/nemo-experimentalist/examples/tau3-nooa-agent/record_tau_airline_traces.py @@ -20,8 +20,8 @@ from nemo_experimentalist_plugin.entities import TrialResult, local_path_from_uri from nemo_experimentalist_plugin.experimentalist.components.evaluator.harbor import HarborDataset from nemo_experimentalist_plugin.experimentalist.components.evaluator.harbor_native import ( - HarborEvaluator, HarborEvaluatorConfig, + HarborNativeOutcomeEvaluator, ) from nemo_experimentalist_plugin.experimentalist.otlp import jsonl_to_protobuf, read_trace_id from nemo_platform import AsyncNeMoPlatform, NotFoundError @@ -214,7 +214,7 @@ async def run(args: argparse.Namespace) -> Path: if args.upload_dir is not None: job_dir, run_dir = _resolve_harbor_output_dir(args.upload_dir) - evaluator = HarborEvaluator(experiment_dir=run_dir) + evaluator = HarborNativeOutcomeEvaluator(experiment_dir=run_dir) trials = list(await evaluator._trials_from_dir(job_dir, dataset.tasks)) uploadable_trials = [trial for trial in trials if trial.status == "completed" and trial.trace is not None] if not uploadable_trials: @@ -279,7 +279,7 @@ async def run(args: argparse.Namespace) -> Path: agent_setup_timeout_multiplier=2.0, environment_build_timeout_multiplier=3.0, ) - result = await HarborEvaluator(experiment_dir=run_dir).run( + result = await HarborNativeOutcomeEvaluator(experiment_dir=run_dir).run( agent=agent_path, dataset=dataset, options=options, diff --git a/plugins/nemo-experimentalist/pyproject.toml b/plugins/nemo-experimentalist/pyproject.toml index 6e8b622a6d..ba16ec7552 100644 --- a/plugins/nemo-experimentalist/pyproject.toml +++ b/plugins/nemo-experimentalist/pyproject.toml @@ -28,6 +28,24 @@ experimentalist = "nemo_experimentalist_plugin.cli:ExperimentalistCLI" [project.entry-points."nemo.skills"] experimentalist = "nemo_experimentalist_plugin.skills:skills_dir" +# Our own components are discovered exactly the way a third party's are: one entry point +# per module, whose only job is to import it so its classes self-register. There are no +# privileged built-ins — if this mechanism breaks, our loop stops resolving too, which is +# the point. +# +# One entry per module, not per component: importing the module registers every class in it. +[project.entry-points."nemo.experimentalist.components"] +"strategy.evolutionary" = "nemo_experimentalist_plugin.experimentalist.strategies.evolutionary" +"builder.code-edit" = "nemo_experimentalist_plugin.experimentalist.components.coder" +"builder.import" = "nemo_experimentalist_plugin.experimentalist.components.importer" +"selector.pareto-diversity" = "nemo_experimentalist_plugin.experimentalist.components.selector" +"proposer.code-change" = "nemo_experimentalist_plugin.experimentalist.components.proposer" +"terminator.convergence" = "nemo_experimentalist_plugin.experimentalist.components.terminator" +"root-cause-analyzer.trace" = "nemo_experimentalist_plugin.experimentalist.components.analyzer" +"trajectory-scorer.goal-tree" = "nemo_experimentalist_plugin.experimentalist.components.trace_scorer" +"outcome-evaluator.harbor-native" = "nemo_experimentalist_plugin.experimentalist.components.evaluator.harbor_native" +"outcome-evaluator.harbor-runner" = "nemo_experimentalist_plugin.experimentalist.components.evaluator.harbor_evaluator" + [build-system] requires = ["hatchling"] build-backend = "hatchling.build" @@ -37,7 +55,7 @@ packages = ["src/nemo_experimentalist_plugin"] [tool.pytest.ini_options] asyncio_mode = "auto" -pythonpath = ["src"] +pythonpath = ["src", "tests"] testpaths = ["tests"] markers = [ "e2e: executes model-written shell and requires a sandbox, Docker, a running Platform, and configured models", diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/cli.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/cli.py index ba4779cca6..509745d3ea 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/cli.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/cli.py @@ -270,7 +270,7 @@ async def _flow() -> str: task_template=plan.task_template, agent_source=plan.agent, storage=plan.config.storage.model_dump(exclude_unset=True), - evaluator=plan.config.evaluator, + evaluator=plan.config.outcome_evaluator_config, require_template=plan.insight is not None, probes=_PREFLIGHT_PROBES, ) @@ -317,6 +317,29 @@ async def _flow() -> str: raise typer.Exit(code=1) from None typer.echo(output_text) + @app.command("components") + def components( + role: str | None = typer.Option(None, "--role", help="Show only this role."), + ) -> None: + """List the components this install can resolve by name. + + Includes anything a `pip install`ed package registered, which is how a + developer checks their own component was picked up. + """ + from nemo_experimentalist_plugin.experimentalist.registry import Component, load_plugins + + load_plugins() + rows = sorted(Component._registry.items()) + if role is not None: + rows = [row for row in rows if row[0][0] == role] + if not rows: + typer.echo(f"No components registered{f' for role {role!r}' if role else ''}.") + raise typer.Exit(1) + width = max(len(registered_role) for (registered_role, _), _ in rows) + for (registered_role, registered_name), component in rows: + where = f"{component.__module__}.{component.__qualname__}" + typer.echo(f"{registered_role:<{width}} {registered_name:<24} {where}") + @app.command("doctor") def doctor( insight: str | None = typer.Option(None, "--insight", help="Optional insight ref to verify."), @@ -414,7 +437,7 @@ def doctor( task_template=plan.task_template, agent_source=plan.agent, storage=plan.config.storage.model_dump(), - evaluator=plan.config.evaluator, + evaluator=plan.config.outcome_evaluator_config, require_template=plan.insight is not None, probes=_PREFLIGHT_PROBES, ) diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/config.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/config.py index b6c4f93c40..886246955f 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/config.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/config.py @@ -12,21 +12,19 @@ The optimizer's own default/fast model pair is selected by ``nemo setup`` and stored in the active Platform CLI context. -Component-owned slices (``CoderConfig``, ``AnalyzerConfig``, ...) are imported from the -components that consume them rather than redeclared here -- ``resolve.py`` used to carry a -second copy of each because importing a component module required credentials, which it no -longer does. +Component-owned slices (``CodeEditBuilderConfig``, ``AnalyzerConfig``, ...) are imported from the +components that consume them rather than redeclared here. """ from pathlib import Path -from typing import Any, Literal, Self +from typing import Any, Self from nemo_eval_author_plugin.eval_author.models import EvalAuthorConfig -from nemo_experimentalist_plugin.experimentalist.components.analyzer import AnalyzerConfig -from nemo_experimentalist_plugin.experimentalist.components.coder import CoderConfig -from nemo_experimentalist_plugin.experimentalist.components.evaluator.base import EvaluatorTypeField -from nemo_experimentalist_plugin.experimentalist.components.goal_tree import GoalTreeConfig -from nemo_experimentalist_plugin.experimentalist.components.proposer import ProposerConfig +from nemo_experimentalist_plugin.entities import MetricTarget +from nemo_experimentalist_plugin.experimentalist.components.models import ( # noqa: F401 - re-exported + has_metric_dimensions, + pareto_objectives, +) from pydantic import BaseModel, Field, model_validator @@ -51,55 +49,6 @@ class CandidateStorageConfig(BaseModel): pr_labels: list[str] = Field(default_factory=list) -class MetricTarget(BaseModel): - """One evaluator-produced metric and the desired direction of change.""" - - name: str = Field(min_length=1, description="Exact metric name emitted by the evaluator.") - direction: Literal["maximize", "minimize"] = Field( - description="Whether higher or lower values are better for this target." - ) - target: float | None = Field( - default=None, - description=( - "Value at which this objective counts as satisfied, in the metric's own " - "units. When every targeted objective is met the run stops, so a solved " - "problem stops paying for rounds. Unset means no such stop: metrics are not " - "required to be normalized, so there is no value that means 'as good as " - "possible' for an arbitrary one." - ), - ) - - def is_satisfied_by(self, value: float | None) -> bool: - """Whether *value* meets this target. False when either side is absent. - - A missing measurement is not evidence of success, and a target that was never - configured must not end a run. - """ - if value is None or self.target is None: - return False - return value >= self.target if self.direction == "maximize" else value <= self.target - - -def pareto_objectives(metrics: dict[str, float], objective_function: list[MetricTarget]) -> dict[str, float]: - """Project evaluator metrics onto the configured objectives for Pareto ranking. - - The generic Pareto utility maximizes every dimension. Minimized objective - values are sign-inverted here; regression metrics are intentionally absent. - """ - objectives: dict[str, float] = {} - for target in objective_function: - value = metrics.get(target.name) - if value is None: - return {} - objectives[target.name] = float(value) if target.direction == "maximize" else -float(value) - return objectives - - -def has_metric_dimensions(metrics: dict[str, float], targets: list[MetricTarget]) -> bool: - """Return whether an evaluator result contains every required metric target.""" - return all(target.name in metrics for target in targets) - - class EvolutionaryOptimizerConfig(BaseModel): """Parameters for one optimizer run, read from ``--config`` and nothing else. @@ -107,6 +56,9 @@ class EvolutionaryOptimizerConfig(BaseModel): named explicitly on the command line, and are recorded in ``config_snapshot`` as the account of what ran. Letting an ambient environment variable override them would make that account wrong. Agent model settings live in the active Platform CLI context. + + Unknown keys are tolerated, so a key that was *removed* has to be rejected explicitly + below — silently ignoring one would change what the run does. """ @model_validator(mode="before") @@ -116,12 +68,105 @@ def reject_legacy_curator_config(cls, data: Any) -> Any: return data if "curator" in data: raise ValueError("'curator' was renamed to 'eval_author'; update the optimizer configuration") + + # A step is turned off by choosing no implementation of its role. + for removed, replacement in ( + ("disable_convergence_check", "terminator"), + ("disable_trajectory_scoring", "trajectory_scorer"), + ): + if removed in data: + raise ValueError( + f"{removed!r} is no longer a run-config key; write '{replacement}: null' instead. " + "Turning a step off is how you choose no implementation of its role." + ) + + # The role key now names a component, so its tuning moved under '_config'. + # A string is a component name and stays valid; only a config block is legacy. + for removed, replacement in (("analyzer", "analyzer_config"), ("proposer", "proposer_config")): + if isinstance(data.get(removed), dict): + raise ValueError( + f"{removed!r} now names the component to run, so its settings moved to {replacement!r}." + ) + + # Renamed outright. These are not fields at all, so any value here is legacy and + # would otherwise be dropped in silence — changing what the run does. + for removed, role, config_key in ( + ("evaluator", "outcome_evaluator", "outcome_evaluator_config"), + ("evaluation", "outcome_evaluator", "outcome_evaluator_config"), + ("evaluation_config", "outcome_evaluator", "outcome_evaluator_config"), + ("coder", "builder", "builder_config"), + ("goal_config", "trajectory_scorer", "trajectory_scorer_config"), + ): + if removed in data: + raise ValueError( + f"{removed!r} is no longer a run-config key; the role is {role!r} and its settings " + f"belong under {config_key!r}." + ) if "models" in data: raise ValueError( "'models' is no longer a run-config key. Run `nemo setup` to select the default and fast agent models." ) + + # `harbor` shipped as an evaluator_type before the evaluator was split into two + # implementations, so configs in the wild carry it. + if data.get("outcome_evaluator") == "harbor": + raise ValueError( + "outcome_evaluator: 'harbor' was split into 'harbor-native' (the default) and " + "'harbor-runner', which drives Harbor through the NeMo Evaluator SDK." + ) return data + strategy: str = Field( + default="evolutionary", + description=( + "Registered 'strategy' component the runner runs. Ours is resolved by name " + "like any other, so a strategy shipped by another package is selected here " + "with no code change." + ), + ) + # Every step the strategy delegates to is named here, so swapping one is configuration. + # A null means "no such step": turning a step off is the degenerate case of choosing a + # different implementation, which is why there are no disable_* booleans. + analyzer: str | None = Field( + default="trace", + description="Registered 'root-cause-analyzer'. Null skips diagnosis and the train eval feeding it.", + ) + outcome_evaluator: str = Field( + default="harbor-native", + description=( + "Registered 'outcome-evaluator' measuring what a candidate achieved. Named for " + "the outcome because the trajectory-scorer measures the process of the same run." + ), + ) + proposer: str = Field(default="code-change", description="Registered 'proposer' emitting each round's Proposals.") + terminator: str | None = Field( + default="convergence", + description="Registered 'terminator'. Null stops only on max_rounds.", + ) + trajectory_scorer: str | None = Field( + default="goal-tree", + description="Registered 'trajectory-scorer'. Null skips step scoring and the goal tree it needs.", + ) + selector: str = Field( + default="pareto-diversity", + description="Registered 'selector' component choosing survivors and the winner.", + ) + selector_config: dict[str, Any] = Field( + default_factory=dict, + description="Settings for the selector named above, validated against *that component's* `config_type` when it is built — so a component from another package is configurable without this schema knowing its fields.", + ) + terminator_config: dict[str, Any] = Field( + default_factory=dict, + description="Settings for the terminator named above, validated against *that component's* `config_type` when it is built — so a component from another package is configurable without this schema knowing its fields.", + ) + builder: str = Field( + default="code-edit", + description=( + "Registered 'builder' component that turns a Proposal into a Candidate. " + "Swap it for one shipped by another package to change how candidates are " + "built without touching the loop." + ), + ) max_rounds: int = Field(default=15, description="Hard ceiling on optimization rounds.") min_rounds_before_stopping: int = Field( default=3, description="Rounds that must complete before the convergence check may stop the run." @@ -137,12 +182,7 @@ def reject_legacy_curator_config(cls, data: Any) -> Any: model_catalog_path: Path | None = Field( default=None, description="Model catalog overriding the packaged assets/models.yaml." ) - disable_trajectory_scoring: bool = Field( - default=False, description="Skip goal-tree trajectory scoring and the goal tree it needs." - ) - disable_convergence_check: bool = Field( - default=False, description="Stop only on max_rounds, never on the terminator's convergence judgement." - ) + objective_function: list[MetricTarget] = Field( default_factory=lambda: [MetricTarget(name="reward", direction="maximize")], min_length=1, @@ -154,12 +194,26 @@ def reject_legacy_curator_config(cls, data: Any) -> Any: ) source: AgentSourceConfig = Field(default_factory=AgentSourceConfig) storage: CandidateStorageConfig = Field(default_factory=CandidateStorageConfig) - goal_config: GoalTreeConfig = Field(default_factory=GoalTreeConfig) - coder: CoderConfig = Field(default_factory=CoderConfig) - analyzer: AnalyzerConfig = Field(default_factory=AnalyzerConfig) - proposer: ProposerConfig = Field(default_factory=ProposerConfig) - evaluator_type: EvaluatorTypeField = "harbor_native" - evaluator: dict[str, Any] = Field(default_factory=dict) + trajectory_scorer_config: dict[str, Any] = Field( + default_factory=dict, + description="Settings for the trajectory-scorer named above, validated against *that component's* `config_type` when it is built — so a component from another package is configurable without this schema knowing its fields.", + ) + builder_config: dict[str, Any] = Field( + default_factory=dict, + description="Settings for the builder named above, validated against *that component's* `config_type` when it is built — so a component from another package is configurable without this schema knowing its fields.", + ) + analyzer_config: dict[str, Any] = Field( + default_factory=dict, + description="Settings for the root-cause-analyzer named above, validated against *that component's* `config_type` when it is built — so a component from another package is configurable without this schema knowing its fields.", + ) + proposer_config: dict[str, Any] = Field( + default_factory=dict, + description="Settings for the proposer named above, validated against *that component's* `config_type` when it is built — so a component from another package is configurable without this schema knowing its fields.", + ) + outcome_evaluator_config: dict[str, Any] = Field( + default_factory=dict, + description="Config for the selected 'evaluation' component; its own model validates it.", + ) eval_author: EvalAuthorConfig = Field(default_factory=EvalAuthorConfig) @model_validator(mode="after") @@ -190,3 +244,25 @@ def render(target: MetricTarget) -> str: f"Do not regress these metric(s): {regressions}. " "Metric values, including aggregates, are produced by the evaluator; do not invent formulas or weights." ) + + +def with_insight_objective( + config: "EvolutionaryOptimizerConfig", metric_keys: tuple[str, ...] +) -> "EvolutionaryOptimizerConfig": + """Make insight metrics objectives and preserve all configured targets as guardrails.""" + if not metric_keys: + return config + insight_metric_names = set(metric_keys) + objective = [MetricTarget(name=metric_key, direction="maximize") for metric_key in metric_keys] + regression_by_name = { + target.name: target + for target in [*config.objective_function, *config.regression_metrics] + if target.name not in insight_metric_names + } + return EvolutionaryOptimizerConfig.model_validate( + config.model_dump(mode="python") + | { + "objective_function": [target.model_dump(mode="python") for target in objective], + "regression_metrics": [target.model_dump(mode="python") for target in regression_by_name.values()], + } + ) diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/entities.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/entities.py index f3f5075878..634de2f57b 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/entities.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/entities.py @@ -23,7 +23,7 @@ from urllib.parse import unquote, urlparse from nemo_platform_plugin.entity import NemoEntity -from pydantic import BaseModel, ConfigDict, Field, SerializeAsAny, model_validator +from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, SerializeAsAny, model_validator DataValue: TypeAlias = str | int | float | bool | dict[str, Any] | list[Any] | None MetricValue: TypeAlias = float | int @@ -386,15 +386,32 @@ class ExperimentRun(NemoEntity, entity_type="experiment_run"): default="running", description="Lifecycle status of this optimization run.", ) - rounds_completed: int = Field( + progress_completed: int = Field( default=0, - description="Number of full optimization rounds completed so far.", + description="Units of work the strategy reports finished so far. Display only.", + ) + progress_total: int | None = Field( + default=None, + description=( + "Units of work expected in total, when the strategy can say. None means it " + "cannot — an opaque strategy has no honest denominator, so consumers show a " + "counter rather than a bar." + ), + ) + progress_unit: str = Field( + default="step", + description="What one unit of progress is: 'round' for the evolutionary loop, 'trial' for a search.", + ) + progress_note: str | None = Field( + default=None, + description="What the strategy is currently doing, for strategies with no meaningful total.", ) winner_agent: str | None = Field( default=None, description=( - "Entity id of the winning Candidate, or a local filesystem path " - "to the agent directory when running offline; set on completion." + "Label of the winning Candidate — 'agent-3', not its id. Every artifact " + "path in a run is built from the label, so this is the form a reader can " + "resolve; set on completion." ), ) summary: str | None = Field( @@ -413,7 +430,6 @@ def _restore_id_from_json(cls, data: Any, handler: Any) -> "ExperimentRun": Without this, resumed runs lose their identity and projection names collapse. """ if isinstance(data, dict) and "id" in data: - # Pydantic wraps the dict during validation, extract the raw data instance = handler(data) instance._id = data["id"] # type: ignore[attr-defined] return instance @@ -437,70 +453,158 @@ class RewardRecord(BaseModel): metadata: dict[str, DataValue] = Field(default_factory=dict, description="Provenance for this measurement.") +class RewardMap(dict[str, RewardRecord]): + """A candidate's measurements, keyed by reward channel. + + One mapping answers both questions: ``rewards[channel]`` always yields a record, so + ``rewards["train"].metrics`` needs no presence check, while ``channel in rewards`` + answers *was this measured at all* — which is what gates whether to evaluate. + + ``__missing__`` **returns** without inserting, which is the whole point and why this + is not a ``defaultdict``: that one's ``__missing__`` inserts, so merely reading a + channel would mark it measured, skip its evaluation, and persist a phantom record. + """ + + def __missing__(self, channel: str) -> RewardRecord: + return RewardRecord() + + def __setitem__(self, channel: str, record: RewardRecord) -> None: + """Refuse a direct write: it mutates memory and is never persisted. + + A measurement reaches the store through ``ctx.record_reward``, which also + persists the evaluation's traces and updates the candidate. Assigning here + instead leaves a candidate that looks measured until the next reload. + """ + raise TypeError( + f"cannot set rewards[{channel!r}] directly; record a measurement with " + "ctx.record_reward(candidate, channel=..., result=...) so it is persisted" + ) + + @classmethod + def __get_pydantic_core_schema__(cls, source_type: Any, handler: Any) -> Any: + """Validate as a plain channel map, then re-wrap so ``__missing__`` survives. + + Pydantic rejects a bare ``dict`` subclass, and validating into one would hand + back a plain ``dict`` that has lost the behaviour this class exists for. + """ + from pydantic_core import core_schema + + return core_schema.no_info_after_validator_function(cls, handler.generate_schema(dict[str, RewardRecord])) + + +class Proposal(BaseModel): + """A request to build one candidate — the Proposer → Builder contract. + + A transient component message, not a separately persisted entity and not an + unfinished Candidate: a proposal describes work to perform, and a Candidate is the + durable result after a Builder completes it. Failed proposals produce no Candidate + and are not retained; a successful one is embedded in ``Candidate.generated_from``. + + ``kind`` is an opaque compatibility discriminator, not a global enumeration: it + routes the proposal to a Builder that declares it can accept it. ``payload`` is + owned by that Proposer/Builder pair — Layer A stores and transports it and + interprets neither. + """ + + ancestor: str | None = Field( + default=None, + description="Parent Candidate id to build from. None means the baseline.", + ) + description: str = Field( + min_length=1, + description="Human-readable explanation of the proposed variant.", + ) + kind: str = Field( + min_length=1, + description="Builder compatibility discriminator, e.g. 'code-change' or 'parameters'.", + ) + payload: dict[str, DataValue] = Field( + default_factory=dict, + description="Build instructions, validated by the component-owned schema for this kind.", + ) + + class Candidate(NemoEntity, entity_type="candidate"): """A candidate agent version produced during an Experimentalist run. - Lives in the entity store so candidates are queryable, resumable, and - survive the local working directory being deleted. ``run_id`` groups all - candidates for a single ExperimentRun. + Metadata and measurements live in the entity store; the completed work is + *addressed*, not contained. ``artifact`` points at the resource that defines this + candidate and, when external evaluation is used, is directly consumable by the + run's evaluation component. Its format belongs to the components that produce and + consume that candidate kind — the host only stores, transports, archives and + publishes the reference. - Like every entity in this plugin, a Candidate's durable identity is its - store-assigned ``id`` (``name`` is left for the store to auto-slug). - ``label`` is the run-scoped handle ("agent-0", "agent-1", ...) used for the - working directory, evolution-tree key, and ``ancestor`` references; it is - unique within a run, not globally. + A Candidate is only ever created once its artifact exists and validates, so + ``artifact`` is required and no durable record points at partial work. Incomplete + work is a runner-owned path, not a Candidate. - A candidate's completed artifact is not stored here yet: it is still the - ``agents/