feat(experimentalist): add Harbor examples

packages/nemo_evaluator_sdk/tests/agent_eval/test_harbor_runtime_e2e.py

@@ -28,6 +28,27 @@
 _DATASET_DIR = Path(__file__).resolve().parents[2] / "examples" / "harbor" / "hello_world_dataset"
 _TASK_NAME = "harbor/hello-world"
 
+# Minimal BaseAgent for the resume probe. Satisfies the bundled hello-world verifier,
+# which passes iff /app/hello.txt contains exactly "Hello, world!".
+_RESUME_PROBE_AGENT = """\
+from harbor import BaseAgent
+
+
+class WrappedAgent(BaseAgent):
+    @staticmethod
+    def name() -> str:
+        return "resume-probe"
+
+    def version(self) -> str | None:
+        return "1.0.0"
+
+    async def setup(self, environment) -> None:
+        return None
+
+    async def run(self, instruction, environment, context) -> None:
+        await environment.exec("printf 'Hello, world!' > /app/hello.txt")
+"""
+
 
 def _docker_available() -> bool:
     if shutil.which("docker") is None:
@@ -66,3 +87,68 @@ async def test_sdk_runs_harbor_hello_world_natively(tmp_path: Path) -> None:
     payload = reward_payload_from_result(result)
     assert payload["reward"]["harbor_reward.reward"] == 1.0
     assert payload["exceptions"] == {}
+
+
+@pytest.mark.asyncio
+async def test_harbor_resumes_a_partial_job_with_a_custom_agent_dir(tmp_path: Path) -> None:
+    """Regression for AALGO-430 — a real Harbor resume with ``agent_dir`` set.
+
+    This is the case every faked-``Job`` test misses, and the reason the bug went
+    unnoticed: the scoped agent import path used to carry a fresh uuid per run, so
+    Harbor's ``JobConfig`` comparison failed on the second call and it raised
+    ``FileExistsError`` rather than resuming. Now the path is content-addressed, so
+    an unchanged agent resumes and only the missing trial is re-run.
+
+    Runs two attempts and drops one, so "resumed" is distinguishable from "discarded
+    and re-run from scratch" — with a single attempt the two are observationally
+    identical and the test would pass either way.
+    """
+    pytest.importorskip("harbor")
+    if not _docker_available():
+        pytest.skip("Docker daemon is required to run a Harbor job")
+
+    # A loose wrapper file next to the dataset — the shape `agent_dir` exists for,
+    # and the shape the Experimentalist always uses.
+    agent_dir = tmp_path / "agent"
+    agent_dir.mkdir()
+    (agent_dir / "harbor_wrapper.py").write_text(_RESUME_PROBE_AGENT, encoding="utf-8")
+
+    jobs_dir = tmp_path / "jobs"
+    config = HarborRuntimeConfig(
+        jobs_dir=jobs_dir,
+        job_name="resume-job",  # pinned: the cache and Harbor's resume both need it
+        agent_import_path="harbor_wrapper:WrappedAgent",
+        agent_dir=agent_dir,
+        # Two attempts so one can be dropped and one kept. With a single attempt,
+        # discarding the whole job dir and re-running is observationally identical to
+        # resuming, and the assertions below could not tell them apart.
+        n_attempts=2,
+    )
+
+    first = await run_harbor_eval(config, _DATASET_DIR)
+    assert [trial.status for trial in first.trials] == [AgentEvalTrialStatus.COMPLETED] * 2
+
+    job_dir = jobs_dir / "resume-job"
+    trial_dirs = sorted(path.parent for path in job_dir.glob("*/result.json"))
+    assert len(trial_dirs) == 2, "the first run must have written both attempts"
+    survivor, dropped = trial_dirs
+    survivor_result = (survivor / "result.json").read_text(encoding="utf-8")
+
+    # Drop one attempt's result so the job is under-covered, leaving the job dir (and
+    # its config.json) in place — the exact state that used to raise FileExistsError.
+    (dropped / "result.json").unlink()
+
+    second = await run_harbor_eval(config, _DATASET_DIR)
+
+    # The point of the test: the completed attempt was *resumed*, not re-run. Harbor
+    # suffixes each trial dir with a shortuuid, so a discarded job dir would come back
+    # under a different name, and a re-executed trial would rewrite result.json.
+    assert survivor.is_dir(), "the completed attempt's trial dir must survive the rerun"
+    assert (survivor / "result.json").read_text(encoding="utf-8") == survivor_result, (
+        "the completed attempt must be reused untouched, not re-executed"
+    )
+    assert not dropped.is_dir(), "the result-less attempt must be cleared and re-run"
+
+    assert [trial.status for trial in second.trials] == [AgentEvalTrialStatus.COMPLETED] * 2
+    assert {trial.task_id for trial in second.trials} == {_TASK_NAME}
+    assert [trial.metadata["reward"] for trial in second.trials] == [1.0, 1.0]

plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/config.yaml

@@ -10,7 +10,7 @@
 workspace: default
 base_url: http://localhost:8080
 mode: local
-evaluator_type: harbor
+evaluator_type: harbor_native
 
 # Required per run.
 insight: ""

plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/run.py

@@ -49,7 +49,7 @@ async def run_eval_author(
     base_url: str | None,
     config: EvalAuthorConfig,
     agent: Path | str | None = None,
-    evaluator_type: EvaluatorType = "harbor",
+    evaluator_type: EvaluatorType = "harbor_native",
     mode: Literal["local", "remote"] = "local",
 ) -> EvalAuthorResult:
     """Build and run the Eval Author against an Insight and evaluator datasets.

plugins/nemo-eval-author/tests/test_eval_author_repair_e2e.py

@@ -18,6 +18,8 @@
 )
 from nemo_experimentalist_plugin.experimentalist.components.evaluator.harbor import (
     HarborDataset,
+)
+from nemo_experimentalist_plugin.experimentalist.components.evaluator.harbor_native import (
     HarborEvaluator,
     HarborEvaluatorConfig,
 )

plugins/nemo-eval-author/tests/test_eval_author_run.py

@@ -8,6 +8,7 @@
 import pytest
 from nemo_eval_author_plugin.eval_author import run as eval_author_run
 from nemo_eval_author_plugin.eval_author.models import EvalAuthorConfig, EvalAuthorResult
+from nemo_experimentalist_plugin.experimentalist.components.evaluator.base import EvaluatorType
 from nemo_experimentalist_plugin.experimentalist.components.evaluator.models import Dataset, DatasetRef, Task
 from nemo_insights_plugin.entities import Insight
 
@@ -113,10 +114,12 @@ async def run(
         )
 
 
+@pytest.mark.parametrize("evaluator_type", ["harbor_native", "harbor_evaluator"])
 @pytest.mark.asyncio
 async def test_run_eval_author_builds_and_runs_complete_contract(
     monkeypatch: pytest.MonkeyPatch,
     tmp_path: Path,
+    evaluator_type: EvaluatorType,
 ) -> None:
     client = ClosingClient()
     insight = Insight(
@@ -168,6 +171,7 @@ def build_eval_author_agent(*, experiment_dir: Path, config: EvalAuthorConfig) -
         workspace="workspace-a",
         base_url="http://platform.test",
         config=config,
+        evaluator_type=evaluator_type,
         mode="local",
     )
 
@@ -187,10 +191,13 @@ def build_eval_author_agent(*, experiment_dir: Path, config: EvalAuthorConfig) -
             dest=experiment_dir / "eval_author" / "source-agent",
         )
     ]
-    assert dataset_factory.dataset_refs == [("harbor", train_ref), ("harbor", validation_ref)]
+    assert dataset_factory.dataset_refs == [
+        (evaluator_type, train_ref),
+        (evaluator_type, validation_ref),
+    ]
     assert dataset_factory.template_refs == [
         (
-            "harbor",
+            evaluator_type,
             template_ref.model_copy(update={"uri": str(experiment_dir / "dataset" / "task-template")}),
         )
     ]
@@ -253,7 +260,9 @@ async def download(self, *, remote_path: str, local_path: str, workspace: str) -
             "workspace": "workspace-a",
         }
     ]
-    assert dataset_factory.template_refs == [("harbor", template_ref.model_copy(update={"uri": str(staged_path)}))]
+    assert dataset_factory.template_refs == [
+        ("harbor_native", template_ref.model_copy(update={"uri": str(staged_path)}))
+    ]
     assert client.closed
 
 

plugins/nemo-experimentalist/README.md

@@ -10,27 +10,22 @@ or Git-backed agent against Harbor-compatible train and validation datasets.
 
 ## Install and develop
 
-From the root of this checkout:
+This plugin is a workspace member of the NeMo Platform monorepo. Its agent
+framework (NOOA) and evaluator (Harbor) are both Python 3.12-only, so the whole
+plugin sits behind an optional dependency group. From the **platform root**:
 
 ```bash
-uv sync
+uv sync --group experimentalist
 export NEMO="$PWD/.venv/bin/nemo"
 ```
 
-For a NeMo Platform source checkout, use the
-[source-Platform installer](docs/e2e/install-experimentalist-plugin.sh), which keeps
-Platform packages editable while installing both plugins' direct runtime
-dependencies:
-
-```bash
-REPO="$PWD" PLAT=/path/to/nemo-platform bash docs/e2e/install-experimentalist-plugin.sh
-export NEMO=/path/to/nemo-platform/.venv/bin/nemo
-```
-
 The source dependencies are pinned to tagged or immutable revisions in
 `pyproject.toml`. NVIDIA-labs OO Agents (NOOA) is pinned to a public GitHub
 commit, currently one past `v0.0.6` that carries an MCP transport-timeout fix.
 
+Verify with `$NEMO experimentalist doctor`. Harbor evaluation also needs a
+running Docker daemon — `doctor` treats both as required checks.
+
 ## Insight-to-experiment flow
 
 The supported handoff is:
@@ -117,9 +112,25 @@ $NEMO experimentalist run \
 ```
 
 Pass one or more framework skill directories with `--framework-skills` when
-the agent needs framework-specific modification guidance. The checked-in Tau2
-profile demonstrates profile-owned datasets and task template configuration:
-[`examples/tau2-nemo-oo-agent/optimizer.yaml`](examples/tau2-nemo-oo-agent/optimizer.yaml).
+the agent needs framework-specific modification guidance. The checked-in
+[`tau3-nooa-agent`](examples/tau3-nooa-agent/README.md) demonstrates the
+realistic NOOA, MCP, and inference-backed path. Follow the
+[getting-started guide](../../docs/get-started/example-agent.mdx) to prepare its
+train and validation datasets before running the SDK-backed smoke config.
+
+For a first run, prefer the fully local example — no dataset registry, no
+Platform, and a validation evaluation that finishes in seconds. From the
+platform root:
+
+```bash
+$NEMO experimentalist run \
+  --profile plugins/nemo-experimentalist/examples/hello-harbor-agent/optimizer.yaml \
+  --no-insight \
+  --experiment-dir tmp/exp-hello
+```
+
+See [`examples/hello-harbor-agent/README.md`](examples/hello-harbor-agent/README.md)
+for what it contains and how to run it.
 
 Each run writes its local artifacts under `--experiment-dir`, or under
 `.nemo-optimizer/experiments/` beside the governing profile by default.

plugins/nemo-experimentalist/benchmarks/run.py

@@ -15,8 +15,8 @@
 
 import yaml
 from harbor.registry.client.package import PackageDatasetClient
-from nemo_experimentalist_plugin.experimentalist.components.evaluator.harbor import (
-    HarborDataset,
+from nemo_experimentalist_plugin.experimentalist.components.evaluator.harbor import HarborDataset
+from nemo_experimentalist_plugin.experimentalist.components.evaluator.harbor_native import (
     HarborEvaluator,
     HarborEvaluatorConfig,
 )

plugins/nemo-experimentalist/examples/README.md

@@ -0,0 +1,30 @@
+<!-- SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -->
+<!-- SPDX-License-Identifier: Apache-2.0 -->
+
+# Experimentalist examples
+
+These examples make Experimentalist behavior reviewable at three levels. They
+show the complete agent-to-Harbor adapter contract without presenting sample
+agents as production applications.
+
+| Example | Why it exists | What it demonstrates | How to use it |
+|---|---|---|---|
+| [`hello-harbor-agent`](hello-harbor-agent/README.md) | Small onboarding and debugger fixture. | Fully local agent, train/validation tasks, deterministic traces, two metrics, and a deliberate arithmetic gap for one optimizer round to diagnose. | Start with its README. No model key is needed for the Docker-backed evaluator A/B pytest. |
+| [`tau3-nooa-agent`](tau3-nooa-agent/README.md) | Realistic interactive-agent target. | NOOA CodeAct agent, Tau3 airline tasks, MCP runtime sidecar, prepared train/validation datasets, and inference-backed user simulation. | Follow the [getting-started guide](../../../docs/get-started/example-agent.mdx) to prepare datasets, record traces, and run one SDK-backed Experimentalist smoke round. |
+| [`terminal-bench-agent`](terminal-bench-agent/README.md) | Canonical benchmark runtime fixture. | Locked LangChain agent installed inside unmodified Terminal-Bench task containers, with no sidecar or task-definition changes. | Use through the [canonical benchmark runner](../benchmarks/README.md), or invoke its module directly as documented in its agent spec. |
+
+The examples expose the same agent-to-Harbor adapter shape:
+
+```text
+AGENT-SPEC.md        behavior contract supplied to optimizer components
+agent.py / main.py   code under optimization and its entry point
+harbor_wrapper.py    upload, install, execute, trace, and artifact bridge
+dataset/             optional local tasks
+optimizer.yaml       optional profile for self-contained fixtures
+```
+
+Recommended order:
+
+1. Use `hello-harbor-agent` to inspect evaluator wiring and one optimizer round.
+2. Use `tau3-nooa-agent` to inspect realistic MCP and inference-backed behavior.
+3. Use `terminal-bench-agent` for reproducible benchmark runs.

plugins/nemo-experimentalist/examples/hello-harbor-agent/.env.example

@@ -0,0 +1,28 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Copy to `.env` in this directory. `nemo experimentalist run` auto-loads it from
+# the profile directory on every run (variables already exported in your shell
+# win). Verify with `nemo experimentalist doctor --profile ./optimizer.yaml`.
+#
+# This agent makes no LLM calls of its own — these credentials are for the
+# Experimentalist's own components (Coder, Analyzer, Proposer, Terminator).
+
+# The one required credential: an NVIDIA Inference Gateway virtual key (sk-...).
+# On the gateway, EXPERIMENTALIST_API_KEY is filled from this automatically.
+INFERENCE_API_KEY=sk-...
+
+# Only needed for a non-gateway LLM provider (then set both; the gateway key is
+# never sent to a custom endpoint):
+# EXPERIMENTALIST_API_BASE=https://inference-api.nvidia.com/v1
+# EXPERIMENTALIST_API_KEY=sk-...
+
+# Optional model overrides. Must be models your key serves — list them with:
+#   curl -s $EXPERIMENTALIST_API_BASE/models -H "Authorization: Bearer $EXPERIMENTALIST_API_KEY"
+#
+# Prefix the served id with an EXTRA "openai/": LiteLLM strips the first path
+# segment as the provider, so only the remainder reaches the gateway.
+#   served "openai/openai/gpt-5.6-luna"  ->  set "openai/openai/openai/gpt-5.6-luna"
+#   served "azure/openai/gpt-5.6-terra"  ->  set "openai/azure/openai/gpt-5.6-terra"
+# EXPERIMENTALIST_SMART_MODEL_NAME=openai/openai/openai/gpt-5.5
+# EXPERIMENTALIST_FAST_MODEL_NAME=openai/openai/openai/gpt-5-mini

plugins/nemo-experimentalist/examples/hello-harbor-agent/AGENT-SPEC.md

@@ -0,0 +1,39 @@
+<!-- SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -->
+<!-- SPDX-License-Identifier: Apache-2.0 -->
+
+# hello-harbor-agent
+
+## Job
+
+Read one task instruction, produce the single line of text the instruction asks
+for, and write that line to `/app/artifacts/output.txt`.
+
+## Interface
+
+- Invoked as `python main.py --prompt "<instruction text>"` with `/app` as the
+  working directory.
+- Writes exactly one line (plus a trailing newline) to
+  `/app/artifacts/output.txt`.
+- Writes an OTLP JSONL trace to `/app/traces/agent.jsonl`.
+
+## Design
+
+`HelloAgent.solve` dispatches the instruction across an ordered list of
+handlers and returns the first non-`None` answer, falling back to a fixed
+"I do not know how to answer that." string. Today the only handler is
+`handle_greeting`, which echoes a `Hello, <target>!` line quoted in the
+instruction.
+
+## Constraints
+
+- Standard library only. The task container is a bare Python image with no
+  package installs, no network access, and no LLM credentials.
+- Deterministic: the same instruction must always produce the same answer, so
+  reward differences between candidates come from code changes rather than
+  sampling noise.
+
+## Known gap
+
+The agent has no arithmetic capability, so any task that asks it to compute a
+value scores 0. This is intentional — it gives the optimization loop a real
+root cause to diagnose and close.