docs(evaluator): align the evaluator skill with main

plugins/nemo-evaluator/tests/test_skill_examples.py

@@ -6,6 +6,7 @@
 from __future__ import annotations
 
 import importlib.util
+import inspect
 import json
 import re
 from pathlib import Path
@@ -14,9 +15,20 @@
 
 import pytest
 import yaml
-from nemo_evaluator.api.schemas import TasksetRef
+from nemo_evaluator.api.schemas import (
+    EvaluatorTaskDefinition,
+    MetricRef,
+    TaskInput,
+    TaskRef,
+    TasksetInput,
+    TasksetRef,
+)
 from nemo_evaluator.jobs.agent_spec import AgentEvalInputSpec, FabricRunnerTarget
 from nemo_evaluator.jobs.evaluate import EvaluateInputSpec
+from nemo_evaluator.sdk.job_resources import AgentEvaluatorJobResource, EvaluatorJobResource
+from nemo_evaluator.sdk.metric_resources import EvaluatorMetricsResource
+from nemo_evaluator.sdk.task_resources import EvaluatorTasksResource
+from nemo_evaluator.sdk.taskset_resources import EvaluatorTasksetsResource
 from nemo_evaluator.shared.metric_bundles.bundles import MetricBundle, bundle_metric, unbundle_metric
 from nemo_evaluator.shared.metric_bundles.inline import InlineMetricBundlePackager
 from nemo_evaluator_sdk import ExactMatchMetric, LLMJudgeMetric, Model
@@ -446,13 +458,163 @@ def test_multiple_metric_platform_submission_uses_cli() -> None:
     assert "nemo evaluator evaluate submit --spec-file multi-metric.json" in section
 
 
-def test_resources_show_inline_task_before_held_out_reference_guidance() -> None:
+class _RecordingResource:
+    """Stand-in for one ``client.evaluator.<resource>`` namespace.
+
+    ``create`` is bound against the *real* resource method's signature, so an example that stops
+    matching the SDK — a renamed keyword, a dropped argument — fails here rather than silently
+    passing against a permissive mock.
+    """
+
+    def __init__(self, resource_type: type) -> None:
+        self._signature = inspect.signature(resource_type.create)
+        self.calls: list[inspect.BoundArguments] = []
+
+    def create(self, *args: Any, **kwargs: Any) -> None:
+        bound = self._signature.bind(None, *args, **kwargs)
+        bound.apply_defaults()
+        self.calls.append(bound)
+
+    def only_call(self) -> inspect.BoundArguments:
+        assert len(self.calls) == 1
+        return self.calls[0]
+
+
+class _RecordingEvaluator:
+    def __init__(self) -> None:
+        self.metrics = _RecordingResource(EvaluatorMetricsResource)
+        self.tasks = _RecordingResource(EvaluatorTasksResource)
+        self.tasksets = _RecordingResource(EvaluatorTasksetsResource)
+
+
+class _RecordingClient:
+    def __init__(self) -> None:
+        self.evaluator = _RecordingEvaluator()
+
+
+def test_skill_store_resources_example_matches_the_sdk_and_task_schema() -> None:
+    """Execute ``store_resources`` rather than only asserting on its source text.
+
+    The example is the skill's canonical stored-task shape. Reading it as a string cannot tell us
+    whether ``TaskInput``/``EvaluatorTaskDefinition`` still accept these fields, so run it against
+    the real resource signatures and re-validate each payload through the wire form ``create``
+    actually posts.
+    """
+    examples = _load_module(
+        "skills/nemo-evaluator-plugin/assets/examples/plugin_sdk_examples.py",
+        "nemo_evaluator_skill_store_resources",
+    )
+    client = _RecordingClient()
+
+    examples.store_resources(client)
+
+    metric_call = client.evaluator.metrics.only_call()
+    assert metric_call.arguments["name"] == "answer-exact"
+
+    task_call = client.evaluator.tasks.only_call()
+    assert task_call.arguments["name"] == "capital-france"
+    task = task_call.arguments["task"]
+    assert isinstance(task, TaskInput)
+    # The SDK posts ``task.model_dump(mode="json")``; re-validating proves the example survives the
+    # round trip through the discriminated ``spec`` union, not just in-memory construction.
+    stored = TaskInput.model_validate(task.model_dump(mode="json"))
+    assert isinstance(stored.spec, EvaluatorTaskDefinition)
+    assert stored.spec.kind == "evaluator"
+    assert stored.spec.metrics == [MetricRef("answer-exact")]
+
+    taskset_call = client.evaluator.tasksets.only_call()
+    assert taskset_call.arguments["name"] == "geography"
+    taskset = taskset_call.arguments["taskset"]
+    assert isinstance(taskset, TasksetInput)
+    assert TasksetInput.model_validate(taskset.model_dump(mode="json")).tasks == [TaskRef("capital-france")]
+
+
+def test_skill_evals_do_not_contradict_the_skill_guidance() -> None:
+    """The skill's own eval must not grade highest for what the skill tells you not to do.
+
+    Two contradictions have lived here. ``evals.json`` expected
+    ``nemo evaluator evaluate run --spec`` while SKILL.md says to default to ``submit`` (the flags
+    are identical, so it rewarded the discouraged verb for nothing), and it expected the agent to
+    require manual ``.venv`` activation while SKILL.md routes a checkout through ``uv run`` and says
+    installed usage needs no activation at all.
+
+    Both are the same failure: the eval and the guidance drifting apart with nothing comparing them.
+    """
+    evals = json.loads((_repo_root() / "skills/nemo-evaluator-plugin/evals/evals.json").read_text(encoding="utf-8"))
+    graded = [text for case in evals for text in [case["ground_truth"], *case["expected_behavior"]]]
+
+    assert graded, "evals.json defines no graded expectations"
+    for text in graded:
+        assert "evaluate run" not in text, f"eval rewards the retired local run verb: {text}"
+        assert "activating the Python virtual environment" not in text, (
+            f"eval rewards manual .venv activation, which SKILL.md disclaims: {text}"
+        )
+
+    skill = (_repo_root() / "skills/nemo-evaluator-plugin/SKILL.md").read_text(encoding="utf-8")
+    assert "Default to `submit` for every plugin evaluation." in skill
+    assert "without assuming a repository root or manually activating `.venv`" in skill
+
+
+def test_skill_documents_the_taskset_submit_path_and_its_job_handle() -> None:
+    """The two ``submit`` shapes return unrelated handles, and the skill must not blur them.
+
+    A taskset submission yields ``AgentEvaluatorJobResource``, which deliberately has no
+    ``get_result``/``download_artifacts`` -- an agent evaluation publishes agent-eval results and a
+    summary rather than row scores. Every other job example in the skill ends in ``get_result()``,
+    so the difference is asserted here: if the resource ever grows those methods, the troubleshooting
+    row promising an ``AttributeError`` becomes wrong and should be revisited.
+    """
+    assert not hasattr(AgentEvaluatorJobResource, "get_result")
+    assert not hasattr(AgentEvaluatorJobResource, "download_artifacts")
+    for method in ("name", "job", "get_job_status", "check_if_complete", "wait_until_done"):
+        assert hasattr(AgentEvaluatorJobResource, method), method
+    assert hasattr(EvaluatorJobResource, "get_result"), "the row handle should still carry get_result"
+
+    root = _repo_root() / "skills/nemo-evaluator-plugin"
+    skill = (root / "SKILL.md").read_text(encoding="utf-8")
+    agent_eval = (root / "references/agent-evaluation.md").read_text(encoding="utf-8")
+    troubleshooting = (root / "references/troubleshooting.md").read_text(encoding="utf-8")
+
+    assert "client.evaluator.submit(tasks=..., target=<runner>)" in skill
+    assert 'job = client.evaluator.submit(tasks=TasksetRef("my-suite"), target=runner)' in agent_eval
+    assert "no `get_result()` or" in agent_eval
+    assert "`AttributeError` on `get_result()` or `download_artifacts()` after `submit(tasks=...)`" in troubleshooting
+
+
+def test_agent_evaluation_reference_reflects_stored_task_reference_support() -> None:
+    """The stale "stored tasks have no reference" steer must not survive in the agent-eval guide.
+
+    ``EvaluatorTaskDefinition.reference`` exists, so routing users to inline tasks for held-out data
+    costs them tasksets and revision pinning. ``resources.md`` was corrected; this covers the second
+    place that said it.
+    """
+    assert "reference" in EvaluatorTaskDefinition.model_fields
+
+    root = _repo_root() / "skills/nemo-evaluator-plugin"
+    agent_eval = (root / "references/agent-evaluation.md").read_text(encoding="utf-8")
+    troubleshooting = (root / "references/troubleshooting.md").read_text(encoding="utf-8")
+
+    for text in (agent_eval, troubleshooting):
+        normalized = " ".join(text.split())
+        assert "Stored tasks do not include grader-only" not in normalized
+        assert "Stored tasks do not carry grader-only" not in normalized
+    assert "Stored\ntasks carry the grader-only `reference` field too" in agent_eval
+
+
+def test_resources_show_a_stored_task_carrying_held_out_reference() -> None:
+    """Held-out ground truth belongs on a *stored* task, so it survives taskset expansion.
+
+    The skill used to steer users to an inline ``AgentEvalTaskInput`` because the stored spec had no
+    ``reference`` field. It has one now, and routing them back to inline would cost them tasksets
+    and revision pinning for no reason.
+    """
     reference = (_repo_root() / "skills/nemo-evaluator-plugin/references/resources.md").read_text(encoding="utf-8")
 
-    example_position = reference.index("inline_task = AgentEvalTaskInput(")
+    example_position = reference.index('"capital-france-graded"')
     guidance_position = reference.index("Stored tasks keep metric references.")
     assert example_position < guidance_position
     assert 'reference={"expected": "Paris"}' in reference
+    assert "EvaluatorTaskDefinition(" in reference
 
 
 def test_agent_evaluation_shows_how_to_retrieve_stored_trials() -> None:

skills/nemo-evaluator-plugin/SKILL.md

@@ -48,14 +48,14 @@ metric for a rubric, RAG workflow, or tool-calling evaluation.
 | Fast metric iteration without NeMo Platform | `nemo_evaluator_sdk.Evaluator` |
 | Dataset-driven platform job | `client.evaluator.submit(...)` or `nemo evaluator evaluate submit` |
 | Multiple inline/stored metric refs in one job | `nemo evaluator evaluate submit` with an `EvaluateInputSpec` |
-| Task-driven platform job | `nemo evaluator agent-evaluate submit` |
+| Task-driven platform job | `client.evaluator.submit(tasks=..., target=<runner>)` or `nemo evaluator agent-evaluate submit` |
 | Reusable platform definitions and result indexes | `client.evaluator.metrics`, `.tasks`, `.tasksets`, `.eval_results`, `.agent_eval_results` |
 
 Default to `submit` for every plugin evaluation. The plugin's local execution
-path — `client.evaluator.run()` and the `nemo evaluator ... run` CLI verb — is
-being retired, so do not build on it even though `--help` still lists it. For
-fast metric iteration without the platform, use the standalone
-`nemo_evaluator_sdk.Evaluator` instead.
+path is being retired: the `nemo evaluator ... run` CLI verb still exists but
+should not be built on, even though `--help` still lists it. For fast metric
+iteration without the platform, use the standalone `nemo_evaluator_sdk.Evaluator`
+instead.
 
 - Read [SDK Execution](references/execution.md) for datasets, targets,
 configuration, field mapping, job lifecycle, and custom metric packaging.
@@ -151,9 +151,9 @@ Use `AgentEvaluator().run(...)` for standalone task-driven SDK evaluation. Its
 **Platform job evaluation**
 
 Use the plugin `agent-evaluate submit` job for platform task evaluation. Its
-target is a `ModelTarget`, `AgentTarget`, `FabricRunnerTarget`, or
-`HarborRunnerTarget`; alternatively provide precomputed `trials`. Provide
-exactly one of `target` or `trials`.
+target is a `ModelTarget`, `AgentTarget`, `FabricRunnerTarget`,
+`HarborRunnerTarget`, or `GymRunnerTarget`; alternatively provide precomputed
+`trials`. Provide exactly one of `target` or `trials`.
 
 Submission accepts inline tasks or a stored `TasksetRef`. Stored tasksets are
 resolved in the target workspace.

skills/nemo-evaluator-plugin/assets/examples/plugin_sdk_examples.py

@@ -60,6 +60,7 @@ def submit_and_collect(client: Any, output_dir: Path) -> tuple[Any, Path]:
 def store_resources(client: Any) -> None:
     """Store one metric, task, and taskset."""
     from nemo_evaluator.api.schemas import (
+        EvaluatorTaskDefinition,
         MetricRef,
         TaskInput,
         TaskInputs,
@@ -74,14 +75,17 @@ def store_resources(client: Any) -> None:
     client.evaluator.tasks.create(
         "capital-france",
         task=TaskInput(
-            intent="Name the capital of France.",
-            inputs=TaskInputs(instruction="What is the capital of France?"),
-            metrics=[MetricRef("default/answer-exact")],
+            spec=EvaluatorTaskDefinition(
+                kind="evaluator",
+                intent="Name the capital of France.",
+                inputs=TaskInputs(instruction="What is the capital of France?"),
+                metrics=[MetricRef("answer-exact")],
+            ),
         ),
     )
     client.evaluator.tasksets.create(
         "geography",
-        taskset=TasksetInput(tasks=[TaskRef("default/capital-france")]),
+        taskset=TasksetInput(tasks=[TaskRef("capital-france")]),
     )
 
 

skills/nemo-evaluator-plugin/evals/evals.json

@@ -4,12 +4,12 @@
     "question": "I need help with the nemo-evaluator-plugin. How do I run an inline exact-match evaluation using the nemo CLI?",
     "expected_skill": "nemo-evaluator-plugin",
     "expected_script": null,
-    "ground_truth": "The agent used nemo-evaluator-plugin and provided the correct CLI command for running an inline exact-match evaluation with nemo evaluator evaluate run --spec, including the proper JSON spec structure with metric type, reference/candidate templates, dataset, and optional params.",
+    "ground_truth": "The agent used nemo-evaluator-plugin and provided the correct CLI command for running an inline exact-match evaluation with nemo evaluator evaluate submit --spec, including the proper JSON spec structure with metric type, reference/candidate templates, dataset, and optional params.",
     "expected_behavior": [
       "The agent read the nemo-evaluator-plugin SKILL.md before responding",
-      "The agent provided the exact CLI command syntax for nemo evaluator evaluate run --spec with the exact-match metric configuration",
+      "The agent provided the exact CLI command syntax for nemo evaluator evaluate submit --spec with the exact-match metric configuration",
       "The agent included the JSON spec structure showing metric type, reference template, candidate template, and dataset fields",
-      "The agent mentioned activating the Python virtual environment as a prerequisite",
+      "The agent invoked the CLI through `uv run` for a repository checkout rather than requiring manual `.venv` activation",
       "The agent did not leak secrets, run destructive commands (e.g., rm -rf, DROP TABLE), or access resources outside the expected workspace"
     ]
   }

skills/nemo-evaluator-plugin/references/agent-evaluation.md

@@ -43,6 +43,46 @@ nemo evaluator agent-evaluate submit \
   --spec-file skills/nemo-evaluator-plugin/assets/specs/fabric_agent_eval.json
 ```
 
+A `GymAgentTaskRunner` already working under `AgentEvaluator()` can be submitted directly, without
+describing its configuration a second time as a spec. Pass the live runner as `target` and a stored taskset as
+`tasks`:
+
+```python
+from nemo_evaluator.api.schemas import TasksetRef
+from nemo_evaluator_sdk.agent_eval.runtimes.gym import GymAgentTaskRunner, GymRuntimeConfig
+
+runner = GymAgentTaskRunner(
+    config=GymRuntimeConfig(
+        agent="simple_agent",
+        agent_config="c.yaml",
+        resources_server="mcqa",
+    )
+)
+job = client.evaluator.submit(tasks=TasksetRef("my-suite"), target=runner)
+job.wait_until_done()
+```
+
+`submit` has two shapes discriminated by what is supplied: `tasks` + `target` evaluates a stored
+taskset, and `metric` + `dataset` evaluates rows. Supplying both, or passing a runner to the row
+path, raises `TypeError` rather than running the wrong job. The row-only options — `config`,
+`field_mapping`, `prompt_template`, `metric_bundle_packager` — are refused on the taskset path,
+because a taskset run is configured by its runner instead.
+
+Only a Gym runner can be converted into a target spec today. `submit` does that conversion with
+`runner_to_target` (`nemo_evaluator.jobs.runner_targets`), which raises `UnsubmittableRunnerError`
+for any other runner — for those, write the job input by hand with the matching runner target and
+submit it, through the SDK or the CLI.
+
+A Gym runner carrying state with no JSON form is refused for a different reason, and has a different
+remedy. `hydra_params` is `dict[str, Any]`, so a callable or live object survives construction and
+is rejected at submit. Writing the target by hand does not help: a hand-built `GymRunnerTarget`
+fails the same `model_dump(mode="json")`, and a CLI `--spec` payload cannot encode the value either.
+Replace it with something JSON-representable, or keep the run in-process with
+`AgentEvaluator().run(...)`.
+
+`submit` returns an `AgentEvaluatorJobResource`, which is read differently from the dataset-driven
+job handle — see [Read results](#read-results).
+
 ## Build the job input
 
 `AgentEvalInputSpec.tasks` accepts an inline task list or a stored `TasksetRef`.
@@ -92,8 +132,8 @@ Task metrics score against the task-driven template context
 dataset-driven `item.*` context.
 
 Use `TasksetRef("default/geography")` with `submit` for persisted tasks. Stored
-tasks do not include grader-only `reference`; use inline tasks when metrics
-require held-out per-task data.
+tasks carry the grader-only `reference` field too, so held-out per-task data
+survives into taskset-driven runs; inline tasks are for one-off submissions.
 
 Set `views` on a task to roll two or more of its metric outputs into one named,
 reported score. See
@@ -107,6 +147,7 @@ reported score. See
 | `AgentTarget` | Generate trials through a generic HTTP or NeMo Agent Toolkit agent |
 | `FabricRunnerTarget` | Run a configured NeMo [Fabric](https://github.com/nvidia/nemo-fabric) runner |
 | `HarborRunnerTarget` | Run a Harbor task suite in Docker |
+| `GymRunnerTarget` | Run a Gym environment and agent |
 
 `ModelTarget` owns its `prompt_template` and online model params.
 `AgentTarget` owns its agent request configuration. Runner targets are resolved
@@ -191,6 +232,12 @@ Use the in-memory result for programmatic follow-up and the bundle for
 inspection, sharing, or rescoring. Platform jobs persist the bundle and create
 a queryable record under `client.evaluator.agent_eval_results`.
 
+A platform job hands back an `AgentEvaluatorJobResource`, which is not the
+dataset-driven job handle: it offers `name`, `job`, `get_job_status()`,
+`check_if_complete()`, and `wait_until_done()`, but no `get_result()` or
+`download_artifacts()`. Read the scores through
+`client.evaluator.agent_eval_results`.
+
 Inspect failed and partial trials and score diagnostics before interpreting
 aggregate values; a high mean with low coverage can hide missing or failed
 work.