From fd8b102cc2637eb37da6d199ba0ca5d9d1458840 Mon Sep 17 00:00:00 2001
From: Sandy Chapman <schapman@nvidia.com>
Date: Thu, 13 Aug 2026 17:15:44 -0300
Subject: [PATCH 1/2] fix(evaluator): make Gym run failures legible and catch
 rollouts where no agent ran
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Running the AALGO-485 coverage sweep across five Gym environments turned up two
ways a Gym run fails without saying anything useful, and one way it fails while
reporting success.

A startup timeout said only "Gym servers not ready within 240.0s", sending the
reader to a log dominated by the servers that started fine. Gym polls and prints
the outstanding set every time, so the message now names it, and says which knob
to raise — `legal_agent_bench` legitimately needs ~8 minutes on a cold cache
because startup installs its dependencies and prepares a Harbor task tree.

A collection failure named the two eval logs, neither of which holds the cause.
Observed on `wmt_translation`: `gym eval run` reports a bare HTTP 500 while the
traceback explaining it sits in gym_env.log, which the message never mentioned.

Worse, an environment whose agent never starts produces rollouts that look
normal: `legal_agent_bench` recorded two "completed" trials scoring 0.0 with an
empty failures sidecar, having never called the model. A verifier-scored runner
reports whatever reward Gym computed, so that is indistinguishable from an agent
that tried and scored zero — and it would have been read as poor performance in
the Gym/Harbor/Fabric comparison AALGO-434 depends on. `_agent_never_ran` now
detects it from stated zero *input*-side token usage plus no output, marks those
trials FAILED with the reward withheld, and raises when that is the whole run.
Input-side only, because a model call always sends a prompt: zero output tokens
merely describes an empty answer, which is a legitimate result.

Also fixes `gym eval run` littering `outputs/<date>/<time>/` into the caller's
cwd — it builds its own argv and never received the `hydra.run.dir` redirect that
`_selection_args` carries.

The coverage tests no longer skip for want of a model endpoint: a stdlib stub
serves the rollouts, answering each prompt with that row's own ground truth so a
mis-attributed rollout scores 0 and fails the test rather than passing quietly.
`NEMO_GYM_POLICY_BASE_URL` remains an override for running against a real model.

Signed-off-by: Sandy Chapman <schapman@nvidia.com>
---
 .../agent_eval/runtimes/gym_runtime.py        | 220 +++++++++++-
 .../test_gym_environment_coverage.py          | 320 ++++++++++++++++--
 .../tests/agent_eval/test_gym_runtime.py      | 291 ++++++++++++++++
 .../agent_eval/runtimes/gym_runtime.py        | 220 +++++++++++-
 4 files changed, 1007 insertions(+), 44 deletions(-)

diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym_runtime.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym_runtime.py
index f40ac28ce9..224bef2f4e 100644
--- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym_runtime.py
+++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym_runtime.py
@@ -109,6 +109,16 @@
 _VALIDATE_TIMEOUT_S = 120.0
 #: Where Gym's Hydra run directories are redirected, relative to the run's work dir.
 _HYDRA_SUBDIR = "gym_hydra"
+#: `gym env start`'s combined output, under the run's work dir. Named here because a *collection*
+#: failure often has to point at it: the eval logs show the symptom, this shows the cause.
+_ENV_LOG_NAME = "gym_env.log"
+#: Token-count keys read to decide whether a model was called at all. Both vocabularies, because
+#: Gym's model servers report in either depending on the adapter: the Responses API spells them
+#: `input_tokens`/`output_tokens`, Chat Completions `prompt_tokens`/`completion_tokens`.
+_TOKEN_USAGE_KEYS = ("total_tokens", "input_tokens", "output_tokens", "prompt_tokens", "completion_tokens")
+#: The subset that proves a call happened. A model call always consumes input tokens, so zero on any
+#: of these is evidence the model was never reached; zero output tokens only means an empty answer.
+_INPUT_TOKEN_KEYS = ("total_tokens", "input_tokens", "prompt_tokens")
 #: Gym's index fields on each rollout record. ``_ng_task_index`` is the only join back to the input
 #: rows that survives a round-trip: Gym mutates ``responses_create_params`` (even the prompt) and
 #: copies only a fixed allowlist of row keys onto the result, so no field we invent comes back. Gym
@@ -728,7 +738,7 @@ async def _run_two_step(self, input_path: Path, output_path: Path, work_dir: Pat
         """Start the Gym servers, collect against them with ``--no-serve``, then tear them down."""
         cfg = self._config
         gym = _gym_executable()
-        env_log = work_dir / "gym_env.log"
+        env_log = work_dir / _ENV_LOG_NAME
 
         # Gym launches each server from its own subdir with its own .venv. Ray (>=2.56) otherwise
         # detects a `uv run` ancestor and tries to replicate that uv project onto its workers,
@@ -791,6 +801,10 @@ async def _collect_rollouts(
             str(cfg.num_repeats),
             "--concurrency",
             str(cfg.concurrency),
+            # `gym eval run` is a Hydra app too, and unlike validate/start it does not go through
+            # _selection_args — so without this it writes `outputs/<date>/<time>/` into the caller's
+            # cwd on every single collection.
+            f"hydra.run.dir={work_dir / _HYDRA_SUBDIR}",
         ]
         stdout_log = work_dir / "gym_eval.stdout.log"
         stderr_log = work_dir / "gym_eval.stderr.log"
@@ -823,8 +837,17 @@ async def _collect_rollouts(
             await _drain_pumps(pumps, grace_s=cfg.shutdown_grace_s, what="gym eval run")
         if eval_proc.returncode != 0:
             tail = "\n".join(tails.get("stderr") or tails.get("stdout") or [])
+            # A collection failure is frequently *server*-side: the resources-server raises, the
+            # agent surfaces it as a bare HTTP 500, and the traceback that explains it is only in
+            # gym_env.log. Naming the two eval logs alone sends the reader to the one place the
+            # cause is not. (Observed on wmt_translation: a 500 here, `PermissionError: /opt/Gym`
+            # there.)
+            env_log = work_dir / _ENV_LOG_NAME
             raise RuntimeError(
-                f"`gym eval run` failed (rc={eval_proc.returncode}). Full output: {stdout_log} and {stderr_log}\n"
+                f"`gym eval run` failed (rc={eval_proc.returncode}). Collection output: {stdout_log} "
+                f"and {stderr_log}\n"
+                f"If the tail below is an HTTP error from a Gym server, the cause is server-side: "
+                f"see {env_log}, whose lines are prefixed with the server that emitted them.\n"
                 f"--- last {_LOG_TAIL_LINES} line(s) ---\n{tail}"
             )
 
@@ -839,6 +862,7 @@ async def _wait_for_servers(self, env_log: Path, env_proc: asyncio.subprocess.Pr
         loop = asyncio.get_running_loop()
         ready = re.compile(r"All \d+ / \d+ servers ready")
         deadline = loop.time() + cfg.startup_timeout_s
+        text = ""
         while loop.time() < deadline:
             if env_proc.returncode is not None:
                 raise RuntimeError(f"`gym env start` exited early (rc={env_proc.returncode}); see {env_log}")
@@ -848,7 +872,129 @@ async def _wait_for_servers(self, env_log: Path, env_proc: asyncio.subprocess.Pr
             if ready.search(text):
                 return
             await asyncio.sleep(2)
-        raise TimeoutError(f"Gym servers not ready within {cfg.startup_timeout_s}s; see {env_log}")
+        raise TimeoutError(self._startup_timeout_message(text, env_log))
+
+    def _startup_timeout_message(self, env_log_text: str, env_log: Path) -> str:
+        """Explain a startup timeout in terms of *which* server did not come up.
+
+        The bare "servers not ready" message sends you to a log that is mostly noise from the servers
+        that started fine. Gym polls and prints the outstanding set every time, so naming it costs
+        nothing and is usually the whole diagnosis: one server out of four means that environment's
+        own startup, not the platform.
+
+        The timeout is also a plausible *cause* rather than a symptom — a Docker-backed environment
+        builds its image on first run, which routinely exceeds the 240s default — so the message says
+        how to extend it instead of leaving the reader to find the knob.
+        """
+        cfg = self._config
+        pending = _pending_servers(env_log_text)
+        if pending is None:
+            detail = (
+                "Gym never reported server readiness, so it likely failed before starting them "
+                "(config composition or dependency installation)."
+            )
+        elif not pending[2]:
+            # Gym printed a readiness line but named no outstanding server. Saying "waiting on:
+            # unknown" would contradict the counts in the same sentence, so report only what is known.
+            ready_count, total, _ = pending
+            detail = (
+                f"{ready_count} of {total} server(s) started; Gym did not name which remained "
+                f"outstanding. The per-server output in {env_log.name} is prefixed with the server "
+                "that emitted each line."
+            )
+        else:
+            ready_count, total, names = pending
+            detail = (
+                f"{ready_count} of {total} server(s) started; still waiting on: {', '.join(names)}. "
+                f"Search {env_log.name} for '({names[0]})' — each line is prefixed with the server "
+                "that emitted it."
+            )
+        return (
+            f"Gym servers for resources-server {cfg.resources_server!r} were not ready within "
+            f"startup_timeout_s={cfg.startup_timeout_s}s. {detail}\n"
+            f"Full startup output: {env_log}\n"
+            "If the environment installs heavy dependencies or builds a container image on first run, "
+            "this timeout is expected on a cold cache — raise GymRuntimeConfig.startup_timeout_s and "
+            "re-run; the second run reuses what the first installed."
+        )
+
+
+#: Gym's per-poll readiness line, e.g.
+#: ``3 / 4 servers ready. Waiting for servers to spin up: ['legal_agent_bench']``.
+#: Gym knows exactly which servers are outstanding; without this we discard that and report only
+#: that *something* timed out.
+_PENDING_SERVERS_RE = re.compile(r"(\d+)\s*/\s*(\d+) servers ready\. Waiting for servers to spin up: \[([^\]]*)\]")
+
+
+def _pending_servers(text: str) -> tuple[int, int, tuple[str, ...]] | None:
+    """Recover ``(ready, total, still-pending)`` from the *last* readiness line Gym logged.
+
+    The last line is the one that matters: earlier polls list servers that have since come up.
+    Returns ``None`` when Gym never got as far as printing one (it died during composition, say),
+    which the caller must treat as "no detail available" rather than "nothing pending".
+
+    Never raises — this only ever decorates an error that is already being reported, and a parsing
+    failure here must not replace the real one. Gym's log format is not contractual.
+    """
+    matches = _PENDING_SERVERS_RE.findall(text)
+    if not matches:
+        return None
+    ready, total, names = matches[-1]
+    try:
+        return int(ready), int(total), tuple(re.findall(r"['\"]([^'\"]+)['\"]", names))
+    except ValueError:  # pragma: no cover - the regex only matches digits
+        return None
+
+
+def _agent_never_ran(record: Mapping[str, Any]) -> bool:
+    """True when a result record shows the agent produced nothing *and* never called the model.
+
+    A verifier-scored runner reports whatever score the environment computed, so an agent that
+    silently failed to start is indistinguishable from one that tried and scored 0.0 — the record
+    looks normal, no failure is reported alongside it, and the run succeeds. Observed on
+    ``legal_agent_bench``, whose agent died locating its task tree and returned an empty response:
+    two "completed" trials scoring 0.0, no model call, no failure recorded.
+
+    Deliberately conservative — **both** conditions must hold:
+
+    * no output, and
+    * a stated zero on the input side of token usage.
+
+    An empty answer alone is a legitimate result (a model can return nothing and earn 0.0), so it is
+    not sufficient. Zero *input* tokens is what says the model was never called: a call always sends
+    a prompt, whereas zero output tokens only describes an empty answer. A record carrying no
+    ``response`` key at all is left alone — some runners populate it differently, and this must not
+    reclassify trials it does not understand.
+    """
+    response = record.get("response")
+    if not isinstance(response, Mapping):
+        return False
+    if "output" not in response:
+        # Nothing to judge: some runners shape `response` differently, and absence is not emptiness.
+        return False
+    # Any non-empty value counts as output, whatever its shape. Checking `len(...) > 0` on a list
+    # alone would treat a plain string answer — or a single mapping — as "produced nothing" and fail
+    # a trial that ran perfectly well.
+    if response["output"]:
+        return False
+    usage = response.get("usage")
+    if not isinstance(usage, Mapping):
+        # No usage block to corroborate with; an empty output on its own is not enough to judge.
+        return False
+    # Zero usage has to be *stated*, not inferred from a key being absent. Two vocabularies are in
+    # play — the Responses API's `input_tokens`/`output_tokens` and Chat Completions'
+    # `prompt_tokens`/`completion_tokens` — and Gym's model servers speak both depending on the
+    # adapter. Reading only one set would make a real call reported in the other read as zero.
+    reported = [usage[key] for key in _TOKEN_USAGE_KEYS if isinstance(usage.get(key), (int, float))]
+    if any(value > 0 for value in reported):
+        return False
+    # The discriminating evidence is *input*-side, and only input-side. Calling a model always sends
+    # a prompt, so zero input tokens cannot describe a call that happened; zero **output** tokens
+    # merely describes an empty answer, which is a legitimate result. Requiring an input-side count
+    # rules out a whole class of false positives — a provider reporting only `output_tokens: 0`, or
+    # only `completion_tokens: 0` — rather than excluding those shapes one at a time.
+    input_side = [usage[key] for key in _INPUT_TOKEN_KEYS if isinstance(usage.get(key), (int, float))]
+    return bool(input_side) and all(value == 0 for value in input_side)
 
 
 def _source_datasets(tasks: Sequence[AgentEvalTask]) -> str:
@@ -1261,6 +1407,10 @@ def _trials_from_rollouts(
         descriptors={"rollouts": EvidenceDescriptor(kind="filesystem", format="file", ref=str(rollouts_path))}
     )
     unattributed_successes = 0
+    empty_output_count = 0
+    #: Tasks with at least one trial where the agent never ran. Tracked separately from trial
+    #: status, since the failures sidecar also produces FAILED trials for unrelated reasons.
+    empty_output_task_ids: set[str] = set()
     for record in _read_jsonl(rollouts_path):
         task_id = _resolve_task_id(record, index_to_task_id)
         if task_id is None:
@@ -1271,18 +1421,36 @@ def _trials_from_rollouts(
             continue
         raw_rollout_index = record.get(NG_ROLLOUT_INDEX)
         reward = _coerce_reward(record.get(reward_key))
+        # An agent that never ran did not earn its reward — reporting 0.0 as a completed trial would
+        # let a broken environment read as a poorly-performing one.
+        never_ran = _agent_never_ran(record)
+        if never_ran:
+            empty_output_count += 1
+            empty_output_task_ids.add(task_id)
+            status = AgentEvalTrialStatus.FAILED
+        elif reward is not None:
+            status = AgentEvalTrialStatus.COMPLETED
+        else:
+            status = AgentEvalTrialStatus.PARTIAL
+        metadata: dict[str, Any] = {
+            "reward": None if never_ran else reward,
+            NG_TASK_INDEX: record.get(NG_TASK_INDEX),
+            NG_ROLLOUT_INDEX: raw_rollout_index,
+        }
+        if never_ran:
+            # Kept alongside the reward Gym reported, so the discarded value is still auditable.
+            metadata["gym_failure"] = (
+                "the agent produced no output and consumed no tokens, so the model was never called; "
+                f"the reported score of {reward!r} is not a measurement of this agent"
+            )
         trials.append(
             AgentEvalTrial(
                 id=_rollout_trial_id(task_id, raw_rollout_index, synth_seq, missing_label="noidx"),
                 task_id=task_id,
-                status=AgentEvalTrialStatus.COMPLETED if reward is not None else AgentEvalTrialStatus.PARTIAL,
+                status=status,
                 output=AgentOutput(response=record.get("response"), metadata={"agent_ref": record.get("agent_ref")}),
                 evidence=success_evidence,
-                metadata={
-                    "reward": reward,
-                    NG_TASK_INDEX: record.get(NG_TASK_INDEX),
-                    NG_ROLLOUT_INDEX: raw_rollout_index,
-                },
+                metadata=metadata,
             )
         )
 
@@ -1321,6 +1489,40 @@ def _trials_from_rollouts(
                     },
                 )
             )
+    if empty_output_count:
+        # Every trial empty is a configuration failure, not a result: nothing was measured, so there
+        # is no score to report and letting the run "succeed" would publish zeros as findings. A
+        # partial count is left as failed trials — a flaky environment is still worth the attempts
+        # that did run.
+        #
+        # Measured against *every* trial, including those the failures sidecar contributed. A run
+        # that also recorded real failures is not the uniform "nothing ran" picture this raise is
+        # for: those trials carry their own diagnosis (a verifier timeout, say, from an attempt that
+        # did reach the model), and raising would discard it for a less specific message. Likewise
+        # `unattributed_successes` never became trials and so are invisible to `len(trials)`, but
+        # each is a result the environment produced and one may have run perfectly well — claiming
+        # nothing was measured while such a record exists would be false.
+        if empty_output_count == len(trials) and not unattributed_successes:
+            affected_tasks = {trial.task_id for trial in trials}
+            raise RuntimeError(
+                f"No agent ran: all {empty_output_count} trial(s) across all {len(affected_tasks)} "
+                "task(s) produced no output and consumed no tokens, so the model was never called "
+                "and nothing was measured. The scores reported for them describe an agent that did "
+                "not run.\n"
+                "This is normally the environment failing to start its agent rather than a bad "
+                f"model: check the per-server startup output in {rollouts_path.parent / _ENV_LOG_NAME} "
+                "for a traceback from the environment's own agent server.\n"
+                f"Results as collected: {rollouts_path}"
+            )
+        logger.warning(
+            "%d of %d trial(s), across %d of %d task(s), produced no agent output and consumed no "
+            "tokens; those are marked FAILED rather than scored, since the model was never called "
+            "for them.",
+            empty_output_count,
+            len(trials),
+            len(empty_output_task_ids),
+            len({trial.task_id for trial in trials}),
+        )
     if unattributed_successes:
         logger.warning(
             "%d Gym rollout record(s) in %s carried no usable %s and could not be attributed to a task; "
diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_gym_environment_coverage.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_gym_environment_coverage.py
index e1b1f40d70..eadace04f4 100644
--- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_gym_environment_coverage.py
+++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_gym_environment_coverage.py
@@ -15,8 +15,9 @@
 **Two layers, deliberately.** ``gym env validate`` merges configs, flags, and overrides and reports
 unset ``???`` values, bad paths, and dangling cross-references *without Ray, servers, a model
 endpoint, or credentials*. It is fast, offline, and answers most of what this sweep wants to know, so
-every environment gets a validate test. Actually collecting rollouts additionally needs a reachable
-model endpoint, so those tests skip unless one is configured.
+every environment gets a validate test. The rollout tests then start the servers and collect for
+real, against a built-in stub endpoint (:class:`_StubPolicyServer`) so they need no model of their
+own; set ``NEMO_GYM_POLICY_BASE_URL`` to run them against a real one instead.
 
 That split matters because model wiring is not standardized. ``mcqa``/``gpqa_diamond``/
 ``wmt_translation`` reference a ``policy_model`` server fed by the global ``policy_base_url`` /
@@ -32,19 +33,26 @@
 * **The target environment's own dependencies** — each ``resources_server`` ships a
   ``requirements.txt``; install it from that directory so its ``-e nemo-gym[dev] @ ../../`` resolves.
 * **Docker**, for environments whose agent works inside containers.
-* **A model endpoint** (``NEMO_GYM_POLICY_BASE_URL`` and friends), for the rollout tests only.
+
+A model endpoint is *not* a prerequisite: the stub supplies one. ``NEMO_GYM_POLICY_BASE_URL`` is an
+override for running against a real model, not a gate.
 
 Tracked by AALGO-485; the CI and GPU-runner story is AALGO-494.
 """
 
 from __future__ import annotations
 
+import json
 import os
 import shutil
 import subprocess
 import tempfile
-from collections.abc import Mapping
+import threading
+import time
+import uuid
+from collections.abc import Callable, Mapping, Sequence
 from dataclasses import dataclass, field
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 from pathlib import Path
 from typing import Any
 
@@ -56,13 +64,15 @@
     _flatten_overrides,
     discover_gym_tasks,
 )
-from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalRunConfig
+from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask
 from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrialStatus
 
 pytestmark = [pytest.mark.integration, pytest.mark.slow]
 
-#: Endpoint for the rollout tests. Gym itself reads credentials from a gitignored ``env.yaml`` in the
-#: working directory; this only gates whether we attempt a run at all.
+#: Opt-in override pointing the rollout tests at a *real* model endpoint instead of the built-in
+#: stub. Unset (the normal case) means the stub serves the rollouts, which is what lets these tests
+#: run unattended. Gym reads its own credentials from a gitignored ``env.yaml``; this only decides
+#: which endpoint the runner is configured against.
 POLICY_BASE_URL_ENV = "NEMO_GYM_POLICY_BASE_URL"
 
 _DEFAULT_AGENT = "simple_agent"
@@ -71,6 +81,123 @@
 #: Keep runs small — this establishes that an environment works at all, not how well it scores.
 _TASK_LIMIT = 2
 
+#: What the stub answers when a case defines no gradeable answer. Deliberately not empty: an empty
+#: response is indistinguishable from an agent that never ran (see `_agent_never_ran`), so a canned
+#: answer keeps "the plumbing works" separable from "the agent produced nothing".
+_CANNED_ANSWER = "This is a stub response from the NeMo Platform test suite."
+
+#: Stands in for a per-run absolute directory inside a case's `env_overrides`. A committed
+#: absolute path would be wrong on every machine, so the run substitutes a real one.
+_ABS_ASSETS_PLACEHOLDER = "_ABS_ASSETS"
+
+
+class _StubPolicyServer:
+    """An OpenAI-compatible ``/v1/chat/completions`` endpoint that answers from a prompt-keyed map.
+
+    **Why a stub at all.** Collecting rollouts needs a reachable model, and requiring one turned the
+    whole rollout half of this module into a permanent skip — the runner's `validate -> env start ->
+    eval run` sequence had never executed end to end. A stub removes the only prerequisite these
+    tests could not satisfy for themselves.
+
+    **Why prompt-keyed rather than a canned reply.** The oracle answers each prompt with *that row's*
+    own expected answer, so a passing run means every task received the answer belonging to it. Gym
+    attributes rollouts back to tasks through an index this runner assigns, and an off-by-one there
+    would still produce a full set of trials — just with the answers paired to the wrong tasks. Under
+    a canned reply that bug scores identically to a correct run; under the oracle every reward drops
+    to 0. A queue-based mock cannot express this either, since rollouts are collected concurrently
+    and arrival order is not the dataset order.
+
+    Stdlib rather than ``pytest-httpserver``: Gym reaches this endpoint from *subprocesses*, so it
+    has to be a real socket, and `packages/nemo_evaluator_sdk` does not depend on `nmp-testing`.
+    ``plugins/nemo-guardrails/tests/unit/benchmarks/test_shim.py`` sets the same precedent.
+    """
+
+    def __init__(self) -> None:
+        self._answers: dict[str, str] = {}
+        self._unmatched: list[str] = []
+        self.request_count = 0
+        server_self = self
+
+        class _Handler(BaseHTTPRequestHandler):
+            def do_POST(self) -> None:  # noqa: N802 - BaseHTTPRequestHandler's spelling
+                length = int(self.headers.get("Content-Length") or 0)
+                try:
+                    body = json.loads(self.rfile.read(length) or b"{}")
+                except ValueError:
+                    body = {}
+                content = server_self._answer_for(body)
+                payload = {
+                    "id": f"chatcmpl-{uuid.uuid4().hex}",
+                    "object": "chat.completion",
+                    "created": int(time.time()),
+                    "model": body.get("model") or "stub-model",
+                    "choices": [
+                        {"index": 0, "message": {"role": "assistant", "content": content}, "finish_reason": "stop"}
+                    ],
+                    # Non-zero on purpose: the runner reads token usage to tell a real attempt from
+                    # an agent that never called the model.
+                    "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
+                }
+                encoded = json.dumps(payload).encode("utf-8")
+                self.send_response(200)
+                self.send_header("Content-Type", "application/json")
+                self.send_header("Content-Length", str(len(encoded)))
+                self.end_headers()
+                self.wfile.write(encoded)
+
+            def log_message(self, format: str, *args: object) -> None:  # noqa: A002 - base signature
+                """Silence per-request logging to stderr; failures are asserted on, not read."""
+
+        self._server = ThreadingHTTPServer(("127.0.0.1", 0), _Handler)
+        self._thread = threading.Thread(target=self._server.serve_forever, daemon=True)
+
+    @staticmethod
+    def _prompt_text(body: Mapping[str, Any]) -> str:
+        """Every message's text, concatenated — matching is against what the model actually saw."""
+        parts: list[str] = []
+        for message in body.get("messages") or ():
+            content = message.get("content") if isinstance(message, Mapping) else None
+            if isinstance(content, str):
+                parts.append(content)
+            elif isinstance(content, Sequence):
+                parts.extend(
+                    block.get("text", "") for block in content if isinstance(block, Mapping) and block.get("text")
+                )
+        return "\n".join(parts)
+
+    def _answer_for(self, body: Mapping[str, Any]) -> str:
+        self.request_count += 1
+        text = self._prompt_text(body)
+        for needle, answer in self._answers.items():
+            if needle in text:
+                return answer
+        # Recorded rather than raised: a handler exception would surface as an opaque 500 inside
+        # Gym. The test asserts on this list, which names what went unmatched.
+        self._unmatched.append(text[-200:])
+        return _CANNED_ANSWER
+
+    def expect(self, prompt_fragment: str, answer: str) -> None:
+        """Answer any prompt containing ``prompt_fragment`` with ``answer``."""
+        self._answers[prompt_fragment] = answer
+
+    @property
+    def unmatched(self) -> list[str]:
+        return list(self._unmatched)
+
+    def __enter__(self) -> _StubPolicyServer:
+        self._thread.start()
+        return self
+
+    def __exit__(self, *_exc: object) -> None:
+        self._server.shutdown()
+        self._server.server_close()
+        self._thread.join(timeout=10)
+
+    @property
+    def base_url(self) -> str:
+        host, port = self._server.server_address[:2]
+        return f"http://{host}:{port}/v1"
+
 
 @dataclass(frozen=True)
 class GymEnvironmentCase:
@@ -91,6 +218,17 @@ class GymEnvironmentCase:
     #: where the standard `policy_model` wiring is enough; populated as the sweep discovers what an
     #: environment actually requires.
     env_overrides: Mapping[str, Any] = field(default_factory=dict)
+    #: Row field holding this environment's ground truth, and how it wants an answer spelled. When
+    #: set, the stub answers each prompt with that row's own correct answer and the rollout test can
+    #: assert a perfect score — which is what makes per-task attribution testable (see
+    #: :class:`_StubPolicyServer`). Left None where no answer is mechanically derivable (a judged
+    #: rubric, a Docker-executed verifier), and those cases assert only that trials completed.
+    answer_key: str | None = None
+    answer_format: Callable[[str], str] | None = None
+    #: Startup budget when the runner's 240s default is not enough. Set per environment rather than
+    #: raised globally: a long default would turn a genuinely wedged server into a long wait for
+    #: every environment, and the environments that need it are identifiable up front.
+    startup_timeout_s: float | None = None
 
     @property
     def id(self) -> str:
@@ -105,6 +243,9 @@ def id(self) -> str:
             "The environment every existing test is built on. Included so a failure elsewhere is "
             "attributable to that environment rather than to the runner or the local setup."
         ),
+        # Graded `strict_single_letter_boxed`, per the row's own `grading_mode`.
+        answer_key="expected_answer",
+        answer_format=lambda letter: f"\\boxed{{{letter}}}",
     ),
     GymEnvironmentCase(
         resources_server="gpqa_diamond",
@@ -113,6 +254,10 @@ def id(self) -> str:
             "Control. Same multiple-choice shape as mcqa with comparably small dependencies, so a "
             "failure here means we encoded mcqa's specifics rather than the category's."
         ),
+        # Same multiple-choice shape as mcqa, different spelling — its prompt asks for
+        # `Answer: LETTER`, not a boxed letter. Encoding both is the point of the control.
+        answer_key="expected_answer",
+        answer_format=lambda letter: f"Answer: {letter}",
     ),
     GymEnvironmentCase(
         resources_server="gdpval",
@@ -164,6 +309,34 @@ def id(self) -> str:
             "examples, so it is the one environment comparable across three runners."
         ),
         needs_docker=True,
+        # Measured: ~8 minutes from a cold cache. Startup installs this environment's dependencies
+        # and then prepares the Harbor task tree (`ensure_assets` + `hydrate_runtime_tasks`) before
+        # the server reports ready, which the 240s default does not cover. Observed as exactly the
+        # timeout this change made legible: "3 of 4 server(s) started; still waiting on:
+        # legal_agent_bench".
+        startup_timeout_s=1800.0,
+        # LAB's three asset directories default to paths *relative to the working directory*. The
+        # resources-server prepares the Harbor task tree under them and the agent reads it back via
+        # `local_dataset_path: ${...harbor_tasks_dir}` — the same key, so one override moves both
+        # sides — but a relative value only resolves if every server shares the cwd the data was
+        # written under, which is not guaranteed. Left relative, the agent dies with
+        # `FileNotFoundError: resources_servers/legal_agent_bench/data/runtime/harbor_tasks/...`,
+        # returns an empty response, and Gym records a rollout scoring 0.0 with no failure — a
+        # silent zero. Absolute paths remove the assumption entirely.
+        #
+        # `_ABS_ASSETS` is a placeholder: the rollout test substitutes a real tmp_path, since a
+        # committed absolute path would be wrong on every machine.
+        env_overrides={
+            "legal_agent_bench": {
+                "resources_servers": {
+                    "legal_agent_bench": {
+                        "harbor_tasks_dir": f"{_ABS_ASSETS_PLACEHOLDER}/runtime/harbor_tasks/legal_agent_bench",
+                        "harbor_tasks_cache_dir": f"{_ABS_ASSETS_PLACEHOLDER}/cache/harbor_tasks/legal_agent_bench",
+                        "harness_skills_dir": f"{_ABS_ASSETS_PLACEHOLDER}/cache/harness/skills",
+                    }
+                }
+            }
+        },
     ),
     GymEnvironmentCase(
         resources_server="wmt_translation",
@@ -248,7 +421,7 @@ def _require_environment(gym: str, case: GymEnvironmentCase) -> Path:
     return environment_dir
 
 
-def _selection(case: GymEnvironmentCase, hydra_dir: Path) -> list[str]:
+def _selection(case: GymEnvironmentCase, hydra_dir: Path, env_overrides: Mapping[str, Any] | None = None) -> list[str]:
     """The selection arguments the runner passes to both `gym env validate` and `gym env start`.
 
     ``hydra.run.dir`` mirrors what the runner does: Gym is a Hydra app and writes a timestamped run
@@ -265,11 +438,70 @@ def _selection(case: GymEnvironmentCase, hydra_dir: Path) -> list[str]:
     ]
     if case.bind_resources_server:
         argv.append(f"+{case.agent}.responses_api_agents.{case.agent}.resources_server.name={case.resources_server}")
-    argv.extend(_flatten_overrides(case.env_overrides))
+    # Resolved overrides when the caller has them, so validate checks the *same* values the rollout
+    # would run — an unresolved `_ABS_ASSETS` placeholder would validate a config nothing ever uses.
+    argv.extend(_flatten_overrides(case.env_overrides if env_overrides is None else env_overrides))
     argv.append(f"hydra.run.dir={hydra_dir}")
     return argv
 
 
+def _task_prompt(task: AgentEvalTask) -> str:
+    """The prompt text Gym will send for a task, recovered from the source row we materialize.
+
+    Read from ``inputs['gym_row']`` rather than ``intent``: for Gym rows ``intent`` is a dataset
+    label (``"Gym row from example.jsonl"``), identical across every row in a file, so keying an
+    oracle on it would collapse all tasks onto one entry.
+    """
+    row = task.inputs.get("gym_row") or {}
+    parts: list[str] = []
+    for item in row.get("input") or ():
+        content = item.get("content") if isinstance(item, Mapping) else None
+        if isinstance(content, str):
+            parts.append(content)
+        elif isinstance(content, Sequence):
+            parts.extend(block.get("text", "") for block in content if isinstance(block, Mapping) and block.get("text"))
+    return "\n".join(parts)
+
+
+#: How much of a prompt's tail identifies it. The head is shared boilerplate ("answer with a letter
+#: inside \boxed{}"), so a prefix would collide across rows; the tail carries the question and its
+#: options.
+_PROMPT_KEY_CHARS = 160
+
+
+def _prime_stub(stub: _StubPolicyServer, case: GymEnvironmentCase, tasks: Sequence[AgentEvalTask]) -> None:
+    """Teach the stub each task's own correct answer, for cases where one is derivable."""
+    if case.answer_key is None or case.answer_format is None:
+        return
+    keys: set[str] = set()
+    for task in tasks:
+        expected = task.metadata.get("gym_row_extras", {}).get(case.answer_key)
+        if expected is None:
+            pytest.skip(f"{case.resources_server}: rows carry no {case.answer_key!r} to build an oracle from")
+        key = _task_prompt(task)[-_PROMPT_KEY_CHARS:]
+        assert key, f"{case.resources_server}: recovered no prompt text for task {task.id}"
+        keys.add(key)
+        stub.expect(key, case.answer_format(str(expected)))
+    # A collision would silently answer one task with another's ground truth, which is precisely the
+    # bug this oracle exists to detect — so it must not be able to cause a false pass.
+    assert len(keys) == len(tasks), (
+        f"{case.resources_server}: prompt tails collide across tasks ({len(keys)} keys for "
+        f"{len(tasks)} tasks); raise _PROMPT_KEY_CHARS"
+    )
+
+
+def _case_overrides(case: GymEnvironmentCase, tmp_path: Path) -> dict[str, Any]:
+    """The case's own overrides, with the `_ABS_ASSETS` placeholder resolved to a real directory.
+
+    `legal_agent_bench`'s asset directories have to be absolute (see its case comment), but the
+    absolute path is per-run, so the table carries a placeholder the run substitutes.
+    """
+    assets = tmp_path / "assets"
+    assets.mkdir(parents=True, exist_ok=True)
+    rendered = json.dumps(case.env_overrides).replace(_ABS_ASSETS_PLACEHOLDER, str(assets))
+    return json.loads(rendered)
+
+
 @pytest.mark.parametrize("case", CASES, ids=[case.id for case in CASES])
 def test_gym_environment_config_validates(case: GymEnvironmentCase, tmp_path: Path) -> None:
     """`gym env validate` accepts the config the runner would run.
@@ -286,7 +518,7 @@ def test_gym_environment_config_validates(case: GymEnvironmentCase, tmp_path: Pa
     _environment_dir(gym, case)
 
     proc = subprocess.run(
-        [gym, "env", "validate", *_selection(case, tmp_path / "hydra")],
+        [gym, "env", "validate", *_selection(case, tmp_path / "hydra", _case_overrides(case, tmp_path))],
         capture_output=True,
         text=True,
         timeout=120,
@@ -307,8 +539,6 @@ async def test_gym_environment_runs_end_to_end(case: GymEnvironmentCase, tmp_pat
     """
     gym = _gym_cli()
     environment_dir = _require_environment(gym, case)
-    if not os.environ.get(POLICY_BASE_URL_ENV):
-        pytest.skip(f"{POLICY_BASE_URL_ENV} is not set; rollout collection needs a reachable model endpoint")
 
     dataset = environment_dir / "data" / "example.jsonl"
     if not dataset.exists():
@@ -318,22 +548,39 @@ async def test_gym_environment_runs_end_to_end(case: GymEnvironmentCase, tmp_pat
     assert tasks, f"{case.resources_server}: discover_gym_tasks found no rows in {dataset}"
     tasks = tasks[:_TASK_LIMIT]
 
-    runner = GymAgentTaskRunner(
-        config=GymRuntimeConfig(
-            agent=case.agent,
-            agent_config=case.agent_config,
-            resources_server=case.resources_server,
-            num_repeats=1,
-            bind_resources_server=case.bind_resources_server,
-            env_overrides=dict(case.env_overrides),
+    with _StubPolicyServer() as stub:
+        # An explicitly configured endpoint wins, so this suite can be pointed at a real model to
+        # produce meaningful scores. Unset, the stub serves the rollouts and the run needs nothing
+        # beyond an installed Gym.
+        policy_base_url = os.environ.get(POLICY_BASE_URL_ENV) or stub.base_url
+        using_stub = policy_base_url == stub.base_url
+        if using_stub:
+            _prime_stub(stub, case, tasks)
+
+        runner = GymAgentTaskRunner(
+            config=GymRuntimeConfig(
+                agent=case.agent,
+                agent_config=case.agent_config,
+                resources_server=case.resources_server,
+                num_repeats=1,
+                bind_resources_server=case.bind_resources_server,
+                # Omit rather than pass None, so the runner's own default stays the single source of
+                # truth for every environment that does not need more.
+                **({"startup_timeout_s": case.startup_timeout_s} if case.startup_timeout_s else {}),
+                env_overrides={
+                    "policy_base_url": policy_base_url,
+                    "policy_api_key": "stub-key",
+                    "policy_model_name": "stub-model",
+                    **_case_overrides(case, tmp_path),
+                },
+            )
         )
-    )
 
-    result = await AgentEvaluator().run(
-        tasks=tasks,
-        target=runner,
-        config=AgentEvalRunConfig(work_dir=tmp_path / "gym_run", parallelism=1),
-    )
+        result = await AgentEvaluator().run(
+            tasks=tasks,
+            target=runner,
+            config=AgentEvalRunConfig(work_dir=tmp_path / "gym_run", parallelism=1),
+        )
 
     assert result.summary.task_count == len(tasks)
     assert len(result.trials) == len(tasks)
@@ -343,3 +590,24 @@ async def test_gym_environment_runs_end_to_end(case: GymEnvironmentCase, tmp_pat
 
     assert runner.runner_info().name == "gym"
     assert result.scores, f"{case.resources_server}: no metric scores were produced"
+
+    if not using_stub:
+        return
+    # The model was genuinely called. Guards against a rollout path that "succeeds" without ever
+    # reaching the endpoint — the legal_agent_bench failure mode, where empty responses scored 0.0
+    # and read as a normal run.
+    assert stub.request_count > 0, f"{case.resources_server}: the stub was never called, so no agent ran"
+    if case.answer_key is None:
+        return
+    # Every prompt matched its own row, and every row therefore scored. A rollout attributed to the
+    # wrong task would have received another row's answer and scored 0.
+    assert not stub.unmatched, (
+        f"{case.resources_server}: {len(stub.unmatched)} prompt(s) matched no task; the runner sent "
+        f"content the oracle did not recognise: {stub.unmatched}"
+    )
+    rewards = [trial.metadata.get("reward") for trial in result.trials]
+    assert all(reward == 1.0 for reward in rewards), (
+        f"{case.resources_server}: every task was answered with its own ground truth, so each should "
+        f"score 1.0; got {rewards}. Unequal rewards here mean rollouts were attributed to the wrong "
+        "tasks rather than that the model was wrong."
+    )
diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_gym_runtime.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_gym_runtime.py
index 274bd24fc4..e777462e4d 100644
--- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_gym_runtime.py
+++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_gym_runtime.py
@@ -26,6 +26,7 @@
     GymAgentTaskRunner,
     GymRewardMetric,
     GymRuntimeConfig,
+    _agent_never_ran,
     _aggregate_metrics_path_for,
     _canonical_row_hash,
     _content_text,
@@ -35,6 +36,7 @@
     _gym_executable,
     _hydra_scalar,
     _materialize_dataset,
+    _pending_servers,
     _pump_stream,
     _read_run_aggregations,
     _render_instruction,
@@ -825,3 +827,292 @@ def test_gym_executable_reports_how_to_install_when_absent(monkeypatch: pytest.M
     monkeypatch.setattr(shutil, "which", lambda _: None)
     with pytest.raises(RuntimeError, match="own environment"):
         _gym_executable()
+
+
+#: Verbatim from a real `legal_agent_bench` startup timeout (AALGO-485 coverage sweep). Gym polls
+#: repeatedly, so the *last* line is the live one — earlier lines name servers that have since come
+#: up, and reporting those would send the reader after servers that are fine.
+_REAL_ENV_LOG = """\
+Waiting for servers to spin up
+Checking for HTTP server statuses.
+0 / 4 servers ready. Waiting for servers to spin up: ['simple_agent', 'legal_agent_bench', \
+'legal_agent_bench_harbor_agent', 'policy_model']
+2 / 4 servers ready. Waiting for servers to spin up: ['legal_agent_bench', 'policy_model']
+3 / 4 servers ready. Waiting for servers to spin up: ['legal_agent_bench']
+"""
+
+
+#: Verbatim shape of a `legal_agent_bench` rollout whose Harbor agent died before running: Gym
+#: recorded a normal-looking rollout with reward 0.0 and wrote nothing to the failures sidecar.
+_EMPTY_RESPONSE = {
+    "id": "resp_f63c4c78",
+    "model": "stub-model",
+    "object": "response",
+    "output": [],
+    "usage": {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0},
+}
+#: The same field for a rollout where the model really answered (mcqa).
+_REAL_RESPONSE = {
+    "id": "resp_aaa",
+    "object": "response",
+    "output": [{"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "\\boxed{B}"}]}],
+    "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2},
+}
+
+
+def _two_tasks_and_map(tmp_path: Path):
+    dataset = tmp_path / "ds.jsonl"
+    dataset.write_text(
+        json.dumps({"responses_create_params": {"input": "first question"}})
+        + "\n"
+        + json.dumps({"responses_create_params": {"input": "second question"}})
+        + "\n",
+        encoding="utf-8",
+    )
+    tasks = discover_gym_tasks(dataset)
+    return tasks, _materialize_dataset(tasks, tmp_path / "gym_input.jsonl")
+
+
+def test_agent_never_ran_detects_the_empty_zero_token_rollout() -> None:
+    assert _agent_never_ran({"response": _EMPTY_RESPONSE, "reward": 0.0}) is True
+
+
+def test_agent_never_ran_is_false_when_the_model_actually_answered() -> None:
+    assert _agent_never_ran({"response": _REAL_RESPONSE, "reward": 1.0}) is False
+
+
+def test_agent_never_ran_is_false_for_a_legitimately_empty_answer() -> None:
+    # A model may answer with nothing and earn 0.0. Tokens were still spent, so it ran — treating
+    # this as a failure would reclassify real (bad) results as broken ones.
+    spent = {"output": [], "usage": {"input_tokens": 120, "output_tokens": 0, "total_tokens": 120}}
+    assert _agent_never_ran({"response": spent, "reward": 0.0}) is False
+
+
+@pytest.mark.parametrize(
+    "output",
+    [
+        pytest.param("the agent answered in plain text", id="string"),
+        pytest.param({"text": "the agent answered"}, id="mapping"),
+        pytest.param([{"type": "message"}], id="list"),
+    ],
+)
+def test_agent_never_ran_is_false_for_any_non_empty_output_shape(output: object) -> None:
+    # `output` is not contractually a list. An earlier version special-cased lists, so a plain-string
+    # or single-mapping answer with zero reported tokens was misread as "the agent never ran" — which
+    # fails a trial that ran fine, and fails the whole run when every trial looks like that.
+    zero = {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}
+    assert _agent_never_ran({"response": {"output": output, "usage": zero}}) is False
+
+
+@pytest.mark.parametrize(
+    ("usage", "expected"),
+    [
+        # Chat Completions vocabulary: 12 prompt tokens prove the model was called, even though the
+        # Responses-API key names are absent. Reading only one vocabulary made this read as zero.
+        pytest.param({"prompt_tokens": 12, "completion_tokens": 0}, False, id="chat-vocab-nonzero"),
+        pytest.param({"prompt_tokens": 0, "completion_tokens": 0}, True, id="chat-vocab-zero"),
+        pytest.param({"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}, True, id="responses-vocab-zero"),
+        # Naming none of the recognized keys proves nothing either way, so do not reclassify.
+        pytest.param({}, False, id="empty-usage-proves-nothing"),
+        pytest.param({"cost_usd": 0.0}, False, id="unrecognized-keys-only"),
+        # Output-side counts alone prove nothing: zero output tokens is what an empty answer looks
+        # like. Only an input-side count can show the model was never reached, because a call always
+        # sends a prompt. These three shapes each used to be misread as "never ran".
+        pytest.param({"output_tokens": 0}, False, id="output-side-only"),
+        pytest.param({"completion_tokens": 0}, False, id="completion-only"),
+        pytest.param({"prompt_tokens": 12, "completion_tokens": 0}, False, id="real-call-empty-answer"),
+    ],
+)
+def test_agent_never_ran_requires_stated_zero_usage(usage: dict, expected: bool) -> None:
+    assert _agent_never_ran({"response": {"output": [], "usage": usage}}) is expected
+
+
+def test_agent_never_ran_is_false_without_usage_to_corroborate() -> None:
+    # No usage block means we cannot tell "never called" from "answered with nothing"; other runners
+    # populate `response` differently and must not be reclassified on a guess.
+    assert _agent_never_ran({"response": {"output": []}, "reward": 0.0}) is False
+    assert _agent_never_ran({"response": "Lyon", "reward": 0.0}) is False
+
+
+def test_all_rollouts_empty_fails_the_run_instead_of_reporting_zeros(tmp_path: Path) -> None:
+    # The legal_agent_bench case: "completed" trials scoring 0.0 that measured nothing at all.
+    # Two attempts per task, so trial and task counts differ and the message has to state both —
+    # "4 trials" alone reads as four tasks to anyone reasoning in tasks.
+    tasks, index_map = _two_tasks_and_map(tmp_path)
+    bundle = tmp_path / "rollouts.jsonl"
+    bundle.write_text(
+        "\n".join(
+            json.dumps({NG_TASK_INDEX: task, NG_ROLLOUT_INDEX: attempt, "response": _EMPTY_RESPONSE, "reward": 0.0})
+            for task in (0, 1)
+            for attempt in (0, 1)
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+
+    with pytest.raises(RuntimeError) as excinfo:
+        _trials_from_rollouts(bundle, tasks, index_map)
+
+    message = str(excinfo.value)
+    assert "never called" in message
+    # Counted in both units: 4 attempts across 2 tasks. Reporting only trials invites reading the
+    # run as four times bigger than it was.
+    assert "4 trial(s)" in message
+    assert "2 task(s)" in message
+    # Points at the log that carries the environment's own traceback, not just the results file.
+    assert "gym_env.log" in message
+    # Evaluator's vocabulary in the prose: "rollout" is Gym's word for an attempt, and this message
+    # is read by people working in trials and tasks. Paths are exempt — `rollouts.jsonl` is the real
+    # filename and has to be quotable verbatim to be useful.
+    prose = " ".join(word for word in message.split() if "/" not in word)
+    assert "rollout" not in prose.lower()
+
+
+def test_an_unattributed_rollout_prevents_the_all_empty_raise(tmp_path: Path) -> None:
+    # A record Gym collected but that carried no usable _ng_task_index never becomes a trial, so it
+    # is invisible to len(trials). It is still a rollout, and it may have run perfectly well —
+    # raising "nothing was measured" while one exists would be false.
+    tasks, index_map = _two_tasks_and_map(tmp_path)
+    bundle = tmp_path / "rollouts.jsonl"
+    bundle.write_text(
+        json.dumps({NG_TASK_INDEX: 0, NG_ROLLOUT_INDEX: 0, "response": _EMPTY_RESPONSE, "reward": 0.0})
+        + "\n"
+        # no NG_TASK_INDEX: unattributable, but a real rollout with real token usage
+        + json.dumps({NG_ROLLOUT_INDEX: 0, "response": _REAL_RESPONSE, "reward": 1.0})
+        + "\n",
+        encoding="utf-8",
+    )
+
+    trials = _trials_from_rollouts(bundle, tasks, index_map)
+
+    assert [trial.status for trial in trials] == [AgentEvalTrialStatus.FAILED]
+
+
+def test_startup_timeout_does_not_claim_an_unknown_server_when_gym_named_none(tmp_path: Path) -> None:
+    # Gym can print a readiness line with an empty list. Reporting "waiting on: unknown" contradicts
+    # the counts in the same sentence; say only what is known.
+    message = GymAgentTaskRunner(config=_config())._startup_timeout_message(
+        "0 / 4 servers ready. Waiting for servers to spin up: []", tmp_path / "gym_env.log"
+    )
+    assert "unknown" not in message
+    assert "did not name which remained outstanding" in message
+
+
+def test_empty_rollouts_alongside_a_sidecar_failure_do_not_raise(tmp_path: Path) -> None:
+    # A run that also recorded a real failure is not "nothing ran": that trial carries its own
+    # diagnosis (here a verifier timeout, from an attempt that may well have reached the model), and
+    # raising would discard it in favour of a vaguer message — while also miscounting the run as
+    # "All 1 rollout(s)" when two attempts happened.
+    tasks, index_map = _two_tasks_and_map(tmp_path)
+    bundle = tmp_path / "rollouts.jsonl"
+    bundle.write_text(
+        json.dumps({NG_TASK_INDEX: 0, NG_ROLLOUT_INDEX: 0, "response": _EMPTY_RESPONSE, "reward": 0.0}) + "\n",
+        encoding="utf-8",
+    )
+    (tmp_path / "rollouts_failures.jsonl").write_text(
+        json.dumps({NG_TASK_INDEX: 1, NG_ROLLOUT_INDEX: 0, "error": "verifier timeout"}) + "\n",
+        encoding="utf-8",
+    )
+
+    trials = _trials_from_rollouts(bundle, tasks, index_map)
+
+    assert {trial.task_id for trial in trials} == {tasks[0].id, tasks[1].id}
+    assert all(trial.status is AgentEvalTrialStatus.FAILED for trial in trials)
+    # The sidecar's own diagnosis survives rather than being replaced by the empty-rollout message.
+    sidecar = next(trial for trial in trials if trial.task_id == tasks[1].id)
+    assert sidecar.metadata["gym_failure"] == "verifier timeout"
+
+
+def test_some_rollouts_empty_fails_only_those_trials(tmp_path: Path) -> None:
+    # A partially broken run still measured something; the trials that ran are worth keeping.
+    tasks, index_map = _two_tasks_and_map(tmp_path)
+    bundle = tmp_path / "rollouts.jsonl"
+    bundle.write_text(
+        json.dumps({NG_TASK_INDEX: 0, NG_ROLLOUT_INDEX: 0, "response": _EMPTY_RESPONSE, "reward": 0.0})
+        + "\n"
+        + json.dumps({NG_TASK_INDEX: 1, NG_ROLLOUT_INDEX: 0, "response": _REAL_RESPONSE, "reward": 1.0})
+        + "\n",
+        encoding="utf-8",
+    )
+
+    trials = _trials_from_rollouts(bundle, tasks, index_map)
+
+    by_task = {trial.task_id: trial for trial in trials}
+    empty = by_task[tasks[0].id]
+    real = by_task[tasks[1].id]
+    assert empty.status is AgentEvalTrialStatus.FAILED
+    # The reward Gym reported is withheld so it cannot be averaged in as a real 0.0...
+    assert empty.metadata["reward"] is None
+    # ...but the reason is recorded.
+    assert "never called" in empty.metadata["gym_failure"]
+    assert real.status is AgentEvalTrialStatus.COMPLETED
+    assert real.metadata["reward"] == 1.0
+
+
+def test_pending_servers_reports_the_last_poll_not_the_first() -> None:
+    assert _pending_servers(_REAL_ENV_LOG) == (3, 4, ("legal_agent_bench",))
+
+
+def test_pending_servers_is_none_when_gym_never_reported_readiness() -> None:
+    # Gym died during composition or dependency install; there is no pending set to report, which
+    # must not be confused with "nothing pending".
+    assert _pending_servers("NeMo Gym is starting a new Ray cluster...\nerror: No solution found\n") is None
+
+
+def test_startup_timeout_names_the_server_that_did_not_come_up(tmp_path: Path) -> None:
+    # The bare "servers not ready" message sent the reader to a log dominated by the three servers
+    # that started fine. Gym knew which one was outstanding the whole time.
+    runner = GymAgentTaskRunner(
+        config=GymRuntimeConfig(
+            agent="simple_agent",
+            agent_config="cfg.yaml",
+            resources_server="legal_agent_bench",
+            startup_timeout_s=240.0,
+        )
+    )
+
+    message = runner._startup_timeout_message(_REAL_ENV_LOG, tmp_path / "gym_env.log")
+
+    assert "legal_agent_bench" in message
+    assert "3 of 4" in message
+    # The knob to change, not just the fact that a limit was hit.
+    assert "startup_timeout_s" in message
+    assert str(tmp_path / "gym_env.log") in message
+
+
+def test_startup_timeout_says_so_when_gym_never_reported_readiness(tmp_path: Path) -> None:
+    message = GymAgentTaskRunner(config=_config())._startup_timeout_message(
+        "NeMo Gym is starting a new Ray cluster...\n", tmp_path / "gym_env.log"
+    )
+    assert "never reported server readiness" in message
+    # Still names the environment, which is the one thing the reader always needs.
+    assert "mcqa" in message
+
+
+@pytest.mark.asyncio
+async def test_collection_failure_points_at_the_server_log_too(tmp_path: Path) -> None:
+    # Observed on wmt_translation: `gym eval run` reports a bare HTTP 500 from the resources-server
+    # while the traceback explaining it (PermissionError on a hardcoded /opt/Gym path) is only in
+    # gym_env.log. Naming just the two eval logs sends the reader to the one place the cause is not.
+    runner = GymAgentTaskRunner(config=_config())
+    gym = _stub_gym(tmp_path, exit_code=1, message="ClientResponseError: 500, Internal Server Error")
+
+    with pytest.raises(RuntimeError) as excinfo:
+        await runner._collect_rollouts(gym, tmp_path / "in.jsonl", tmp_path / "out.jsonl", tmp_path, {})
+
+    message = str(excinfo.value)
+    assert str(tmp_path / "gym_env.log") in message
+    assert str(tmp_path / "gym_eval.stdout.log") in message
+
+
+@pytest.mark.asyncio
+async def test_collection_redirects_hydra_output_under_the_run_work_dir(tmp_path: Path) -> None:
+    # `gym eval run` does not go through _selection_args, so it needs its own redirect — without it
+    # every collection drops an `outputs/<date>/<time>/` into the caller's cwd.
+    runner = GymAgentTaskRunner(config=_config())
+    gym = _stub_gym(tmp_path, exit_code=0, message="done")
+
+    await runner._collect_rollouts(gym, tmp_path / "in.jsonl", tmp_path / "out.jsonl", tmp_path, {})
+
+    argv = (tmp_path / "argv.txt").read_text().splitlines()
+    assert f"hydra.run.dir={tmp_path / 'gym_hydra'}" in argv
diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym_runtime.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym_runtime.py
index a395c5be73..af55a29af6 100644
--- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym_runtime.py
+++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym_runtime.py
@@ -109,6 +109,16 @@
 _VALIDATE_TIMEOUT_S = 120.0
 #: Where Gym's Hydra run directories are redirected, relative to the run's work dir.
 _HYDRA_SUBDIR = "gym_hydra"
+#: `gym env start`'s combined output, under the run's work dir. Named here because a *collection*
+#: failure often has to point at it: the eval logs show the symptom, this shows the cause.
+_ENV_LOG_NAME = "gym_env.log"
+#: Token-count keys read to decide whether a model was called at all. Both vocabularies, because
+#: Gym's model servers report in either depending on the adapter: the Responses API spells them
+#: `input_tokens`/`output_tokens`, Chat Completions `prompt_tokens`/`completion_tokens`.
+_TOKEN_USAGE_KEYS = ("total_tokens", "input_tokens", "output_tokens", "prompt_tokens", "completion_tokens")
+#: The subset that proves a call happened. A model call always consumes input tokens, so zero on any
+#: of these is evidence the model was never reached; zero output tokens only means an empty answer.
+_INPUT_TOKEN_KEYS = ("total_tokens", "input_tokens", "prompt_tokens")
 #: Gym's index fields on each rollout record. ``_ng_task_index`` is the only join back to the input
 #: rows that survives a round-trip: Gym mutates ``responses_create_params`` (even the prompt) and
 #: copies only a fixed allowlist of row keys onto the result, so no field we invent comes back. Gym
@@ -728,7 +738,7 @@ async def _run_two_step(self, input_path: Path, output_path: Path, work_dir: Pat
         """Start the Gym servers, collect against them with ``--no-serve``, then tear them down."""
         cfg = self._config
         gym = _gym_executable()
-        env_log = work_dir / "gym_env.log"
+        env_log = work_dir / _ENV_LOG_NAME
 
         # Gym launches each server from its own subdir with its own .venv. Ray (>=2.56) otherwise
         # detects a `uv run` ancestor and tries to replicate that uv project onto its workers,
@@ -791,6 +801,10 @@ async def _collect_rollouts(
             str(cfg.num_repeats),
             "--concurrency",
             str(cfg.concurrency),
+            # `gym eval run` is a Hydra app too, and unlike validate/start it does not go through
+            # _selection_args — so without this it writes `outputs/<date>/<time>/` into the caller's
+            # cwd on every single collection.
+            f"hydra.run.dir={work_dir / _HYDRA_SUBDIR}",
         ]
         stdout_log = work_dir / "gym_eval.stdout.log"
         stderr_log = work_dir / "gym_eval.stderr.log"
@@ -823,8 +837,17 @@ async def _collect_rollouts(
             await _drain_pumps(pumps, grace_s=cfg.shutdown_grace_s, what="gym eval run")
         if eval_proc.returncode != 0:
             tail = "\n".join(tails.get("stderr") or tails.get("stdout") or [])
+            # A collection failure is frequently *server*-side: the resources-server raises, the
+            # agent surfaces it as a bare HTTP 500, and the traceback that explains it is only in
+            # gym_env.log. Naming the two eval logs alone sends the reader to the one place the
+            # cause is not. (Observed on wmt_translation: a 500 here, `PermissionError: /opt/Gym`
+            # there.)
+            env_log = work_dir / _ENV_LOG_NAME
             raise RuntimeError(
-                f"`gym eval run` failed (rc={eval_proc.returncode}). Full output: {stdout_log} and {stderr_log}\n"
+                f"`gym eval run` failed (rc={eval_proc.returncode}). Collection output: {stdout_log} "
+                f"and {stderr_log}\n"
+                f"If the tail below is an HTTP error from a Gym server, the cause is server-side: "
+                f"see {env_log}, whose lines are prefixed with the server that emitted them.\n"
                 f"--- last {_LOG_TAIL_LINES} line(s) ---\n{tail}"
             )
 
@@ -839,6 +862,7 @@ async def _wait_for_servers(self, env_log: Path, env_proc: asyncio.subprocess.Pr
         loop = asyncio.get_running_loop()
         ready = re.compile(r"All \d+ / \d+ servers ready")
         deadline = loop.time() + cfg.startup_timeout_s
+        text = ""
         while loop.time() < deadline:
             if env_proc.returncode is not None:
                 raise RuntimeError(f"`gym env start` exited early (rc={env_proc.returncode}); see {env_log}")
@@ -848,7 +872,129 @@ async def _wait_for_servers(self, env_log: Path, env_proc: asyncio.subprocess.Pr
             if ready.search(text):
                 return
             await asyncio.sleep(2)
-        raise TimeoutError(f"Gym servers not ready within {cfg.startup_timeout_s}s; see {env_log}")
+        raise TimeoutError(self._startup_timeout_message(text, env_log))
+
+    def _startup_timeout_message(self, env_log_text: str, env_log: Path) -> str:
+        """Explain a startup timeout in terms of *which* server did not come up.
+
+        The bare "servers not ready" message sends you to a log that is mostly noise from the servers
+        that started fine. Gym polls and prints the outstanding set every time, so naming it costs
+        nothing and is usually the whole diagnosis: one server out of four means that environment's
+        own startup, not the platform.
+
+        The timeout is also a plausible *cause* rather than a symptom — a Docker-backed environment
+        builds its image on first run, which routinely exceeds the 240s default — so the message says
+        how to extend it instead of leaving the reader to find the knob.
+        """
+        cfg = self._config
+        pending = _pending_servers(env_log_text)
+        if pending is None:
+            detail = (
+                "Gym never reported server readiness, so it likely failed before starting them "
+                "(config composition or dependency installation)."
+            )
+        elif not pending[2]:
+            # Gym printed a readiness line but named no outstanding server. Saying "waiting on:
+            # unknown" would contradict the counts in the same sentence, so report only what is known.
+            ready_count, total, _ = pending
+            detail = (
+                f"{ready_count} of {total} server(s) started; Gym did not name which remained "
+                f"outstanding. The per-server output in {env_log.name} is prefixed with the server "
+                "that emitted each line."
+            )
+        else:
+            ready_count, total, names = pending
+            detail = (
+                f"{ready_count} of {total} server(s) started; still waiting on: {', '.join(names)}. "
+                f"Search {env_log.name} for '({names[0]})' — each line is prefixed with the server "
+                "that emitted it."
+            )
+        return (
+            f"Gym servers for resources-server {cfg.resources_server!r} were not ready within "
+            f"startup_timeout_s={cfg.startup_timeout_s}s. {detail}\n"
+            f"Full startup output: {env_log}\n"
+            "If the environment installs heavy dependencies or builds a container image on first run, "
+            "this timeout is expected on a cold cache — raise GymRuntimeConfig.startup_timeout_s and "
+            "re-run; the second run reuses what the first installed."
+        )
+
+
+#: Gym's per-poll readiness line, e.g.
+#: ``3 / 4 servers ready. Waiting for servers to spin up: ['legal_agent_bench']``.
+#: Gym knows exactly which servers are outstanding; without this we discard that and report only
+#: that *something* timed out.
+_PENDING_SERVERS_RE = re.compile(r"(\d+)\s*/\s*(\d+) servers ready\. Waiting for servers to spin up: \[([^\]]*)\]")
+
+
+def _pending_servers(text: str) -> tuple[int, int, tuple[str, ...]] | None:
+    """Recover ``(ready, total, still-pending)`` from the *last* readiness line Gym logged.
+
+    The last line is the one that matters: earlier polls list servers that have since come up.
+    Returns ``None`` when Gym never got as far as printing one (it died during composition, say),
+    which the caller must treat as "no detail available" rather than "nothing pending".
+
+    Never raises — this only ever decorates an error that is already being reported, and a parsing
+    failure here must not replace the real one. Gym's log format is not contractual.
+    """
+    matches = _PENDING_SERVERS_RE.findall(text)
+    if not matches:
+        return None
+    ready, total, names = matches[-1]
+    try:
+        return int(ready), int(total), tuple(re.findall(r"['\"]([^'\"]+)['\"]", names))
+    except ValueError:  # pragma: no cover - the regex only matches digits
+        return None
+
+
+def _agent_never_ran(record: Mapping[str, Any]) -> bool:
+    """True when a result record shows the agent produced nothing *and* never called the model.
+
+    A verifier-scored runner reports whatever score the environment computed, so an agent that
+    silently failed to start is indistinguishable from one that tried and scored 0.0 — the record
+    looks normal, no failure is reported alongside it, and the run succeeds. Observed on
+    ``legal_agent_bench``, whose agent died locating its task tree and returned an empty response:
+    two "completed" trials scoring 0.0, no model call, no failure recorded.
+
+    Deliberately conservative — **both** conditions must hold:
+
+    * no output, and
+    * a stated zero on the input side of token usage.
+
+    An empty answer alone is a legitimate result (a model can return nothing and earn 0.0), so it is
+    not sufficient. Zero *input* tokens is what says the model was never called: a call always sends
+    a prompt, whereas zero output tokens only describes an empty answer. A record carrying no
+    ``response`` key at all is left alone — some runners populate it differently, and this must not
+    reclassify trials it does not understand.
+    """
+    response = record.get("response")
+    if not isinstance(response, Mapping):
+        return False
+    if "output" not in response:
+        # Nothing to judge: some runners shape `response` differently, and absence is not emptiness.
+        return False
+    # Any non-empty value counts as output, whatever its shape. Checking `len(...) > 0` on a list
+    # alone would treat a plain string answer — or a single mapping — as "produced nothing" and fail
+    # a trial that ran perfectly well.
+    if response["output"]:
+        return False
+    usage = response.get("usage")
+    if not isinstance(usage, Mapping):
+        # No usage block to corroborate with; an empty output on its own is not enough to judge.
+        return False
+    # Zero usage has to be *stated*, not inferred from a key being absent. Two vocabularies are in
+    # play — the Responses API's `input_tokens`/`output_tokens` and Chat Completions'
+    # `prompt_tokens`/`completion_tokens` — and Gym's model servers speak both depending on the
+    # adapter. Reading only one set would make a real call reported in the other read as zero.
+    reported = [usage[key] for key in _TOKEN_USAGE_KEYS if isinstance(usage.get(key), (int, float))]
+    if any(value > 0 for value in reported):
+        return False
+    # The discriminating evidence is *input*-side, and only input-side. Calling a model always sends
+    # a prompt, so zero input tokens cannot describe a call that happened; zero **output** tokens
+    # merely describes an empty answer, which is a legitimate result. Requiring an input-side count
+    # rules out a whole class of false positives — a provider reporting only `output_tokens: 0`, or
+    # only `completion_tokens: 0` — rather than excluding those shapes one at a time.
+    input_side = [usage[key] for key in _INPUT_TOKEN_KEYS if isinstance(usage.get(key), (int, float))]
+    return bool(input_side) and all(value == 0 for value in input_side)
 
 
 def _source_datasets(tasks: Sequence[AgentEvalTask]) -> str:
@@ -1261,6 +1407,10 @@ def _trials_from_rollouts(
         descriptors={"rollouts": EvidenceDescriptor(kind="filesystem", format="file", ref=str(rollouts_path))}
     )
     unattributed_successes = 0
+    empty_output_count = 0
+    #: Tasks with at least one trial where the agent never ran. Tracked separately from trial
+    #: status, since the failures sidecar also produces FAILED trials for unrelated reasons.
+    empty_output_task_ids: set[str] = set()
     for record in _read_jsonl(rollouts_path):
         task_id = _resolve_task_id(record, index_to_task_id)
         if task_id is None:
@@ -1271,18 +1421,36 @@ def _trials_from_rollouts(
             continue
         raw_rollout_index = record.get(NG_ROLLOUT_INDEX)
         reward = _coerce_reward(record.get(reward_key))
+        # An agent that never ran did not earn its reward — reporting 0.0 as a completed trial would
+        # let a broken environment read as a poorly-performing one.
+        never_ran = _agent_never_ran(record)
+        if never_ran:
+            empty_output_count += 1
+            empty_output_task_ids.add(task_id)
+            status = AgentEvalTrialStatus.FAILED
+        elif reward is not None:
+            status = AgentEvalTrialStatus.COMPLETED
+        else:
+            status = AgentEvalTrialStatus.PARTIAL
+        metadata: dict[str, Any] = {
+            "reward": None if never_ran else reward,
+            NG_TASK_INDEX: record.get(NG_TASK_INDEX),
+            NG_ROLLOUT_INDEX: raw_rollout_index,
+        }
+        if never_ran:
+            # Kept alongside the reward Gym reported, so the discarded value is still auditable.
+            metadata["gym_failure"] = (
+                "the agent produced no output and consumed no tokens, so the model was never called; "
+                f"the reported score of {reward!r} is not a measurement of this agent"
+            )
         trials.append(
             AgentEvalTrial(
                 id=_rollout_trial_id(task_id, raw_rollout_index, synth_seq, missing_label="noidx"),
                 task_id=task_id,
-                status=AgentEvalTrialStatus.COMPLETED if reward is not None else AgentEvalTrialStatus.PARTIAL,
+                status=status,
                 output=AgentOutput(response=record.get("response"), metadata={"agent_ref": record.get("agent_ref")}),
                 evidence=success_evidence,
-                metadata={
-                    "reward": reward,
-                    NG_TASK_INDEX: record.get(NG_TASK_INDEX),
-                    NG_ROLLOUT_INDEX: raw_rollout_index,
-                },
+                metadata=metadata,
             )
         )
 
@@ -1321,6 +1489,40 @@ def _trials_from_rollouts(
                     },
                 )
             )
+    if empty_output_count:
+        # Every trial empty is a configuration failure, not a result: nothing was measured, so there
+        # is no score to report and letting the run "succeed" would publish zeros as findings. A
+        # partial count is left as failed trials — a flaky environment is still worth the attempts
+        # that did run.
+        #
+        # Measured against *every* trial, including those the failures sidecar contributed. A run
+        # that also recorded real failures is not the uniform "nothing ran" picture this raise is
+        # for: those trials carry their own diagnosis (a verifier timeout, say, from an attempt that
+        # did reach the model), and raising would discard it for a less specific message. Likewise
+        # `unattributed_successes` never became trials and so are invisible to `len(trials)`, but
+        # each is a result the environment produced and one may have run perfectly well — claiming
+        # nothing was measured while such a record exists would be false.
+        if empty_output_count == len(trials) and not unattributed_successes:
+            affected_tasks = {trial.task_id for trial in trials}
+            raise RuntimeError(
+                f"No agent ran: all {empty_output_count} trial(s) across all {len(affected_tasks)} "
+                "task(s) produced no output and consumed no tokens, so the model was never called "
+                "and nothing was measured. The scores reported for them describe an agent that did "
+                "not run.\n"
+                "This is normally the environment failing to start its agent rather than a bad "
+                f"model: check the per-server startup output in {rollouts_path.parent / _ENV_LOG_NAME} "
+                "for a traceback from the environment's own agent server.\n"
+                f"Results as collected: {rollouts_path}"
+            )
+        logger.warning(
+            "%d of %d trial(s), across %d of %d task(s), produced no agent output and consumed no "
+            "tokens; those are marked FAILED rather than scored, since the model was never called "
+            "for them.",
+            empty_output_count,
+            len(trials),
+            len(empty_output_task_ids),
+            len({trial.task_id for trial in trials}),
+        )
     if unattributed_successes:
         logger.warning(
             "%d Gym rollout record(s) in %s carried no usable %s and could not be attributed to a task; "

From 27c07aa551aaeccfa0c3b7a0498858a3e02bb86f Mon Sep 17 00:00:00 2001
From: Sandy Chapman <schapman@nvidia.com>
Date: Mon, 17 Aug 2026 09:09:50 -0300
Subject: [PATCH 2/2] refactor(evaluator): split gym_runtime into a package,
 address review
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Review feedback on #1295, from Nick and from CodeRabbit. The fixes and the
split are one commit because the split relocated the file the fixes changed,
so separating them afterwards would mean reconstructing a file that no longer
exists.

**The all-empty raise ignored unattributed failures.** The guard covered
unattributed *successes* but not failures. Both are invisible to
`len(trials)` for the same reason, neither becomes a trial, and a failure
record is a diagnosis in its own right — an OOM-killed agent, say. A run with
one empty rollout and one unattributed failure raised "No agent ran" despite
the environment having recorded something. Reproduced, fixed, regression test.

**The token-usage keys now have a checked source of truth.** The question was
whether they could come from Gym. They cannot: this runtime never imports
`nemo_gym` (it shells out, and we exclude Ray by constraint), and the names
are not Gym's — they are OpenAI's, from the Chat Completions and Responses
usage schemas, and `openai` is a dependency here. Added a test asserting the
tuple matches both schemas, rather than deriving it at import time: what we
match is wire-format JSON, stable regardless of how that package organises
its modules, and importing a nested path for five string literals would break
the runner if it ever moves.

**`hydra.run.dir` was the one override emitted unquoted.** Every other value
goes through `_hydra_scalar`; this one interpolated a `Path` directly.
Checked against Hydra's own parser: a work dir containing `[` raises
`OverrideParseException`, and — worse — one containing `,` silently parses as
a `ChoiceSweep`, so the run directory would be wrong rather than rejected.
Spaces turn out to be fine, so the risk is narrower than it first looks, but
the value is a path and paths are exactly where those characters appear. Now
quoted at both call sites, with a parametrized regression test.

**A missing results file raised a bare `FileNotFoundError`.** Sibling reads in
that module guard existence or degrade; `_trials_from_rollouts` did not. It is
reachable: `_ensure_fresh_output` deletes any stale file before the run and
`_collect_rollouts` already raised on a non-zero exit, so an absent file means
Gym reported success and wrote nothing. Now names the collection log and the
per-server startup log, like the other diagnostics here.

**The split.** At ~1600 lines the module had outgrown a single file. Now a
package named for the runtime, following the `fabric/` and `codex/` precedent
already in `runtimes/` — `runtime.py` as the entry point, siblings named for
what they own:

  config.py    GymRuntimeConfig and the Hydra grammar it serializes into
  records.py   Gym's on-disk artifacts and the index key that joins them
  dataset.py   Gym rows to tasks, and back to a dataset Gym collects against
  results.py   rollouts read back as trials and scores
  process.py   locating, watching and tearing down the gym CLI
  runtime.py   GymAgentTaskRunner — orchestration only

Largest module is now 543 lines. The relocation itself changes no behaviour:
code was moved by slicing line ranges rather than retyped, so the relocated
line count matches the original exactly.

`records.py` earns its place rather than being a leftovers bin. The
`_ng_task_index` / `_ng_rollout_index` pair is the attribution contract
between the half that *writes* it onto a materialized row (dataset) and the
half that *reads* it back off a rollout (results); one definition is what
stops the writer and reader disagreeing, and putting it in either module
would create a cycle. `config.py` likewise holds the config model together
with its Hydra serialization: splitting them needs a cycle, since
`_selection_args` takes a `GymRuntimeConfig`, and they are two halves of one
idea — what Gym is told, and how it is told.

The import path changes from `runtimes.gym_runtime` to `runtimes.gym`. Only
two non-test call sites existed, both updated; the public names re-export
from `__init__` unchanged. No compatibility shim, on the same reasoning as
the field rename in the follow-up: the runner is beta and young enough that a
permanent alias would cost more than it saves.

Signed-off-by: Sandy Chapman <schapman@nvidia.com>
---
 .../nemo_evaluator_sdk/examples/gym/README.md |    4 +-
 .../examples/gym/run_gym_eval.py              |    2 +-
 .../agent_eval/runtimes/gym/__init__.py       |   84 +
 .../agent_eval/runtimes/gym/config.py         |  260 +++
 .../agent_eval/runtimes/gym/dataset.py        |  249 +++
 .../agent_eval/runtimes/gym/process.py        |  188 ++
 .../agent_eval/runtimes/gym/records.py        |   52 +
 .../agent_eval/runtimes/gym/results.py        |  556 ++++++
 .../agent_eval/runtimes/gym/runtime.py        |  383 ++++
 .../agent_eval/runtimes/gym_runtime.py        | 1589 -----------------
 .../agent_eval/test_gym_aggregate_scores.py   |    2 +-
 .../test_gym_environment_coverage.py          |    8 +-
 .../tests/agent_eval/test_gym_runtime.py      |  127 +-
 .../tests/agent_eval/test_run_metadata.py     |    4 +-
 .../src/nemo_evaluator/jobs/agent_evaluate.py |    2 +-
 .../tests/test_agent_evaluate.py              |    2 +-
 .../agent_eval/runtimes/gym/__init__.py       |   84 +
 .../agent_eval/runtimes/gym/config.py         |  260 +++
 .../agent_eval/runtimes/gym/dataset.py        |  249 +++
 .../agent_eval/runtimes/gym/process.py        |  188 ++
 .../agent_eval/runtimes/gym/records.py        |   52 +
 .../agent_eval/runtimes/gym/results.py        |  556 ++++++
 .../agent_eval/runtimes/gym/runtime.py        |  383 ++++
 .../agent_eval/runtimes/gym_runtime.py        | 1589 -----------------
 24 files changed, 3664 insertions(+), 3209 deletions(-)
 create mode 100644 packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/__init__.py
 create mode 100644 packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/config.py
 create mode 100644 packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/dataset.py
 create mode 100644 packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/process.py
 create mode 100644 packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/records.py
 create mode 100644 packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/results.py
 create mode 100644 packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/runtime.py
 delete mode 100644 packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym_runtime.py
 create mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/__init__.py
 create mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/config.py
 create mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/dataset.py
 create mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/process.py
 create mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/records.py
 create mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/results.py
 create mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/runtime.py
 delete mode 100644 sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym_runtime.py

diff --git a/packages/nemo_evaluator_sdk/examples/gym/README.md b/packages/nemo_evaluator_sdk/examples/gym/README.md
index 5a5935a300..d1d5c6275f 100644
--- a/packages/nemo_evaluator_sdk/examples/gym/README.md
+++ b/packages/nemo_evaluator_sdk/examples/gym/README.md
@@ -95,10 +95,10 @@ The dataset handed to step 2 is not your source file. The runner **materializes*
 
 ### Logs
 
-Gym's subprocess output is streamed to files in the run's work directory — `gym_env.log` for `gym env start`, and `gym_eval.stdout.log` / `gym_eval.stderr.log` for the collection — and mirrored to the `nemo_evaluator_sdk.agent_eval.runtimes.gym_runtime` logger at `DEBUG`. Startup and collection failures name the relevant files and inline the last lines. To watch Gym's output in your own terminal, turn that logger up:
+Gym's subprocess output is streamed to files in the run's work directory — `gym_env.log` for `gym env start`, and `gym_eval.stdout.log` / `gym_eval.stderr.log` for the collection — and mirrored to the `nemo_evaluator_sdk.agent_eval.runtimes.gym` logger at `DEBUG`. Startup and collection failures name the relevant files and inline the last lines. To watch Gym's output in your own terminal, turn that logger up:
 
 ```python
-logging.getLogger("nemo_evaluator_sdk.agent_eval.runtimes.gym_runtime").setLevel(logging.DEBUG)
+logging.getLogger("nemo_evaluator_sdk.agent_eval.runtimes.gym").setLevel(logging.DEBUG)
 ```
 
 ## Notes & caveats
diff --git a/packages/nemo_evaluator_sdk/examples/gym/run_gym_eval.py b/packages/nemo_evaluator_sdk/examples/gym/run_gym_eval.py
index 3fa81ea4ca..18ee968f0b 100644
--- a/packages/nemo_evaluator_sdk/examples/gym/run_gym_eval.py
+++ b/packages/nemo_evaluator_sdk/examples/gym/run_gym_eval.py
@@ -38,7 +38,7 @@
 from pathlib import Path
 
 from nemo_evaluator_sdk.agent_eval.evaluator import AgentEvaluator
-from nemo_evaluator_sdk.agent_eval.runtimes.gym_runtime import (
+from nemo_evaluator_sdk.agent_eval.runtimes.gym import (
     GymAgentTaskRunner,
     GymRuntimeConfig,
     discover_gym_tasks,
diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/__init__.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/__init__.py
new file mode 100644
index 0000000000..6e63a9fb06
--- /dev/null
+++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/__init__.py
@@ -0,0 +1,84 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""NeMo Gym-backed :class:`AgentTaskRunner` for the agent-eval pipeline.
+
+Runs an *existing* NeMo Gym environment through its ``gym`` CLI and adapts the
+rollout bundle into SDK :class:`AgentEvalTrial` objects, so an
+:class:`AgentEvaluator` can score and report Gym runs through the same seam as
+Harbor/Fabric/Model. Gym owns execution *and* scoring; this runtime imports an
+existing Gym environment as-is — Gym is treated as a self-scoring engine and the
+runtime only adapts its rollout bundle, it does not re-derive rewards.
+
+**Mapping** (Gym → Evaluator): one Gym dataset → one run; each distinct row →
+one :class:`AgentEvalTask` (id = content hash of the row); each attempt
+(``_ng_rollout_index``) → one :class:`AgentEvalTrial`; the per-attempt verifier
+``reward`` → a :class:`GymRewardMetric` score. ``num_repeats=R`` therefore yields
+up to R trials per task. Row duplication is *not* a way to ask for repeated
+attempts — ``num_repeats`` is (see :func:`discover_gym_tasks`).
+
+**Attribution** is by ``_ng_task_index``, which this runtime *assigns* rather
+than infers. Gym only auto-assigns an index when a row doesn't already carry one
+(``rollout_collection._preprocess_rows_from_config``), and its own fallback
+dedup keys off the **raw jsonl line text** — a rule we cannot reproduce from
+parsed rows. So instead of guessing, :meth:`GymAgentTaskRunner.run_tasks`
+materializes a normalized dataset (one line per requested task, ``_ng_task_index``
+stamped explicitly) and feeds *that* to Gym. Gym echoes the index back on every
+rollout record, giving a total, order-independent ``index → task`` map. This also
+means a caller can run a **subset** of tasks without Gym rolling out the rest.
+
+**Execution** is the two-step Gym flow (the one that reads a dataset directly
+without triggering Gym's split-driven data-prep), preceded by a pre-flight:
+``gym env validate`` merges the composed config and reports unset ``???`` values,
+bad paths, and dangling cross-references without starting anything; then ``gym env
+start`` brings up the resources-server + agent + model servers, and ``gym eval run
+--no-serve --input <materialized dataset>`` collects rollouts against them. Both
+commands receive the identical selection arguments, so what is validated is what
+runs. The runtime shells out to the ``gym`` CLI on PATH, so this SDK never imports
+``nemo_gym``. Subprocess
+output is streamed to log files under the run's work dir *and* mirrored to this
+module's logger at ``DEBUG``, so callers choose terminal visibility through
+ordinary ``logging`` configuration.
+
+**Where Gym finds things.** NeMo Gym must be installed and its ``gym`` on PATH,
+along with the target environment's own dependencies. Generally that means a
+*separate* environment: Gym imports Ray at module load, and nemo-platform
+excludes Ray by constraint over an unfixed CVE, so the two cannot share one. In a
+job image the image owns PATH and this is unremarkable. There is deliberately no
+config field naming a checkout, a venv, or a search root — these runner configs
+become serialized job specs, and a local filesystem path means nothing on the
+other side of that boundary. Environments themselves ship in the ``nemo-gym``
+wheel (``resources_servers`` and friends install beside ``nemo_gym``, configs and
+example data included), so no checkout is needed to reach them.
+
+The subprocesses inherit this process's working directory, which is where Gym
+looks for the gitignored ``env.yaml`` holding the collector's credentials before
+falling back to its install root — so credentials never pass through this SDK.
+Run from the directory holding that file; a Gym checkout there also has its
+components take precedence, which is how you reach an environment the wheel does
+not carry.
+
+**Boundaries**: the caller is responsible for a
+Gym runtime whose deps are installed (each Gym env ships its own
+``requirements.txt``), and for handing a *ready-to-run* dataset file (``--no-serve
+--input`` bypasses Gym's prompt-templating/materialization). Service-side
+provisioning (docker/k8s, Ray) is out of scope here — that is the plugin's job.
+
+A consequence of that bypass: an environment whose rows carry no rendered prompt
+(``responses_create_params.input == []``, the prompt supplied by data-prep or by the
+environment's own agent) is still supported — the row travels through this runtime intact
+and the task simply has no ``inputs['instruction']``. See :func:`discover_gym_tasks`.
+"""
+
+from nemo_evaluator_sdk.agent_eval.runtimes.gym.config import DEFAULT_REWARD_KEY, GymRuntimeConfig
+from nemo_evaluator_sdk.agent_eval.runtimes.gym.dataset import discover_gym_tasks
+from nemo_evaluator_sdk.agent_eval.runtimes.gym.results import GymRewardMetric
+from nemo_evaluator_sdk.agent_eval.runtimes.gym.runtime import GymAgentTaskRunner
+
+__all__ = [
+    "DEFAULT_REWARD_KEY",
+    "GymAgentTaskRunner",
+    "GymRewardMetric",
+    "GymRuntimeConfig",
+    "discover_gym_tasks",
+]
diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/config.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/config.py
new file mode 100644
index 0000000000..21a3dabecd
--- /dev/null
+++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/config.py
@@ -0,0 +1,260 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Run configuration, and the Hydra grammar it is serialized into.
+
+Gym is a Hydra application, so every setting reaches it as an ``++dotted.path=value`` argument.
+That grammar is typed and unforgiving — quoting rules live here alongside the config model they
+serialize, so the two cannot drift. Redaction lives here too: these values are recorded as run
+provenance, and the override map is a free-form escape hatch a caller can put a credential into.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from collections.abc import Mapping
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+logger = logging.getLogger(__name__)
+
+
+DEFAULT_REWARD_KEY = "reward"
+#: Gym's CLI, expected on PATH. Not configurable: these runner configs become serialized job specs,
+#: and a path into somebody's venv is meaningless on the other side of that boundary. Note this name
+#: is only ever *resolved*, never executed: :func:`_gym_executable` turns it into an absolute path
+#: once, and that path is what the subprocesses run — so a child whose PATH differs from ours cannot
+#: end up executing a different Gym.
+_HYDRA_SUBDIR = "gym_hydra"
+#: `gym env start`'s combined output, under the run's work dir. Named here because a *collection*
+#: failure often has to point at it: the eval logs show the symptom, this shows the cause.
+_SECRET_KEY_MARKERS = ("api_key", "apikey", "token", "secret", "password", "passwd", "credential")
+#: Stand-in written in place of a redacted override value.
+_REDACTED = "<redacted>"
+
+
+#: Dict keys Hydra reads back unchanged. Its ``dictKey`` rule accepts no quoting, so a key is
+#: whatever the lexer makes of the bare text — this is deliberately narrower than what parses.
+_HYDRA_DICT_KEY = re.compile(r"[A-Za-z_][A-Za-z0-9_.-]*\Z")
+#: Bare words the lexer types rather than reading as text, so they cannot serve as string keys.
+_HYDRA_KEY_LITERALS = frozenset({"true", "false", "null", "inf", "nan"})
+
+
+def _hydra_dict(value: Mapping[str, Any]) -> str:
+    """Render a mapping as a Hydra dict container, ``{key:value,...}``.
+
+    Reached for a mapping nested inside a container — ``[{"b": 1}]`` — where there is no dotted path
+    to flatten onto, so the dict has to be spelled inline. Values recurse, so the typed spellings
+    below hold at any depth.
+
+    Keys are emitted bare, because Hydra's ``dictKey`` rule has no quoted form: ``{'b':1}`` does not
+    parse at all. That leaves the key at the mercy of the lexer, which types it — ``{true:1}`` keys
+    on the boolean ``True``, ``{1.5:1}`` on a float — and rejects ``:``, ``,``, brackets, and quotes
+    outright. Anything outside the conservative shape above therefore raises here rather than
+    silently keying the config on something the caller did not write.
+    """
+    rendered = []
+    for key, item in value.items():
+        if not isinstance(key, str) or not _HYDRA_DICT_KEY.match(key) or key.casefold() in _HYDRA_KEY_LITERALS:
+            raise ValueError(
+                f"Gym config override has dict key {key!r}, which Hydra's override grammar cannot "
+                "express as a string: keys are unquoted, so only a leading letter or underscore "
+                "followed by letters, digits, '_', '.', or '-' survives the round trip. Set this "
+                "key through the override path instead of nesting it inside a list."
+            )
+        rendered.append(f"{key}:{_hydra_scalar(item)}")
+    return "{" + ",".join(rendered) + "}"
+
+
+def _hydra_scalar(value: Any) -> str:
+    """Render a leaf value the way Hydra's override grammar reads it back.
+
+    Hydra's grammar is typed, so an unquoted string is not necessarily a string: ``true`` parses as a
+    boolean, ``null`` as ``None``, ``1.5`` as a float, ``a,b`` as a *sweep*, and ``A[B`` fails to
+    parse outright. Strings are therefore always single-quoted, which round-trips every one of those
+    (verified against ``hydra.core.override_parser``). Interpolations survive quoting — the override
+    sets the literal text and OmegaConf resolves it on read — so ``${policy_base_url}`` still works.
+
+    Only ``'`` is escaped. Hydra does **not** decode ``\\\\`` inside a quoted value: escaping
+    backslashes doubles them, so they are passed through raw.
+
+    ``None`` and booleans get their own spellings, since ``str()`` would emit ``"None"``/``"True"``
+    and Hydra reads those back as text. Containers recurse for the same reason: ``str()`` on a dict
+    emits Python's repr, whose quoted keys Hydra rejects outright.
+    """
+    if value is None:
+        return "null"
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    if isinstance(value, Mapping):
+        return _hydra_dict(value)
+    if isinstance(value, (list, tuple)):
+        return "[" + ",".join(_hydra_scalar(item) for item in value) + "]"
+    if isinstance(value, str):
+        # A trailing backslash would escape the closing quote and leave the value unterminated, and
+        # there is no spelling that avoids it — better to say so than to emit something unparseable.
+        if value.endswith("\\"):
+            raise ValueError(
+                f"Gym config override value {value!r} ends with a backslash, which Hydra's override "
+                "grammar cannot express: it escapes the closing quote."
+            )
+        return "'" + value.replace("'", "\\'") + "'"
+    return str(value)
+
+
+def _flatten_overrides(overrides: Mapping[str, Any], _prefix: str = "") -> list[str]:
+    """Flatten a nested override mapping into Hydra ``++dotted.path=value`` arguments.
+
+    Callers describe overrides as structured data — ``{"a": {"b": 1}}`` — rather than as
+    pre-serialized Hydra strings, so the config survives being sent somewhere as JSON. Hydra itself
+    only speaks the flat form, so the translation happens here, at the point of invocation.
+
+    ``++`` rather than ``+``: it sets a key whether or not it already exists, which is what an
+    override means. A bare ``+`` fails on a key the merged config already defines.
+    """
+    arguments: list[str] = []
+    for key, value in overrides.items():
+        path = f"{_prefix}{key}"
+        # An empty mapping has no leaves to descend to, so recursing would drop the override
+        # entirely. It is still a value the caller asked to set: emit it as ``++path={}``, which
+        # clears the subtree.
+        if isinstance(value, Mapping) and value:
+            arguments.extend(_flatten_overrides(value, f"{path}."))
+        else:
+            arguments.append(f"++{path}={_hydra_scalar(value)}")
+    return arguments
+
+
+def _redact_env_overrides(overrides: Mapping[str, Any], _prefix: str = "") -> dict[str, Any]:
+    """Redact credential-looking values from overrides before they are recorded as provenance.
+
+    ``env_overrides`` is a free-form escape hatch forwarded to Gym, so nothing stops a caller passing
+    ``{"model": {"api_key": "sk-..."}}``. ``RunnerInfo.config`` is persisted into the run bundle, so a
+    value that looks like a credential must not be written there.
+
+    The *key* is always kept — knowing that a run overrode ``model.api_key`` is useful provenance;
+    knowing the value is a leak. Matching is on the full dotted path, so a marker anywhere in it
+    redacts, and nesting cannot hide a credential behind an innocuous leaf name.
+
+    Lists are walked too, since a mapping inside one — ``{"models": [{"api_key": "sk-..."}]}`` —
+    reaches Gym just as a nested mapping does. The index contributes no path segment: what marks a
+    value as a credential is the key it sits under, not where in a list it happens to fall.
+    """
+    redacted: dict[str, Any] = {}
+    for key, value in overrides.items():
+        path = f"{_prefix}{key}"
+        if isinstance(value, Mapping):
+            redacted[key] = _redact_env_overrides(value, f"{path}.")
+        elif any(marker in path.casefold() for marker in _SECRET_KEY_MARKERS):
+            redacted[key] = _REDACTED
+        elif isinstance(value, (list, tuple)):
+            redacted[key] = [_redact_list_item(item, path) for item in value]
+        else:
+            redacted[key] = value
+    return redacted
+
+
+def _redact_list_item(item: Any, path: str) -> Any:
+    """Redact inside one element of a list-valued override. See :func:`_redact_env_overrides`."""
+    if isinstance(item, Mapping):
+        return _redact_env_overrides(item, f"{path}.")
+    if isinstance(item, (list, tuple)):
+        return [_redact_list_item(nested, path) for nested in item]
+    return item
+
+
+def _selection_args(config: GymRuntimeConfig, work_dir: Path) -> list[str]:
+    """The environment/agent/model selection passed to Gym.
+
+    Built once and handed verbatim to both ``gym env validate`` and ``gym env start``, so what is
+    validated is exactly what runs — a pre-flight against a different config would be worse than
+    none.
+    """
+    selection = [
+        "--config",
+        config.agent_config,
+        "--model-type",
+        config.model_type,
+        "--resources-server",
+        config.resources_server,
+    ]
+    if config.bind_resources_server:
+        # Composable (Pattern-A) agents leave resources_server.name unbound ('???'); bind it to the
+        # env we're running. Assumes the agent config's top-level key equals the agent name (the
+        # simple_agent convention) *and* that the resources-server is registered under the
+        # environment's own name — not universally true, so self-contained or differently-named
+        # servers set bind_resources_server=False and bind themselves via env_overrides.
+        selection.append(
+            f"+{config.agent}.responses_api_agents.{config.agent}.resources_server.name={config.resources_server}"
+        )
+    selection.extend(_flatten_overrides(config.env_overrides))
+    # Gym is a Hydra app, so each invocation writes a timestamped run directory — by default
+    # `outputs/<date>/<time>/` under the *current* directory. Since the subprocesses inherit this
+    # process's cwd (so Gym can find env.yaml), the default would litter whatever directory the
+    # caller happened to run from. Redirect it under the run's work dir, with the rest of the run's
+    # artifacts. Applies to every Gym entry point, `gym list` included.
+    #
+    # Quoted like any other override value: this one is a *path*, and Hydra's grammar reads `,` as a
+    # sweep separator and `[` as a list opener, so an unquoted work dir containing either is
+    # silently misread or rejected outright. Verified against Hydra's own parser — unquoted,
+    # `/tmp/a,b` becomes a ChoiceSweep and `/tmp/x[1]` raises OverrideParseException.
+    selection.append(f"hydra.run.dir={_hydra_scalar(str(work_dir / _HYDRA_SUBDIR))}")
+    return selection
+
+
+class GymRuntimeConfig(BaseModel):
+    """Declarative config for running an existing Gym environment via the ``gym`` CLI.
+
+    Holds only plain fields; the two-step invocation is built from them at run
+    time. The dataset itself is recovered from the tasks (stamped by
+    :func:`discover_gym_tasks`), mirroring the Harbor runner.
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    agent: str = Field(description="Agent name to collect rollouts with, e.g. 'simple_agent'.")
+    agent_config: str = Field(description="Repo-relative agent config passed to `gym env start` (--config).")
+    resources_server: str = Field(description="Resources-server (environment) name, e.g. 'mcqa' (--resources-server).")
+    model_type: str = Field(
+        default="inference_provider",
+        description="Model-type config (--model-type). `inference_provider` speaks OpenAI-compatible chat; "
+        "`openai_model` uses the OpenAI Responses API (500s against chat-only endpoints).",
+    )
+    bind_resources_server: bool = Field(
+        default=True,
+        description="Auto-bind the agent's `resources_server.name` to `resources_server` via a Hydra override "
+        "(the composable/Pattern-A agent case, e.g. simple_agent whose config leaves it '???'). Set False for "
+        "self-contained agents that already bind their own resources-server.",
+    )
+    env_overrides: dict[str, Any] = Field(
+        default_factory=dict,
+        description="Nested config overrides merged into Gym's config, applied after the auto-derived "
+        "resources-server binding. Structured rather than pre-serialized Hydra strings so the config "
+        "travels as JSON — `{'a': {'b': 1}}` becomes `++a.b=1` at invocation. This is the escape "
+        "hatch for what Gym does not standardize: an environment whose resources-server is registered "
+        "under a different name, or which references a model server no shipped config defines.",
+    )
+    num_repeats: int = Field(default=1, ge=1, description="Attempts per row; each attempt becomes one trial.")
+    concurrency: int = Field(
+        default=4,
+        ge=1,
+        description="Concurrent rollouts for `gym eval run` (the collection-phase knob, tuned to the model "
+        "endpoint's limits). Distinct from AgentEvalRunConfig.parallelism, which bounds concurrent scoring.",
+    )
+    startup_timeout_s: float = Field(default=240.0, gt=0, description="Max wait for `gym env start` readiness.")
+    collection_timeout_s: float | None = Field(
+        default=None,
+        gt=0,
+        description="Max wait for `gym eval run` collection; None = unbounded (scales with dataset x num_repeats x "
+        "model latency, so no safe fixed default). Set it to bound a hung/slow model endpoint.",
+    )
+    shutdown_grace_s: float = Field(
+        default=30.0,
+        gt=0,
+        description="Grace period for a Gym subprocess *group* to exit on SIGTERM (letting Ray shut down cleanly) "
+        "before escalating to SIGKILL.",
+    )
+    reward_key: str = Field(default=DEFAULT_REWARD_KEY, description="Key read from each rollout record.")
diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/dataset.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/dataset.py
new file mode 100644
index 0000000000..d8dac4aeac
--- /dev/null
+++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/dataset.py
@@ -0,0 +1,249 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Gym rows to Evaluator tasks, and back to a dataset Gym will collect against.
+
+Task identity is the content hash of a row, and attribution is an index this module *assigns*
+rather than infers — see :func:`_materialize_dataset`. Both halves live together because they are
+two directions of one translation: what breaks one silently breaks the other.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+from collections.abc import Mapping, Sequence
+from pathlib import Path
+from typing import Any
+
+from nemo_evaluator_sdk.agent_eval.runtimes.gym.records import _RUNTIME_KEYS, NG_TASK_INDEX, _read_jsonl
+from nemo_evaluator_sdk.agent_eval.runtimes.gym.results import GymRewardMetric
+from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalTask
+
+logger = logging.getLogger(__name__)
+
+
+def _canonical_row_hash(row: Mapping[str, Any]) -> str:
+    """Stable ``sha256`` of a Gym dataset row, excluding runtime-injected fields.
+
+    A given row keeps its id across dataset revisions/reorderings (so Intake
+    rollups stay consistent), and a changed row becomes a new task. No
+    dataset-revision component — identity is the row content alone.
+    """
+    payload = {key: value for key, value in row.items() if key not in _RUNTIME_KEYS}
+    encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str)
+    return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
+
+
+def _content_text(content: Any) -> str:
+    """Flatten a message ``content`` (a string, or a list of ``{type,text}`` parts) to text.
+
+    Anything else contributes nothing. A missing ``content`` is unremarkable, but a *populated* value
+    in some other shape means we're silently dropping prompt text, so warn — :func:`_render_instruction`
+    only fails loudly when the whole row renders empty, which a single skipped message wouldn't trigger.
+    """
+    if isinstance(content, str):
+        return content
+    # `str` is itself a Sequence, so this branch only ever sees non-str sequences (handled above).
+    if isinstance(content, Sequence):
+        parts = [part["text"] for part in content if isinstance(part, Mapping) and isinstance(part.get("text"), str)]
+        return "\n".join(parts)
+    if content is not None:
+        logger.warning(
+            "Ignoring message content of unsupported type %s (expected a string or a list of {type,text} parts); "
+            "any prompt text it carries will be missing from the task instruction.",
+            type(content).__name__,
+        )
+    return ""
+
+
+def _render_instruction(responses_create_params: Mapping[str, Any]) -> str:
+    """Derive a task instruction from a row's ``responses_create_params``, or ``""`` when it carries none.
+
+    Uses ``instructions`` (system prompt, when present) + ``input`` (a plain string
+    or an OpenAI message list).
+
+    An empty result is *not* an error. A whole class of Gym environments ships
+    ``"responses_create_params": {"input": []}`` and keeps the task elsewhere — ``gdpval``
+    in a top-level ``prompt``, ``legal_agent_bench`` in an ``instance_id`` its own agent
+    resolves, ``aviary``/``toolsandbox`` in nothing but a ``task_idx``. Their prompt is
+    materialized by Gym's data-preparation step or by the environment's agent, neither of
+    which this runner drives (``--no-serve --input`` hands Gym a ready-to-run dataset by
+    design). Nothing here can reconstruct it, and no single row field generalizes across
+    them, so the honest answer is no instruction rather than a guess. See
+    :func:`discover_gym_tasks` for what that means downstream.
+    """
+    parts: list[str] = []
+    instructions = responses_create_params.get("instructions")
+    if isinstance(instructions, str) and instructions.strip():
+        parts.append(instructions.strip())
+    raw_input = responses_create_params.get("input")
+    if isinstance(raw_input, str):
+        parts.append(raw_input)
+    elif isinstance(raw_input, Sequence):
+        for item in raw_input:
+            if isinstance(item, Mapping):
+                text = _content_text(item.get("content"))
+                if text:
+                    parts.append(text)
+    return "\n\n".join(part for part in parts if part).strip()
+
+
+def discover_gym_tasks(dataset: str | Path, *, metrics: Sequence[Any] | None = None) -> list[AgentEvalTask]:
+    """Build one :class:`AgentEvalTask` per distinct row in a Gym dataset (jsonl).
+
+    Each task's id is the content hash of the row; its ``instruction`` is rendered
+    from ``responses_create_params.input`` *when the row carries one*; the raw params
+    are stashed under ``inputs['gym_row']`` for provenance; and it is scored by a
+    :class:`GymRewardMetric`. The dataset path is stamped on
+    ``metadata['gym_dataset_path']``, and every *other* row key (the verifier's
+    ground-truth fields, ``agent_ref``, and so on) on ``metadata['gym_row_extras']``.
+    Together with ``inputs['gym_row']`` those reconstruct the complete source row,
+    which :class:`GymAgentTaskRunner` re-materializes into the dataset it hands to Gym.
+    They are kept disjoint deliberately: the task record is persisted to the run
+    bundle's ``tasks.jsonl``, and ``responses_create_params`` is usually the bulk of a
+    row, so storing it in both places would write every dataset twice per run.
+
+    **One distinct row is one task.** Duplicate rows collapse (identity is row content,
+    so they are by definition the same task) and are reported as a warning: repeated
+    attempts are a run-level concern — ``GymRuntimeConfig.num_repeats`` — not something
+    a dataset expresses by repeating a row, so duplicates almost always mean bad data.
+
+    **A row need not carry a prompt.** A sweep of the 106 built-in environments shipping example
+    data found 5 — ``aviary``, ``gdpval``, ``legal_agent_bench``, ``scicode`` and ``toolsandbox``
+    — whose every row ships ``responses_create_params.input == []``, letting Gym's data-prep step
+    or the environment's own agent materialize the prompt instead (see
+    :func:`_render_instruction`). Such a row yields a task with **no** ``inputs['instruction']``
+    key rather than a fabricated one, and is counted in a summary log line. This costs the
+    Gym path nothing: the runner never reads ``instruction`` — it re-materializes the source
+    row from ``inputs['gym_row']`` + ``metadata['gym_row_extras']`` and hands *that* to Gym,
+    and ``intent`` is a dataset label either way. It stays absent (not ``""``) so that
+    :meth:`AgentEvalTask.agent_prompt` still fails loudly for the runners that *do* need a
+    prompt — an instruction-less task must not reach an agent as an empty one.
+
+    ``responses_create_params`` itself remains required: Gym indexes into it unconditionally
+    (``rollout_collection._preprocess_rows_from_config``), so a row without it is rejected here
+    rather than crashing Gym mid-collection.
+    """
+    dataset = Path(dataset)
+    tasks: list[AgentEvalTask] = []
+    seen: set[str] = set()
+    duplicates = 0
+    promptless = 0
+    for position, row in enumerate(_read_jsonl(dataset), 1):
+        params = row.get("responses_create_params")
+        if not isinstance(params, Mapping):
+            raise ValueError(
+                f"row {position} of {dataset} has no 'responses_create_params' mapping; Gym requires that key "
+                "on every dataset row"
+            )
+        task_id = _canonical_row_hash(row)
+        if task_id in seen:
+            duplicates += 1
+            continue
+        seen.add(task_id)
+        instruction = _render_instruction(params)
+        if not instruction:
+            promptless += 1
+        tasks.append(
+            AgentEvalTask(
+                id=task_id,
+                intent=f"Gym row from {dataset.name}",
+                inputs={
+                    # Absent, not empty, when the row carries no prompt — see the docstring.
+                    **({"instruction": instruction} if instruction else {}),
+                    "gym_row": params,
+                },
+                metrics=list(metrics) if metrics is not None else [GymRewardMetric()],
+                metadata={
+                    "gym_dataset_path": str(dataset),
+                    # Everything except responses_create_params, which already lives in inputs['gym_row'].
+                    "gym_row_extras": {
+                        key: value
+                        for key, value in row.items()
+                        if key != "responses_create_params" and key not in _RUNTIME_KEYS
+                    },
+                },
+            )
+        )
+    if duplicates:
+        logger.warning(
+            "Collapsed %d duplicate row(s) in %s into %d distinct task(s): task identity is row content, so "
+            "repeating a row cannot request repeated attempts — set GymRuntimeConfig.num_repeats for that. "
+            "Duplicate rows usually indicate a data problem.",
+            duplicates,
+            dataset,
+            len(tasks),
+        )
+    if promptless:
+        logger.info(
+            "%d of %d task(s) in %s carry no prompt in responses_create_params and were discovered without an "
+            "inputs['instruction']. Expected for environments whose prompt is built by Gym's data-prep step or by "
+            "their own agent (e.g. aviary, gdpval, legal_agent_bench, scicode, toolsandbox); the Gym runner does "
+            "not read the instruction. A runner that hands the task straight to an agent will reject these tasks.",
+            promptless,
+            len(tasks),
+            dataset,
+        )
+    if not tasks:
+        raise ValueError(f"no rows found in Gym dataset {dataset}")
+    return tasks
+
+
+def _source_datasets(tasks: Sequence[AgentEvalTask]) -> str:
+    """Human-readable summary of the dataset(s) the tasks were discovered from, for a log line.
+
+    Purely provenance. Gym reads the dataset :func:`_materialize_dataset` writes from the tasks
+    themselves, so nothing here gates the run: tasks drawn from two datasets are legitimate (the
+    materialized file is their union), and a task with no stamped path runs fine. Never raises —
+    a missing or inconsistent label is not a reason to fail an otherwise valid evaluation.
+    """
+    stamped = {
+        task.metadata["gym_dataset_path"]
+        for task in tasks
+        if isinstance(task.metadata.get("gym_dataset_path"), str) and task.metadata["gym_dataset_path"]
+    }
+    return ", ".join(sorted(stamped)) if stamped else "<tasks with no stamped dataset path>"
+
+
+def _materialize_dataset(tasks: Sequence[AgentEvalTask], dest: Path) -> dict[int, str]:
+    """Write the normalized dataset Gym will read, and return its ``_ng_task_index`` → task-id map.
+
+    One line per requested task, in task order, carrying the task's full source row plus an explicitly
+    stamped ``_ng_task_index``. Gym honors a pre-supplied index instead of deriving one, so this makes
+    the rollout→task join total and order-independent, and confines the run to the tasks we asked for.
+
+    The row is reassembled from ``inputs['gym_row']`` (``responses_create_params``) and
+    ``metadata['gym_row_extras']`` (everything else), which :func:`discover_gym_tasks` keeps disjoint
+    so the run bundle doesn't store the same payload twice. Any pre-existing ``_ng_*`` fields are
+    stripped: ours is authoritative, and Gym assigns ``_ng_rollout_index`` itself per attempt.
+    """
+    index_to_task_id: dict[int, str] = {}
+    seen_task_ids: set[str] = set()
+    lines: list[str] = []
+    for index, task in enumerate(tasks):
+        # The source row is stored split across inputs/metadata so the run bundle doesn't persist
+        # responses_create_params twice; reassemble it here.
+        extras = task.metadata.get("gym_row_extras")
+        params = task.inputs.get("gym_row")
+        if not isinstance(extras, Mapping) or not isinstance(params, Mapping):
+            raise ValueError(
+                f"task {task.id!r} is missing inputs['gym_row'] and/or metadata['gym_row_extras']; build tasks "
+                "with discover_gym_tasks so the Gym dataset can be re-materialized for the run"
+            )
+        if task.id in seen_task_ids:
+            raise ValueError(
+                f"task {task.id!r} was supplied more than once; one distinct row is one task, and repeated "
+                "attempts come from GymRuntimeConfig.num_repeats"
+            )
+        seen_task_ids.add(task.id)
+        payload = {key: value for key, value in extras.items() if key not in _RUNTIME_KEYS}
+        payload["responses_create_params"] = params
+        payload[NG_TASK_INDEX] = index
+        lines.append(json.dumps(payload, default=str))
+        index_to_task_id[index] = task.id
+    if not index_to_task_id:
+        raise ValueError("GymAgentTaskRunner was given no tasks to run")
+    dest.write_text("\n".join(lines) + "\n", encoding="utf-8")
+    return index_to_task_id
diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/process.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/process.py
new file mode 100644
index 0000000000..b4871a9891
--- /dev/null
+++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/process.py
@@ -0,0 +1,188 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Running the ``gym`` CLI: locating it, watching it, and tearing it down.
+
+Gym spawns a Ray cluster and several uvicorn servers, so a subprocess here is a process *tree*.
+Everything about that lifecycle — resolution, streamed logs, readiness parsing, group teardown —
+lives here so the runtime module can read as orchestration rather than process management.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+import re
+import shutil
+import signal
+from collections import deque
+from collections.abc import Sequence
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+_GYM_CLI = "gym"
+#: Bound on `gym env validate`. It merges config without starting anything and returns in about a
+#: second; this only exists so a wedged invocation cannot stall the run before it begins.
+_VALIDATE_TIMEOUT_S = 120.0
+#: Where Gym's Hydra run directories are redirected, relative to the run's work dir.
+#: Token-count keys read to decide whether a model was called at all. Both vocabularies, because
+#: Gym's model servers report in either depending on the adapter: the Responses API spells them
+#: `input_tokens`/`output_tokens`, Chat Completions `prompt_tokens`/`completion_tokens`.
+_LOG_TAIL_LINES = 40
+
+
+#: Substrings that mark an override as carrying a credential. Matched case-insensitively against the
+#: full dotted path, so nesting cannot hide one behind an innocuous leaf name.
+def _gym_executable() -> str:
+    """Locate the ``gym`` CLI on PATH, or fail saying what to do about it.
+
+    Deliberately PATH-only, with no config field pointing at a checkout or a particular venv: these
+    runner configs become serialized job specs, and a local filesystem path cannot cross that
+    boundary. Resolving here rather than at spawn time turns a missing Gym into one legible error
+    instead of an ``ENOENT`` out of ``create_subprocess_exec`` after the run has already started.
+
+    Note that Gym generally cannot live in this SDK's own environment: it imports Ray at module load,
+    and nemo-platform excludes Ray by constraint over an unfixed CVE. Install Gym separately and put
+    its ``bin`` on PATH; in a job image, the image owns PATH and this resolves normally.
+    """
+    resolved = shutil.which(_GYM_CLI)
+    if resolved is None:
+        raise RuntimeError(
+            f"The {_GYM_CLI!r} CLI was not found on PATH. Install NeMo Gym in its own environment "
+            "(it needs Ray, which nemo-platform excludes over an unfixed CVE, so it cannot share this "
+            "one) and put that environment's `bin` directory on PATH. Each resources-server also "
+            "ships its own requirements.txt, installed from that server's directory."
+        )
+    return resolved
+
+
+_PENDING_SERVERS_RE = re.compile(r"(\d+)\s*/\s*(\d+) servers ready\. Waiting for servers to spin up: \[([^\]]*)\]")
+
+
+def _pending_servers(text: str) -> tuple[int, int, tuple[str, ...]] | None:
+    """Recover ``(ready, total, still-pending)`` from the *last* readiness line Gym logged.
+
+    The last line is the one that matters: earlier polls list servers that have since come up.
+    Returns ``None`` when Gym never got as far as printing one (it died during composition, say),
+    which the caller must treat as "no detail available" rather than "nothing pending".
+
+    Never raises — this only ever decorates an error that is already being reported, and a parsing
+    failure here must not replace the real one. Gym's log format is not contractual.
+    """
+    matches = _PENDING_SERVERS_RE.findall(text)
+    if not matches:
+        return None
+    ready, total, names = matches[-1]
+    try:
+        return int(ready), int(total), tuple(re.findall(r"['\"]([^'\"]+)['\"]", names))
+    except ValueError:  # pragma: no cover - the regex only matches digits
+        return None
+
+
+async def _pump_stream(
+    stream: asyncio.StreamReader | None,
+    path: Path,
+    *,
+    label: str,
+    tails: dict[str, deque[str]] | None = None,
+    key: str = "stdout",
+) -> None:
+    """Stream a subprocess pipe to ``path`` while mirroring it to the module logger at ``DEBUG``.
+
+    Reads in chunks rather than by line so a pathologically long line can't overrun asyncio's stream
+    limit, and retains only the last :data:`_LOG_TAIL_LINES` lines in memory (via ``tails[key]``) for
+    inclusion in a failure message — the file on disk is the complete record.
+    """
+    tail: deque[str] = deque(maxlen=_LOG_TAIL_LINES)
+    if tails is not None:
+        tails[key] = tail
+    if stream is None:
+        return
+
+    def emit(raw: bytes) -> None:
+        text = raw.decode("utf-8", errors="replace").rstrip("\r")
+        logger.debug("[%s %s] %s", label, key, text)
+        tail.append(text)
+
+    with path.open("wb") as handle:
+        buffer = b""
+        while True:
+            chunk = await stream.read(65536)
+            if not chunk:
+                break
+            handle.write(chunk)
+            handle.flush()
+            buffer += chunk
+            *complete, buffer = buffer.split(b"\n")
+            for line in complete:
+                emit(line)
+        if buffer:
+            emit(buffer)
+
+
+async def _drain_pumps(pumps: Sequence[asyncio.Task[None]], *, grace_s: float, what: str) -> None:
+    """Await log pumps after teardown, but never block on them indefinitely.
+
+    A pump ends at pipe EOF, which requires *every* inheritor of the write end to close it. Gym's
+    descendants inherit it, and Ray daemonizes ``gcs_server`` into its own session where our
+    process-group signals can't reach it (see :func:`_terminate`) — so a leaked grandchild can hold
+    the pipe open forever. Waiting unconditionally would hang the whole run inside a ``finally``.
+
+    Bounded instead: give the pumps ``grace_s`` to flush, then cancel and move on. Everything read
+    before the stall is already written and flushed to the log file, so cancelling costs at most the
+    tail of a transcript belonging to a process we just killed — and it is reported, not silent.
+    """
+    if not pumps:
+        return
+    _done, pending = await asyncio.wait(pumps, timeout=grace_s)
+    if not pending:
+        return
+    logger.warning(
+        "%d `%s` log pump(s) did not finish within %.1fs; the pipe is still held open (likely a Gym "
+        "grandchild that outlived teardown, e.g. Ray's detached gcs_server). Abandoning them — the log "
+        "file holds everything read up to this point, but its tail may be truncated.",
+        len(pending),
+        what,
+        grace_s,
+    )
+    for task in pending:
+        task.cancel()
+    await asyncio.gather(*pending, return_exceptions=True)
+
+
+def _signal_group(pgid: int, sig: int) -> None:
+    """Send ``sig`` to a process group, tolerating an already-dead group."""
+    try:
+        os.killpg(pgid, sig)
+    except ProcessLookupError:
+        pass
+
+
+async def _terminate(proc: asyncio.subprocess.Process, *, grace_s: float = 30.0) -> None:
+    """Tear down a backgrounded Gym subprocess *and its whole process group*.
+
+    ``gym env start`` fans out into a Ray cluster + uvicorn servers, and Gym stops them from a
+    ``KeyboardInterrupt`` handler (``finally: self.shutdown()``) — so we send **SIGINT** (not SIGTERM,
+    which would bypass that handler and orphan Ray's detached ``gcs_server``). The child is spawned with
+    ``start_new_session=True`` and leads its own group, so SIGINT to the group mimics Ctrl-C exactly;
+    SIGKILL to the group is the escalation if Gym's graceful shutdown overstays ``grace_s``.
+
+    Known limitation: Ray daemonizes ``gcs_server`` into its *own* session, outside this group, so the
+    escalation SIGKILL cannot reach it — clean Ray teardown depends on Gym's SIGINT ``shutdown()``
+    completing within ``grace_s``. POSIX-only (``killpg`` / ``getpgid`` / ``start_new_session`` / SIGINT).
+    """
+    if proc.returncode is not None:
+        return
+    try:
+        pgid = os.getpgid(proc.pid)
+    except ProcessLookupError:
+        return
+    _signal_group(pgid, signal.SIGINT)
+    try:
+        await asyncio.wait_for(proc.wait(), timeout=grace_s)
+    except asyncio.TimeoutError:
+        _signal_group(pgid, signal.SIGKILL)
+        await proc.wait()
diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/records.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/records.py
new file mode 100644
index 0000000000..6fec4fd4a0
--- /dev/null
+++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/records.py
@@ -0,0 +1,52 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Gym's on-disk record format: the artifacts it writes, and the key that joins them.
+
+Shared deliberately between :mod:`dataset` (which *writes* the index onto a materialized row) and
+:mod:`results` (which *reads* it back off a rollout). The attribution contract is exactly this
+pair of key names, so one definition means the writer and the reader cannot disagree.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+
+NG_TASK_INDEX = "_ng_task_index"
+NG_ROLLOUT_INDEX = "_ng_rollout_index"
+#: Fields excluded from a row's content hash (runtime-injected, not task-defining).
+_RUNTIME_KEYS = frozenset({NG_TASK_INDEX, NG_ROLLOUT_INDEX})
+
+
+#: Lines of subprocess output retained in memory for inclusion in a failure message.
+def _read_jsonl(path: str | Path, *, tolerant: bool = False) -> list[dict[str, Any]]:
+    """Read a jsonl file. With ``tolerant=True``, skip (and log) malformed lines instead of raising —
+
+    used for Gym's ``*_failures.jsonl`` sidecar, which is written during abnormal termination and can
+    end in a truncated line; a corrupt failure record must not sink the successfully-collected trials.
+    """
+    rows: list[dict[str, Any]] = []
+    with Path(path).open(encoding="utf-8") as handle:
+        for line_no, line in enumerate(handle, 1):
+            stripped = line.strip()
+            if not stripped:
+                continue
+            try:
+                rows.append(json.loads(stripped))
+            except json.JSONDecodeError:
+                if not tolerant:
+                    raise
+                logger.warning("Skipping malformed JSON at %s:%d", path, line_no)
+    return rows
+
+
+#: `gym env start`'s combined output, under the run's work dir. Named here with the other
+#: artifacts because a *collection* failure often has to point at it: the eval logs show the
+#: symptom, this shows the cause.
+_ENV_LOG_NAME = "gym_env.log"
diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/results.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/results.py
new file mode 100644
index 0000000000..1f14e6f45e
--- /dev/null
+++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/results.py
@@ -0,0 +1,556 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""What Gym produced, read back as trials and scores.
+
+Covers the rollout and failure records, the aggregate metrics Gym computes for itself, and the
+judgement calls in between: which records represent an agent that actually ran, and which reward
+is a measurement rather than an artefact of one that did not.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import math
+from collections.abc import Mapping, Sequence
+from pathlib import Path
+from typing import Any
+
+from nemo_evaluator_sdk.agent_eval.runtimes.gym.config import DEFAULT_REWARD_KEY
+from nemo_evaluator_sdk.agent_eval.runtimes.gym.records import (
+    _ENV_LOG_NAME,
+    NG_ROLLOUT_INDEX,
+    NG_TASK_INDEX,
+    _read_jsonl,
+)
+from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalTask
+from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput
+from nemo_evaluator_sdk.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult
+from nemo_evaluator_sdk.values.evidence import CandidateEvidence, EvidenceDescriptor
+from nemo_evaluator_sdk.values.results import AggregateRangeScore, AggregateScalarScore, AggregateScore
+
+logger = logging.getLogger(__name__)
+
+
+_TOKEN_USAGE_KEYS = ("total_tokens", "input_tokens", "output_tokens", "prompt_tokens", "completion_tokens")
+#: The subset that proves a call happened. A model call always consumes input tokens, so zero on any
+#: of these is evidence the model was never reached; zero output tokens only means an empty answer.
+_INPUT_TOKEN_KEYS = ("total_tokens", "input_tokens", "prompt_tokens")
+
+
+#: Gym's index fields on each rollout record. ``_ng_task_index`` is the only join back to the input
+#: rows that survives a round-trip: Gym mutates ``responses_create_params`` (even the prompt) and
+#: copies only a fixed allowlist of row keys onto the result, so no field we invent comes back. Gym
+#: *honors* a caller-supplied ``_ng_task_index``, which is what makes the join here deterministic.
+class GymRewardMetric:
+    """Score the Gym verifier reward stamped onto trial metadata.
+
+    The Gym analogue of :class:`HarborRewardMetric`: reads the per-trial ``reward``
+    off the candidate metadata (populated by :class:`GymAgentTaskRunner`); a trial
+    with no reward is left **unscored** (``None`` → ``nan``), excluded from the mean
+    and surfaced as ``nan_count`` rather than counted as a spurious ``0.0``. Gym owns
+    the scoring — this metric only surfaces it (Evaluator does not re-derive the reward).
+    """
+
+    def __init__(self, *, output_name: str = "reward", metric_type: str = "gym_reward") -> None:
+        self._output_name = output_name
+        self._metric_type = metric_type
+
+    @property
+    def type(self) -> str:
+        return self._metric_type
+
+    def output_spec(self) -> list[MetricOutputSpec]:
+        return [MetricOutputSpec.continuous_score(self._output_name)]
+
+    async def compute_scores(self, input: MetricInput) -> MetricResult:
+        reward = input.candidate.metadata.get("reward")
+        value = float(reward) if reward is not None else None
+        return MetricResult(outputs=[MetricOutput(name=self._output_name, value=value)])
+
+
+def _agent_never_ran(record: Mapping[str, Any]) -> bool:
+    """True when a result record shows the agent produced nothing *and* never called the model.
+
+    A verifier-scored runner reports whatever score the environment computed, so an agent that
+    silently failed to start is indistinguishable from one that tried and scored 0.0 — the record
+    looks normal, no failure is reported alongside it, and the run succeeds. Observed on
+    ``legal_agent_bench``, whose agent died locating its task tree and returned an empty response:
+    two "completed" trials scoring 0.0, no model call, no failure recorded.
+
+    Deliberately conservative — **both** conditions must hold:
+
+    * no output, and
+    * a stated zero on the input side of token usage.
+
+    An empty answer alone is a legitimate result (a model can return nothing and earn 0.0), so it is
+    not sufficient. Zero *input* tokens is what says the model was never called: a call always sends
+    a prompt, whereas zero output tokens only describes an empty answer. A record carrying no
+    ``response`` key at all is left alone — some runners populate it differently, and this must not
+    reclassify trials it does not understand.
+    """
+    response = record.get("response")
+    if not isinstance(response, Mapping):
+        return False
+    if "output" not in response:
+        # Nothing to judge: some runners shape `response` differently, and absence is not emptiness.
+        return False
+    # Any non-empty value counts as output, whatever its shape. Checking `len(...) > 0` on a list
+    # alone would treat a plain string answer — or a single mapping — as "produced nothing" and fail
+    # a trial that ran perfectly well.
+    if response["output"]:
+        return False
+    usage = response.get("usage")
+    if not isinstance(usage, Mapping):
+        # No usage block to corroborate with; an empty output on its own is not enough to judge.
+        return False
+    # Zero usage has to be *stated*, not inferred from a key being absent. Two vocabularies are in
+    # play — the Responses API's `input_tokens`/`output_tokens` and Chat Completions'
+    # `prompt_tokens`/`completion_tokens` — and Gym's model servers speak both depending on the
+    # adapter. Reading only one set would make a real call reported in the other read as zero.
+    reported = [usage[key] for key in _TOKEN_USAGE_KEYS if isinstance(usage.get(key), (int, float))]
+    if any(value > 0 for value in reported):
+        return False
+    # The discriminating evidence is *input*-side, and only input-side. Calling a model always sends
+    # a prompt, so zero input tokens cannot describe a call that happened; zero **output** tokens
+    # merely describes an empty answer, which is a legitimate result. Requiring an input-side count
+    # rules out a whole class of false positives — a provider reporting only `output_tokens: 0`, or
+    # only `completion_tokens: 0` — rather than excluding those shapes one at a time.
+    input_side = [usage[key] for key in _INPUT_TOKEN_KEYS if isinstance(usage.get(key), (int, float))]
+    return bool(input_side) and all(value == 0 for value in input_side)
+
+
+def _require_full_coverage(tasks: Sequence[AgentEvalTask], *, covered_task_ids: set[str], rollouts_path: Path) -> None:
+    """Fail the run when Gym produced no trial at all for some requested task.
+
+    :class:`AgentEvaluator` already refuses to score a task with no trial
+    (``evaluator._score_trials``), and deliberately so: an incomplete run must not read as a
+    successful one with quieter counts. We keep that contract rather than papering over it with
+    synthesized FAILED trials — an unscored trial is excluded from the mean, so fabricating them
+    would report a plausible-looking average over an unannounced subset of the dataset. A rollout
+    that failed in a *diagnosable* way already becomes a FAILED trial via the failures sidecar;
+    reaching here means the attempt is unaccounted for entirely.
+
+    What this adds over letting the evaluator raise is diagnosis: by the time the run fails the full
+    collection cost is already paid, so the error names the evidence instead of just the task ids.
+    """
+    missing = sorted({task.id for task in tasks} - covered_task_ids)
+    if not missing:
+        return
+    raise RuntimeError(
+        f"Gym produced no rollout for {len(missing)} of {len(tasks)} requested task(s), so the run is "
+        f"incomplete and will not be scored (a partial run must not be reported as a whole one).\n"
+        f"  rollouts:  {rollouts_path}\n"
+        f"  failures:  {_failures_path_for(rollouts_path)}\n"
+        f"  gym logs:  {rollouts_path.parent} (gym_env.log, gym_eval.stdout.log, gym_eval.stderr.log)\n"
+        f"  unrepresented task id(s): {missing}"
+    )
+
+
+def _failures_path_for(rollouts_path: Path) -> Path:
+    """Sidecar Gym writes failed rollouts to (mirrors Gym's own ``_failures_path_for``)."""
+    return rollouts_path.with_name(rollouts_path.stem + "_failures.jsonl")
+
+
+def _aggregate_metrics_path_for(rollouts_path: Path) -> Path:
+    """Sidecar Gym writes run-level aggregate metrics to (``<stem>_aggregate_metrics.json``)."""
+    return rollouts_path.with_name(rollouts_path.stem + "_aggregate_metrics.json")
+
+
+def _read_run_aggregations(rollouts_path: Path) -> dict[str, Any] | None:
+    """Parse Gym's ``rollouts_aggregate_metrics.json``, or ``None`` when absent/unparseable.
+
+    Gym's file is a list with one entry per agent (``agent_ref`` / ``agent_metrics`` / ``key_metrics`` /
+    ``group_level_metrics``), so it is returned keyed by agent name. Carried through as-is — Gym's schema
+    isn't contractual, so the SDK does not type it.
+    """
+    path = _aggregate_metrics_path_for(rollouts_path)
+    if not path.exists():
+        return None
+    try:
+        parsed = json.loads(path.read_text(encoding="utf-8"))
+    except (json.JSONDecodeError, OSError, UnicodeDecodeError):
+        # UnicodeDecodeError is a ValueError, not an OSError: a sidecar truncated mid-codepoint would
+        # otherwise raise straight out of run_tasks and discard a collection that already succeeded.
+        logger.warning("Could not parse Gym aggregate metrics at %s; skipping run aggregations.", path)
+        return None
+    if not isinstance(parsed, list):
+        logger.warning("Unexpected Gym aggregate-metrics shape at %s (%s); skipping.", path, type(parsed).__name__)
+        return None
+    # Gym's schema is not contractual, and this runs after a collection that already succeeded — an
+    # entry in an unexpected shape must not raise out of run_tasks and discard every trial with it.
+    aggregations: dict[str, Any] = {}
+    for entry in parsed:
+        agent_ref = entry.get("agent_ref") if isinstance(entry, Mapping) else None
+        name = agent_ref.get("name") if isinstance(agent_ref, Mapping) else None
+        if not isinstance(name, str):
+            logger.warning(
+                "Skipping a Gym aggregate-metrics entry in %s with no usable agent_ref.name (got %r).", path, agent_ref
+            )
+            continue
+        aggregations[name] = {key: value for key, value in entry.items() if key != "agent_ref"}
+    return aggregations or None
+
+
+#: Gym flattens a distribution into ``<stat>/<metric>`` keys. Its RewardProfiler emits this exact set
+#: together for every numeric column (``describe_dataframe``), minus ``histogram``, which
+#: ``prepare_for_serialization`` strips before the file is written. All five must be present before we
+#: treat a group of keys as one distribution: a resources-server is free to define a metric literally
+#: named ``mean`` (36 of Gym's ~97 servers override ``compute_metrics``), and re-assembling on a partial
+#: match would rename someone's standalone metric into a statistic of a distribution that never existed.
+_GYM_STAT_FAMILY = ("mean", "max", "min", "median", "std")
+
+#: Metric Gym reports that the SDK already computes natively from the same rollouts (``gym_reward.reward``).
+_GYM_REDUNDANT_METRICS = frozenset({"reward"})
+
+
+def _aggregate_scores_from_gym(aggregations: Mapping[str, Any] | None) -> list[AggregateScore]:
+    """Map Gym's run-level ``agent_metrics`` onto typed aggregate scores named ``runner.gym.<metric>``.
+
+    Reads ``agent_metrics``, not ``key_metrics``: ``key_metrics`` is a *subset* of it chosen by the
+    resources-server, and the default selection (``get_key_metrics``) keeps only the ``mean/*`` entries —
+    so the max/min/median/std that make a distribution never appear there, and every metric would arrive
+    as a lone ``mean/<name>`` scalar.
+
+    Keys forming a full stat family are re-assembled into one :class:`AggregateRangeScore`; every other
+    numeric key becomes an :class:`AggregateScalarScore`. Non-numeric values are skipped — they are
+    labels or notes, not measurements.
+
+    Names are namespaced by runner (not by agent): each run is instrumented with a single agent, so the
+    agent adds no disambiguation, and a reader needs to know which *backend* produced a number.
+    """
+    if not aggregations:
+        return []
+    multi_agent = len(aggregations) > 1
+    scores: list[AggregateScore] = []
+    for agent_name, payload in sorted(aggregations.items()):
+        agent_metrics = payload.get("agent_metrics") if isinstance(payload, Mapping) else None
+        if not isinstance(agent_metrics, Mapping):
+            continue
+        # One agent per run is the norm, so `runner.gym.<metric>` reads cleanly; qualify by agent only
+        # when a run really did produce several, where the unqualified names would collide.
+        prefix = f"runner.gym.{agent_name}." if multi_agent else "runner.gym."
+        scores.extend(_scores_from_agent_metrics(agent_metrics, prefix=prefix))
+    return scores
+
+
+def _scores_from_agent_metrics(agent_metrics: Mapping[str, Any], *, prefix: str) -> list[AggregateScore]:
+    families: dict[str, dict[str, float]] = {}
+    scalars: dict[str, float] = {}
+    for key, value in agent_metrics.items():
+        number = _as_float(value)
+        if number is None:
+            continue
+        stat, _, metric = key.partition("/")
+        if metric and stat in _GYM_STAT_FAMILY:
+            families.setdefault(metric, {})[stat] = number
+        else:
+            scalars[key] = number
+
+    scores: list[AggregateScore] = []
+    for metric, stats in sorted(families.items()):
+        # Redundancy is a property of the metric, not of how complete its family is. Checking after the
+        # fallback below would let a partial `reward` family survive as `mean/reward` scalars, since the
+        # scalar filter matches the bare name -- reintroducing the duplicate of `gym_reward.reward` that
+        # skipping reward exists to prevent.
+        if metric in _GYM_REDUNDANT_METRICS:
+            continue
+        if set(stats) < set(_GYM_STAT_FAMILY):
+            # Not a distribution we can vouch for; keep each key as the standalone number it may well be.
+            scalars.update({f"{stat}/{metric}": value for stat, value in stats.items()})
+            continue
+        scores.append(
+            AggregateRangeScore(
+                name=f"{prefix}{metric}",
+                # Gym reports the statistics but not the sample size behind them, and inventing one
+                # would misreport coverage. `None` says "unknown"; 0 would assert nothing was evaluated.
+                count=None,
+                nan_count=0,
+                mean=stats["mean"],
+                min=stats["min"],
+                max=stats["max"],
+                median=stats["median"],
+                # Gym computes this with pandas (ddof=1), so it is the sample standard deviation.
+                sample_std_dev=stats["std"],
+                sample_variance=stats["std"] ** 2,
+            )
+        )
+    scores.extend(
+        AggregateScalarScore(name=f"{prefix}{key}", count=None, nan_count=0, value=value)
+        for key, value in sorted(scalars.items())
+        if key not in _GYM_REDUNDANT_METRICS
+    )
+    return scores
+
+
+def _as_float(value: Any) -> float | None:
+    """``float`` for a numeric Gym metric value; None for anything else (bools included: not measurements)."""
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        return None
+    return float(value) if math.isfinite(value) else None
+
+
+def _ensure_fresh_output(rollouts_path: Path) -> None:
+    """Enforce one Gym run per output dir (the AgentEvaluator convention).
+
+    Gym appends to the failures sidecar (``open("ab")``) and doesn't clear it between runs, so reusing
+    a populated directory would silently mix this run's failures with a prior run's. Rather than clear
+    (which would clobber an earlier run's results — infra failures are useful signal), refuse to run
+    into a directory that already holds Gym rollout output.
+    """
+    preexisting = [path for path in (rollouts_path, _failures_path_for(rollouts_path)) if path.exists()]
+    if preexisting:
+        names = ", ".join(path.name for path in preexisting)
+        raise FileExistsError(
+            f"{rollouts_path.parent} already holds Gym rollout output ({names}); give each run a fresh "
+            "output_dir (the AgentEvaluator convention). Gym appends to the failures sidecar, so reusing a "
+            "directory would mix runs and could obscure a prior run's results."
+        )
+
+
+def _coerce_reward(value: Any) -> float | None:
+    """Best-effort ``float`` for a reward; returns None (never raises) on a missing/malformed value."""
+    if value is None:
+        return None
+    try:
+        return float(value)
+    except (TypeError, ValueError):
+        logger.warning("Gym rollout carried a non-numeric reward %r; recording the trial as unscored.", value)
+        return None
+
+
+def _resolve_task_id(
+    record: Mapping[str, Any], index_to_task_id: Mapping[int, str], *, strict: bool = True
+) -> str | None:
+    """Map a rollout record to a task id via the ``_ng_task_index`` we stamped in the input dataset.
+
+    Returns None when the record carries no usable index (kill-shaped failures can lack one).
+
+    An index we never stamped is handled by ``strict``. In the **successes** file (``strict=True``)
+    it is fatal: every row Gym read came from :func:`_materialize_dataset`, so an unknown index means
+    Gym reindexed the dataset rather than honoring our stamp, and every reward attribution is
+    therefore suspect. In the **failures sidecar** (``strict=False``) it is merely unattributable and
+    is counted instead — that file is written during abnormal termination and read tolerantly by
+    design, so one odd record must not discard the successes already parsed. That relaxation cannot
+    inflate a result: failure records carry no reward, and a task left with no trial still trips
+    :func:`_require_full_coverage`.
+    """
+    index = record.get(NG_TASK_INDEX)
+    if not isinstance(index, int) or isinstance(index, bool):
+        return None
+    task_id = index_to_task_id.get(index)
+    if task_id is None:
+        if not strict:
+            logger.warning(
+                "Gym failure record carried %s=%s, which was not stamped on the materialized dataset; "
+                "counting it as unattributable rather than failing the run.",
+                NG_TASK_INDEX,
+                index,
+            )
+            return None
+        raise ValueError(
+            f"Gym emitted {NG_TASK_INDEX}={index}, which is not one of the {len(index_to_task_id)} index value(s) "
+            f"stamped on the materialized dataset. Gym appears to have reassigned {NG_TASK_INDEX} instead "
+            "of honoring the supplied one, so rollout-to-task attribution is unsafe."
+        )
+    return task_id
+
+
+def _rollout_trial_id(task_id: str, raw_rollout_index: Any, synth_seq: dict[str, int], *, missing_label: str) -> str:
+    """Per-attempt trial id: use the record's ``_ng_rollout_index`` when it's a real int, else a
+    per-task-unique ``{missing_label}{n}`` suffix so records lacking an index don't collide on one id."""
+    if isinstance(raw_rollout_index, int) and not isinstance(raw_rollout_index, bool):
+        return f"{task_id}:{raw_rollout_index}"
+    seq = synth_seq[task_id] = synth_seq.get(task_id, 0) + 1
+    return f"{task_id}:{missing_label}{seq}"
+
+
+def _trials_from_rollouts(
+    rollouts_path: Path,
+    tasks: Sequence[AgentEvalTask],
+    index_to_task_id: Mapping[int, str],
+    *,
+    reward_key: str = DEFAULT_REWARD_KEY,
+) -> list[AgentEvalTrial]:
+    """Fan Gym's rollout records out into one :class:`AgentEvalTrial` per attempt.
+
+    Reads *both* Gym output files: successes from ``rollouts.jsonl`` (COMPLETED, carrying the reward)
+    and failures from the ``*_failures.jsonl`` sidecar (FAILED — so a failed attempt is counted and
+    diagnosed rather than silently dropped). Every record is attributed to its task by
+    ``_ng_task_index`` → ``index_to_task_id``, the map we stamped in :func:`_materialize_dataset`.
+    """
+    if not rollouts_path.exists():
+        # `_ensure_fresh_output` removed any stale file before the run, and `_collect_rollouts`
+        # already raised on a non-zero exit — so reaching here means Gym reported success and wrote
+        # nothing. Opening the path regardless would surface that as a bare FileNotFoundError
+        # naming a file the reader has no reason to know about, which is the same illegibility the
+        # rest of this change set exists to remove.
+        raise RuntimeError(
+            f"`gym eval run` reported success but wrote no results to {rollouts_path}, so there is "
+            "nothing to score. This usually means collection stopped before any attempt completed.\n"
+            f"See {rollouts_path.parent / 'gym_eval.stdout.log'} for what collection did, and "
+            f"{rollouts_path.parent / _ENV_LOG_NAME} for a traceback from the environment's own "
+            "servers."
+        )
+    known_task_ids = {task.id for task in tasks}
+    # Defence in depth: the index map and the task list must describe the same run. Gym is an external
+    # project on its own release cadence, and people make agent-optimization decisions from these
+    # numbers — so any disagreement about *which task a result belongs to* is a hard error, never a
+    # best-effort join. Cheap to check once; impossible to notice downstream if it silently drifts.
+    mapped_task_ids = set(index_to_task_id.values())
+    if mapped_task_ids != known_task_ids:
+        raise ValueError(
+            f"the {NG_TASK_INDEX} map does not match the task list it will be joined against "
+            f"({len(mapped_task_ids - known_task_ids)} mapped task(s) absent from the list, "
+            f"{len(known_task_ids - mapped_task_ids)} listed task(s) absent from the map); "
+            "attributing rollouts across a mismatch would report results against the wrong tasks"
+        )
+    trials: list[AgentEvalTrial] = []
+    # Shared per-task counter for records lacking a usable _ng_rollout_index (success + failure); a
+    # missing/null index must not collapse multiple attempts for one task onto the same trial id.
+    synth_seq: dict[str, int] = {}
+
+    success_evidence = CandidateEvidence(
+        descriptors={"rollouts": EvidenceDescriptor(kind="filesystem", format="file", ref=str(rollouts_path))}
+    )
+    unattributed_successes = 0
+    empty_output_count = 0
+    #: Tasks with at least one trial where the agent never ran. Tracked separately from trial
+    #: status, since the failures sidecar also produces FAILED trials for unrelated reasons.
+    empty_output_task_ids: set[str] = set()
+    for record in _read_jsonl(rollouts_path):
+        task_id = _resolve_task_id(record, index_to_task_id)
+        if task_id is None:
+            # No usable index (an unknown one raises). Skipping is right, but silence is not: with
+            # num_repeats > 1 the task keeps its other trials, so _require_full_coverage won't fire and
+            # the task would simply be scored on fewer attempts than were actually run.
+            unattributed_successes += 1
+            continue
+        raw_rollout_index = record.get(NG_ROLLOUT_INDEX)
+        reward = _coerce_reward(record.get(reward_key))
+        # An agent that never ran did not earn its reward — reporting 0.0 as a completed trial would
+        # let a broken environment read as a poorly-performing one.
+        never_ran = _agent_never_ran(record)
+        if never_ran:
+            empty_output_count += 1
+            empty_output_task_ids.add(task_id)
+            status = AgentEvalTrialStatus.FAILED
+        elif reward is not None:
+            status = AgentEvalTrialStatus.COMPLETED
+        else:
+            status = AgentEvalTrialStatus.PARTIAL
+        metadata: dict[str, Any] = {
+            "reward": None if never_ran else reward,
+            NG_TASK_INDEX: record.get(NG_TASK_INDEX),
+            NG_ROLLOUT_INDEX: raw_rollout_index,
+        }
+        if never_ran:
+            # Kept alongside the reward Gym reported, so the discarded value is still auditable.
+            metadata["gym_failure"] = (
+                "the agent produced no output and consumed no tokens, so the model was never called; "
+                f"the reported score of {reward!r} is not a measurement of this agent"
+            )
+        trials.append(
+            AgentEvalTrial(
+                id=_rollout_trial_id(task_id, raw_rollout_index, synth_seq, missing_label="noidx"),
+                task_id=task_id,
+                status=status,
+                output=AgentOutput(response=record.get("response"), metadata={"agent_ref": record.get("agent_ref")}),
+                evidence=success_evidence,
+                metadata=metadata,
+            )
+        )
+
+    failures_path = _failures_path_for(rollouts_path)
+    unattributed_failures = 0
+    if failures_path.exists():
+        failure_evidence = CandidateEvidence(
+            descriptors={
+                "rollout_failures": EvidenceDescriptor(kind="filesystem", format="file", ref=str(failures_path))
+            }
+        )
+        # tolerant: a truncated line in the abnormal-termination sidecar must not sink the good trials.
+        for record in _read_jsonl(failures_path, tolerant=True):
+            # strict=False to match this loop's tolerant read: an odd record is counted, not fatal.
+            task_id = _resolve_task_id(record, index_to_task_id, strict=False)
+            if task_id is None:
+                # kill-shaped failures (SIGTERM/OOM/actor-death) can lack a usable index — count, don't drop.
+                unattributed_failures += 1
+                continue
+            raw_rollout_index = record.get(NG_ROLLOUT_INDEX)
+            trials.append(
+                AgentEvalTrial(
+                    id=_rollout_trial_id(task_id, raw_rollout_index, synth_seq, missing_label="fail"),
+                    task_id=task_id,
+                    status=AgentEvalTrialStatus.FAILED,
+                    output=AgentOutput(
+                        response=record.get("response"), metadata={"agent_ref": record.get("agent_ref")}
+                    ),
+                    evidence=failure_evidence,
+                    metadata={
+                        "reward": None,
+                        # best-effort: Gym's failure-record schema isn't contractual, so probe common keys.
+                        "gym_failure": record.get("error") or record.get("exception") or record.get("failure_reason"),
+                        NG_TASK_INDEX: record.get(NG_TASK_INDEX),
+                        NG_ROLLOUT_INDEX: raw_rollout_index,
+                    },
+                )
+            )
+    if empty_output_count:
+        # Every trial empty is a configuration failure, not a result: nothing was measured, so there
+        # is no score to report and letting the run "succeed" would publish zeros as findings. A
+        # partial count is left as failed trials — a flaky environment is still worth the attempts
+        # that did run.
+        #
+        # Measured against *every* trial, including those the failures sidecar contributed. A run
+        # that also recorded real failures is not the uniform "nothing ran" picture this raise is
+        # for: those trials carry their own diagnosis (a verifier timeout, say, from an attempt that
+        # did reach the model), and raising would discard it for a less specific message.
+        #
+        # The unattributed counts need the same treatment and are easy to miss, because those
+        # records never became trials and so are invisible to `len(trials)`. Each is still a result
+        # the environment produced — an unattributed *success* may have run perfectly well, and an
+        # unattributed *failure* is a diagnosis of its own (an OOM-killed agent, say). Claiming
+        # nothing was measured while either exists would be false.
+        if empty_output_count == len(trials) and not unattributed_successes and not unattributed_failures:
+            affected_tasks = {trial.task_id for trial in trials}
+            raise RuntimeError(
+                f"No agent ran: all {empty_output_count} trial(s) across all {len(affected_tasks)} "
+                "task(s) produced no output and consumed no tokens, so the model was never called "
+                "and nothing was measured. The scores reported for them describe an agent that did "
+                "not run.\n"
+                "This is normally the environment failing to start its agent rather than a bad "
+                f"model: check the per-server startup output in {rollouts_path.parent / _ENV_LOG_NAME} "
+                "for a traceback from the environment's own agent server.\n"
+                f"Results as collected: {rollouts_path}"
+            )
+        logger.warning(
+            "%d of %d trial(s), across %d of %d task(s), produced no agent output and consumed no "
+            "tokens; those are marked FAILED rather than scored, since the model was never called "
+            "for them.",
+            empty_output_count,
+            len(trials),
+            len(empty_output_task_ids),
+            len({trial.task_id for trial in trials}),
+        )
+    if unattributed_successes:
+        logger.warning(
+            "%d Gym rollout record(s) in %s carried no usable %s and could not be attributed to a task; "
+            "affected tasks are scored on fewer attempts than were collected.",
+            unattributed_successes,
+            rollouts_path,
+            NG_TASK_INDEX,
+        )
+    if unattributed_failures:
+        logger.warning(
+            "%d Gym failure record(s) lacked a usable %s and could not be attributed to a task.",
+            unattributed_failures,
+            NG_TASK_INDEX,
+        )
+
+    # Reported here, enforced by _require_full_coverage in run_tasks (which knows the run's log paths).
+    missing = known_task_ids - {trial.task_id for trial in trials}
+    if missing:
+        logger.warning("No Gym rollout produced a trial for %d requested task(s): %s", len(missing), sorted(missing))
+    return trials
diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/runtime.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/runtime.py
new file mode 100644
index 0000000000..69a09339b8
--- /dev/null
+++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym/runtime.py
@@ -0,0 +1,383 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""The runner itself: the two-step Gym invocation, start to scored trials.
+
+Orchestration only. The stages it drives live in sibling modules — :mod:`config` for what Gym is
+told, :mod:`dataset` for what it is given, :mod:`process` for how it is run, and :mod:`results`
+for what comes back.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+import re
+import tempfile
+from collections import deque
+from collections.abc import Mapping, Sequence
+from pathlib import Path
+from typing import Any
+
+from nemo_evaluator_sdk.agent_eval.runtimes.gym.config import (
+    _HYDRA_SUBDIR,
+    GymRuntimeConfig,
+    _hydra_scalar,
+    _redact_env_overrides,
+    _selection_args,
+)
+from nemo_evaluator_sdk.agent_eval.runtimes.gym.dataset import _materialize_dataset, _source_datasets
+from nemo_evaluator_sdk.agent_eval.runtimes.gym.process import (
+    _LOG_TAIL_LINES,
+    _VALIDATE_TIMEOUT_S,
+    _drain_pumps,
+    _gym_executable,
+    _pending_servers,
+    _pump_stream,
+    _terminate,
+)
+from nemo_evaluator_sdk.agent_eval.runtimes.gym.records import _ENV_LOG_NAME
+from nemo_evaluator_sdk.agent_eval.runtimes.gym.results import (
+    _aggregate_scores_from_gym,
+    _ensure_fresh_output,
+    _read_run_aggregations,
+    _require_full_coverage,
+    _trials_from_rollouts,
+)
+from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask
+from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial, RunnerInfo
+from nemo_evaluator_sdk.values.results import AggregateScore
+
+logger = logging.getLogger(__name__)
+
+
+class GymAgentTaskRunner:
+    """An :class:`AgentTaskRunner` that runs an existing Gym env, then adapts its rollouts.
+
+    :meth:`run_tasks` materializes the requested tasks into a normalized dataset with
+    ``_ng_task_index`` stamped per row, runs the two-step Gym flow (``env start`` +
+    ``eval run --no-serve --input``) against it via subprocess, and fans the rollout
+    records out into one trial per ``_ng_rollout_index``. Because we assign the index,
+    attribution is a dict lookup rather than a positional guess, and Gym only rolls out
+    the tasks we asked for. The durable task **identity** stays our content hash.
+    """
+
+    def __init__(self, *, config: GymRuntimeConfig) -> None:
+        self._config = config
+        self._run_aggregations: dict[str, Any] | None = None
+
+    def run_aggregate_scores(self) -> Sequence[AggregateScore]:
+        """Gym's ``agent_metrics`` mapped onto typed aggregate scores, namespaced ``runner.gym.<metric>``.
+
+        Satisfies :class:`RunAggregationsProvider`. ``reward`` is skipped: the SDK already scores it
+        natively as ``gym_reward.reward``, and two differently-derived numbers under one name invites
+        exactly the confusion the namespace is there to prevent.
+        """
+        return _aggregate_scores_from_gym(self._run_aggregations)
+
+    def runner_info(self) -> RunnerInfo:
+        """Identify this runner and the Gym settings that shape its results.
+
+        Credentials normally live in the Gym checkout's gitignored ``env.yaml`` and never reach this
+        object — but ``env_overrides`` is a free-form escape hatch, so its values are redacted by key
+        (see :func:`_redact_env_overrides`) rather than trusted.
+        """
+        cfg = self._config
+        return RunnerInfo(
+            name="gym",
+            kind="runner",
+            config={
+                "resources_server": cfg.resources_server,
+                "agent": cfg.agent,
+                "agent_config": cfg.agent_config,
+                "model_type": cfg.model_type,
+                "num_repeats": cfg.num_repeats,
+                "concurrency": cfg.concurrency,
+                "bind_resources_server": cfg.bind_resources_server,
+                "env_overrides": _redact_env_overrides(cfg.env_overrides),
+                "reward_key": cfg.reward_key,
+            },
+        )
+
+    async def run_tasks(
+        self,
+        tasks: Sequence[AgentEvalTask],
+        config: AgentEvalRunConfig | None = None,
+    ) -> list[AgentEvalTrial]:
+        cfg = self._config
+        self._run_aggregations = None  # reset per run so a reused runner never leaks a prior run's numbers
+        # Provenance for the log line only — the file Gym actually reads is the normalized one we
+        # materialize below from the tasks themselves.
+        source_dataset = _source_datasets(tasks)
+
+        # config.parallelism still governs how the evaluator *scores* the trials we return (its scoring
+        # semaphore). It is deliberately not mapped onto Gym's rollout `--concurrency`: those are different
+        # phases — parallelism bounds concurrent scoring (SDK-side, cheap), while Gym's `--concurrency`
+        # bounds concurrent rollouts against the model endpoint during collection (tuned to that endpoint's
+        # limits via GymRuntimeConfig.concurrency).
+        if config is not None and config.work_dir is not None:
+            work_dir = Path(config.work_dir) / "gym_run"
+        else:
+            work_dir = Path(tempfile.mkdtemp(prefix="gym_run_"))
+        work_dir.mkdir(parents=True, exist_ok=True)
+
+        rollouts_path = work_dir / "rollouts.jsonl"
+        _ensure_fresh_output(rollouts_path)
+
+        # Hand Gym a dataset we control: one row per requested task, each carrying the _ng_task_index
+        # we assign. Gym honors a pre-stamped index, so the rollout->task join becomes a total map
+        # instead of a positional guess about Gym's raw-line dedup (see the module docstring).
+        input_path = work_dir / "gym_input.jsonl"
+        index_to_task_id = _materialize_dataset(tasks, input_path)
+        logger.info(
+            "Materialized %d task(s) from %s into %s for Gym collection.",
+            len(index_to_task_id),
+            source_dataset,
+            input_path,
+        )
+
+        await self._run_two_step(input_path, rollouts_path, work_dir)
+        self._run_aggregations = _read_run_aggregations(rollouts_path)
+        trials = _trials_from_rollouts(rollouts_path, tasks, index_to_task_id, reward_key=cfg.reward_key)
+        _require_full_coverage(tasks, covered_task_ids={trial.task_id for trial in trials}, rollouts_path=rollouts_path)
+        return trials
+
+    async def _validate_config(
+        self, gym: str, selection: Sequence[str], subprocess_env: Mapping[str, str], work_dir: Path
+    ) -> None:
+        """Pre-flight the composed Gym config with ``gym env validate`` before starting anything.
+
+        Gym does not publish what configuration an environment requires — the typed
+        ``*ResourcesServerConfig`` classes cover behavioural knobs, while the model wiring lives in
+        each environment's YAML under names that vary per environment. ``gym env validate`` is the
+        only way to find out short of running: it merges configs, flags, and overrides, then reports
+        unresolved ``???`` values, bad paths, and cross-references to servers that are not defined —
+        without Ray and without starting a server.
+
+        Running it every time is worth the second it costs. A config mistake otherwise surfaces after
+        ``gym env start`` has brought up a Ray cluster and several uvicorn servers, as a readiness
+        timeout up to ``startup_timeout_s`` (240s by default) whose message says nothing about the
+        actual problem.
+        """
+        proc = await asyncio.create_subprocess_exec(
+            gym,
+            "env",
+            "validate",
+            *selection,
+            env=dict(subprocess_env),
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.STDOUT,
+            # Its own process group, like the other two Gym invocations. `_terminate` signals the
+            # group via `killpg`, so without this a validate timeout would signal *our* group — the
+            # SDK process included.
+            start_new_session=True,
+        )
+        try:
+            stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=_VALIDATE_TIMEOUT_S)
+        except TimeoutError:
+            await _terminate(proc, grace_s=self._config.shutdown_grace_s)
+            raise RuntimeError(
+                f"`gym env validate` did not finish within {_VALIDATE_TIMEOUT_S}s for resources-server "
+                f"{self._config.resources_server!r}"
+            ) from None
+
+        report = stdout.decode("utf-8", errors="replace").strip()
+        (work_dir / "gym_validate.log").write_text(report, encoding="utf-8")
+        if proc.returncode != 0:
+            raise RuntimeError(
+                f"Gym rejected the composed config for resources-server {self._config.resources_server!r} "
+                f"before any server started. `gym env validate` said:\n\n{report}"
+            )
+        logger.debug("gym env validate: %s", report)
+
+    async def _run_two_step(self, input_path: Path, output_path: Path, work_dir: Path) -> None:
+        """Start the Gym servers, collect against them with ``--no-serve``, then tear them down."""
+        cfg = self._config
+        gym = _gym_executable()
+        env_log = work_dir / _ENV_LOG_NAME
+
+        # Gym launches each server from its own subdir with its own .venv. Ray (>=2.56) otherwise
+        # detects a `uv run` ancestor and tries to replicate that uv project onto its workers,
+        # asserting the project pyproject.toml lives in the driver's cwd — which aborts startup. That
+        # hook is wrong for Gym (servers manage their own deps), so disable it for the subprocesses.
+        subprocess_env = {**os.environ, "RAY_ENABLE_UV_RUN_RUNTIME_ENV": "0"}
+
+        selection = _selection_args(cfg, work_dir)
+
+        await self._validate_config(gym, selection, subprocess_env, work_dir)
+
+        env_cmd = [gym, "env", "start", *selection]
+        # start_new_session=True puts `gym env start` in its own process group so teardown can signal
+        # the *whole* Ray-cluster + uvicorn tree, not just the direct child (else they orphan).
+        # stderr is merged into stdout here (unlike `eval run`, which splits them) because readiness
+        # detection scans a single chronological transcript and Gym's readiness line is not guaranteed
+        # to land on a particular stream.
+        env_proc = await asyncio.create_subprocess_exec(
+            *env_cmd,
+            env=subprocess_env,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.STDOUT,
+            start_new_session=True,
+        )
+        env_pump = asyncio.create_task(_pump_stream(env_proc.stdout, env_log, label="gym env start"))
+        try:
+            await self._wait_for_servers(env_log, env_proc)
+            await self._collect_rollouts(gym, input_path, output_path, work_dir, subprocess_env)
+        finally:
+            await _terminate(env_proc, grace_s=cfg.shutdown_grace_s)
+            # Bounded: a leaked grandchild holding the inherited pipe must not wedge teardown.
+            await _drain_pumps([env_pump], grace_s=cfg.shutdown_grace_s, what="gym env start")
+
+    async def _collect_rollouts(
+        self, gym: str, input_path: Path, output_path: Path, work_dir: Path, subprocess_env: dict[str, str]
+    ) -> None:
+        """Run ``gym eval run --no-serve`` against the live servers, bounded and self-cleaning.
+
+        Spawned in its own process group and always torn down (even on timeout/cancellation), so a hung
+        collection can't wedge the run or leak Ray workers.
+
+        Output is *streamed* to ``gym_eval.stdout.log`` / ``gym_eval.stderr.log`` in ``work_dir`` rather
+        than buffered in memory (a long collection's transcript can be large), and mirrored to this
+        module's logger at ``DEBUG`` so callers can surface it in their own terminal through ordinary
+        ``logging`` configuration. Failures point at both files and inline the last few lines.
+        """
+        cfg = self._config
+        eval_cmd = [
+            gym,
+            "eval",
+            "run",
+            "--no-serve",
+            "--agent",
+            cfg.agent,
+            "--input",
+            str(input_path),
+            "--output",
+            str(output_path),
+            "--num-repeats",
+            str(cfg.num_repeats),
+            "--concurrency",
+            str(cfg.concurrency),
+            # `gym eval run` is a Hydra app too, and unlike validate/start it does not go through
+            # _selection_args — so without this it writes `outputs/<date>/<time>/` into the caller's
+            # cwd on every single collection. Quoted for the same reason as there: a work dir
+            # containing `,` or `[` is otherwise misread as sweep syntax.
+            f"hydra.run.dir={_hydra_scalar(str(work_dir / _HYDRA_SUBDIR))}",
+        ]
+        stdout_log = work_dir / "gym_eval.stdout.log"
+        stderr_log = work_dir / "gym_eval.stderr.log"
+        eval_proc = await asyncio.create_subprocess_exec(
+            *eval_cmd,
+            env=subprocess_env,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+            start_new_session=True,
+        )
+        tails: dict[str, deque[str]] = {}
+        pumps = [
+            asyncio.create_task(
+                _pump_stream(eval_proc.stdout, stdout_log, label="gym eval", tails=tails, key="stdout")
+            ),
+            asyncio.create_task(
+                _pump_stream(eval_proc.stderr, stderr_log, label="gym eval", tails=tails, key="stderr")
+            ),
+        ]
+        try:
+            await asyncio.wait_for(eval_proc.wait(), timeout=cfg.collection_timeout_s)
+        except asyncio.TimeoutError as exc:
+            raise TimeoutError(
+                f"`gym eval run` exceeded collection_timeout_s={cfg.collection_timeout_s}s; collection aborted. "
+                f"See {stdout_log} and {stderr_log}."
+            ) from exc
+        finally:
+            # Terminate first so the pipes close, then drain (bounded) to flush whatever was buffered.
+            await _terminate(eval_proc, grace_s=cfg.shutdown_grace_s)
+            await _drain_pumps(pumps, grace_s=cfg.shutdown_grace_s, what="gym eval run")
+        if eval_proc.returncode != 0:
+            tail = "\n".join(tails.get("stderr") or tails.get("stdout") or [])
+            # A collection failure is frequently *server*-side: the resources-server raises, the
+            # agent surfaces it as a bare HTTP 500, and the traceback that explains it is only in
+            # gym_env.log. Naming the two eval logs alone sends the reader to the one place the
+            # cause is not. (Observed on wmt_translation: a 500 here, `PermissionError: /opt/Gym`
+            # there.)
+            env_log = work_dir / _ENV_LOG_NAME
+            raise RuntimeError(
+                f"`gym eval run` failed (rc={eval_proc.returncode}). Collection output: {stdout_log} "
+                f"and {stderr_log}\n"
+                f"If the tail below is an HTTP error from a Gym server, the cause is server-side: "
+                f"see {env_log}, whose lines are prefixed with the server that emitted them.\n"
+                f"--- last {_LOG_TAIL_LINES} line(s) ---\n{tail}"
+            )
+
+    async def _wait_for_servers(self, env_log: Path, env_proc: asyncio.subprocess.Process) -> None:
+        """Wait until ``gym env start`` reports every composed server ready.
+
+        Keys off Gym's own readiness line (``All N / N servers ready!``), which it prints only after
+        HTTP-health-checking each server it composed. So the caller never has to know or guess the
+        server count — Gym derives it from the config and health-checks for us.
+        """
+        cfg = self._config
+        loop = asyncio.get_running_loop()
+        ready = re.compile(r"All \d+ / \d+ servers ready")
+        deadline = loop.time() + cfg.startup_timeout_s
+        text = ""
+        while loop.time() < deadline:
+            if env_proc.returncode is not None:
+                raise RuntimeError(f"`gym env start` exited early (rc={env_proc.returncode}); see {env_log}")
+            text = env_log.read_text(encoding="utf-8", errors="replace") if env_log.exists() else ""
+            if "required uv version" in text.lower():
+                raise RuntimeError(f"`gym env start` hit the uv required-version gate; see {env_log}")
+            if ready.search(text):
+                return
+            await asyncio.sleep(2)
+        raise TimeoutError(self._startup_timeout_message(text, env_log))
+
+    def _startup_timeout_message(self, env_log_text: str, env_log: Path) -> str:
+        """Explain a startup timeout in terms of *which* server did not come up.
+
+        The bare "servers not ready" message sends you to a log that is mostly noise from the servers
+        that started fine. Gym polls and prints the outstanding set every time, so naming it costs
+        nothing and is usually the whole diagnosis: one server out of four means that environment's
+        own startup, not the platform.
+
+        The timeout is also a plausible *cause* rather than a symptom — a Docker-backed environment
+        builds its image on first run, which routinely exceeds the 240s default — so the message says
+        how to extend it instead of leaving the reader to find the knob.
+        """
+        cfg = self._config
+        pending = _pending_servers(env_log_text)
+        if pending is None:
+            detail = (
+                "Gym never reported server readiness, so it likely failed before starting them "
+                "(config composition or dependency installation)."
+            )
+        elif not pending[2]:
+            # Gym printed a readiness line but named no outstanding server. Saying "waiting on:
+            # unknown" would contradict the counts in the same sentence, so report only what is known.
+            ready_count, total, _ = pending
+            detail = (
+                f"{ready_count} of {total} server(s) started; Gym did not name which remained "
+                f"outstanding. The per-server output in {env_log.name} is prefixed with the server "
+                "that emitted each line."
+            )
+        else:
+            ready_count, total, names = pending
+            detail = (
+                f"{ready_count} of {total} server(s) started; still waiting on: {', '.join(names)}. "
+                f"Search {env_log.name} for '({names[0]})' — each line is prefixed with the server "
+                "that emitted it."
+            )
+        return (
+            f"Gym servers for resources-server {cfg.resources_server!r} were not ready within "
+            f"startup_timeout_s={cfg.startup_timeout_s}s. {detail}\n"
+            f"Full startup output: {env_log}\n"
+            "If the environment installs heavy dependencies or builds a container image on first run, "
+            "this timeout is expected on a cold cache — raise GymRuntimeConfig.startup_timeout_s and "
+            "re-run; the second run reuses what the first installed."
+        )
+
+
+#: Gym's per-poll readiness line, e.g.
+#: ``3 / 4 servers ready. Waiting for servers to spin up: ['legal_agent_bench']``.
+#: Gym knows exactly which servers are outstanding; without this we discard that and report only
+#: that *something* timed out.
diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym_runtime.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym_runtime.py
deleted file mode 100644
index 224bef2f4e..0000000000
--- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym_runtime.py
+++ /dev/null
@@ -1,1589 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""NeMo Gym-backed :class:`AgentTaskRunner` for the agent-eval pipeline.
-
-Runs an *existing* NeMo Gym environment through its ``gym`` CLI and adapts the
-rollout bundle into SDK :class:`AgentEvalTrial` objects, so an
-:class:`AgentEvaluator` can score and report Gym runs through the same seam as
-Harbor/Fabric/Model. Gym owns execution *and* scoring; this runtime imports an
-existing Gym environment as-is — Gym is treated as a self-scoring engine and the
-runtime only adapts its rollout bundle, it does not re-derive rewards.
-
-**Mapping** (Gym → Evaluator): one Gym dataset → one run; each distinct row →
-one :class:`AgentEvalTask` (id = content hash of the row); each attempt
-(``_ng_rollout_index``) → one :class:`AgentEvalTrial`; the per-attempt verifier
-``reward`` → a :class:`GymRewardMetric` score. ``num_repeats=R`` therefore yields
-up to R trials per task. Row duplication is *not* a way to ask for repeated
-attempts — ``num_repeats`` is (see :func:`discover_gym_tasks`).
-
-**Attribution** is by ``_ng_task_index``, which this runtime *assigns* rather
-than infers. Gym only auto-assigns an index when a row doesn't already carry one
-(``rollout_collection._preprocess_rows_from_config``), and its own fallback
-dedup keys off the **raw jsonl line text** — a rule we cannot reproduce from
-parsed rows. So instead of guessing, :meth:`GymAgentTaskRunner.run_tasks`
-materializes a normalized dataset (one line per requested task, ``_ng_task_index``
-stamped explicitly) and feeds *that* to Gym. Gym echoes the index back on every
-rollout record, giving a total, order-independent ``index → task`` map. This also
-means a caller can run a **subset** of tasks without Gym rolling out the rest.
-
-**Execution** is the two-step Gym flow (the one that reads a dataset directly
-without triggering Gym's split-driven data-prep), preceded by a pre-flight:
-``gym env validate`` merges the composed config and reports unset ``???`` values,
-bad paths, and dangling cross-references without starting anything; then ``gym env
-start`` brings up the resources-server + agent + model servers, and ``gym eval run
---no-serve --input <materialized dataset>`` collects rollouts against them. Both
-commands receive the identical selection arguments, so what is validated is what
-runs. The runtime shells out to the ``gym`` CLI on PATH, so this SDK never imports
-``nemo_gym``. Subprocess
-output is streamed to log files under the run's work dir *and* mirrored to this
-module's logger at ``DEBUG``, so callers choose terminal visibility through
-ordinary ``logging`` configuration.
-
-**Where Gym finds things.** NeMo Gym must be installed and its ``gym`` on PATH,
-along with the target environment's own dependencies. Generally that means a
-*separate* environment: Gym imports Ray at module load, and nemo-platform
-excludes Ray by constraint over an unfixed CVE, so the two cannot share one. In a
-job image the image owns PATH and this is unremarkable. There is deliberately no
-config field naming a checkout, a venv, or a search root — these runner configs
-become serialized job specs, and a local filesystem path means nothing on the
-other side of that boundary. Environments themselves ship in the ``nemo-gym``
-wheel (``resources_servers`` and friends install beside ``nemo_gym``, configs and
-example data included), so no checkout is needed to reach them.
-
-The subprocesses inherit this process's working directory, which is where Gym
-looks for the gitignored ``env.yaml`` holding the collector's credentials before
-falling back to its install root — so credentials never pass through this SDK.
-Run from the directory holding that file; a Gym checkout there also has its
-components take precedence, which is how you reach an environment the wheel does
-not carry.
-
-**Boundaries**: the caller is responsible for a
-Gym runtime whose deps are installed (each Gym env ships its own
-``requirements.txt``), and for handing a *ready-to-run* dataset file (``--no-serve
---input`` bypasses Gym's prompt-templating/materialization). Service-side
-provisioning (docker/k8s, Ray) is out of scope here — that is the plugin's job.
-
-A consequence of that bypass: an environment whose rows carry no rendered prompt
-(``responses_create_params.input == []``, the prompt supplied by data-prep or by the
-environment's own agent) is still supported — the row travels through this runtime intact
-and the task simply has no ``inputs['instruction']``. See :func:`discover_gym_tasks`.
-"""
-
-from __future__ import annotations
-
-import asyncio
-import hashlib
-import json
-import logging
-import math
-import os
-import re
-import shutil
-import signal
-import tempfile
-from collections import deque
-from collections.abc import Mapping, Sequence
-from pathlib import Path
-from typing import Any
-
-from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask
-from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput, RunnerInfo
-from nemo_evaluator_sdk.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult
-from nemo_evaluator_sdk.values.evidence import CandidateEvidence, EvidenceDescriptor
-from nemo_evaluator_sdk.values.results import AggregateRangeScore, AggregateScalarScore, AggregateScore
-from pydantic import BaseModel, ConfigDict, Field
-
-logger = logging.getLogger(__name__)
-
-#: Reward key read from each Gym rollout record.
-DEFAULT_REWARD_KEY = "reward"
-#: Gym's CLI, expected on PATH. Not configurable: these runner configs become serialized job specs,
-#: and a path into somebody's venv is meaningless on the other side of that boundary. Note this name
-#: is only ever *resolved*, never executed: :func:`_gym_executable` turns it into an absolute path
-#: once, and that path is what the subprocesses run — so a child whose PATH differs from ours cannot
-#: end up executing a different Gym.
-_GYM_CLI = "gym"
-#: Bound on `gym env validate`. It merges config without starting anything and returns in about a
-#: second; this only exists so a wedged invocation cannot stall the run before it begins.
-_VALIDATE_TIMEOUT_S = 120.0
-#: Where Gym's Hydra run directories are redirected, relative to the run's work dir.
-_HYDRA_SUBDIR = "gym_hydra"
-#: `gym env start`'s combined output, under the run's work dir. Named here because a *collection*
-#: failure often has to point at it: the eval logs show the symptom, this shows the cause.
-_ENV_LOG_NAME = "gym_env.log"
-#: Token-count keys read to decide whether a model was called at all. Both vocabularies, because
-#: Gym's model servers report in either depending on the adapter: the Responses API spells them
-#: `input_tokens`/`output_tokens`, Chat Completions `prompt_tokens`/`completion_tokens`.
-_TOKEN_USAGE_KEYS = ("total_tokens", "input_tokens", "output_tokens", "prompt_tokens", "completion_tokens")
-#: The subset that proves a call happened. A model call always consumes input tokens, so zero on any
-#: of these is evidence the model was never reached; zero output tokens only means an empty answer.
-_INPUT_TOKEN_KEYS = ("total_tokens", "input_tokens", "prompt_tokens")
-#: Gym's index fields on each rollout record. ``_ng_task_index`` is the only join back to the input
-#: rows that survives a round-trip: Gym mutates ``responses_create_params`` (even the prompt) and
-#: copies only a fixed allowlist of row keys onto the result, so no field we invent comes back. Gym
-#: *honors* a caller-supplied ``_ng_task_index``, which is what makes the join here deterministic.
-NG_TASK_INDEX = "_ng_task_index"
-NG_ROLLOUT_INDEX = "_ng_rollout_index"
-#: Fields excluded from a row's content hash (runtime-injected, not task-defining).
-_RUNTIME_KEYS = frozenset({NG_TASK_INDEX, NG_ROLLOUT_INDEX})
-#: Lines of subprocess output retained in memory for inclusion in a failure message.
-_LOG_TAIL_LINES = 40
-
-
-#: Substrings that mark an override as carrying a credential. Matched case-insensitively against the
-#: full dotted path, so nesting cannot hide one behind an innocuous leaf name.
-_SECRET_KEY_MARKERS = ("api_key", "apikey", "token", "secret", "password", "passwd", "credential")
-#: Stand-in written in place of a redacted override value.
-_REDACTED = "<redacted>"
-
-
-#: Dict keys Hydra reads back unchanged. Its ``dictKey`` rule accepts no quoting, so a key is
-#: whatever the lexer makes of the bare text — this is deliberately narrower than what parses.
-_HYDRA_DICT_KEY = re.compile(r"[A-Za-z_][A-Za-z0-9_.-]*\Z")
-#: Bare words the lexer types rather than reading as text, so they cannot serve as string keys.
-_HYDRA_KEY_LITERALS = frozenset({"true", "false", "null", "inf", "nan"})
-
-
-def _hydra_dict(value: Mapping[str, Any]) -> str:
-    """Render a mapping as a Hydra dict container, ``{key:value,...}``.
-
-    Reached for a mapping nested inside a container — ``[{"b": 1}]`` — where there is no dotted path
-    to flatten onto, so the dict has to be spelled inline. Values recurse, so the typed spellings
-    below hold at any depth.
-
-    Keys are emitted bare, because Hydra's ``dictKey`` rule has no quoted form: ``{'b':1}`` does not
-    parse at all. That leaves the key at the mercy of the lexer, which types it — ``{true:1}`` keys
-    on the boolean ``True``, ``{1.5:1}`` on a float — and rejects ``:``, ``,``, brackets, and quotes
-    outright. Anything outside the conservative shape above therefore raises here rather than
-    silently keying the config on something the caller did not write.
-    """
-    rendered = []
-    for key, item in value.items():
-        if not isinstance(key, str) or not _HYDRA_DICT_KEY.match(key) or key.casefold() in _HYDRA_KEY_LITERALS:
-            raise ValueError(
-                f"Gym config override has dict key {key!r}, which Hydra's override grammar cannot "
-                "express as a string: keys are unquoted, so only a leading letter or underscore "
-                "followed by letters, digits, '_', '.', or '-' survives the round trip. Set this "
-                "key through the override path instead of nesting it inside a list."
-            )
-        rendered.append(f"{key}:{_hydra_scalar(item)}")
-    return "{" + ",".join(rendered) + "}"
-
-
-def _hydra_scalar(value: Any) -> str:
-    """Render a leaf value the way Hydra's override grammar reads it back.
-
-    Hydra's grammar is typed, so an unquoted string is not necessarily a string: ``true`` parses as a
-    boolean, ``null`` as ``None``, ``1.5`` as a float, ``a,b`` as a *sweep*, and ``A[B`` fails to
-    parse outright. Strings are therefore always single-quoted, which round-trips every one of those
-    (verified against ``hydra.core.override_parser``). Interpolations survive quoting — the override
-    sets the literal text and OmegaConf resolves it on read — so ``${policy_base_url}`` still works.
-
-    Only ``'`` is escaped. Hydra does **not** decode ``\\\\`` inside a quoted value: escaping
-    backslashes doubles them, so they are passed through raw.
-
-    ``None`` and booleans get their own spellings, since ``str()`` would emit ``"None"``/``"True"``
-    and Hydra reads those back as text. Containers recurse for the same reason: ``str()`` on a dict
-    emits Python's repr, whose quoted keys Hydra rejects outright.
-    """
-    if value is None:
-        return "null"
-    if isinstance(value, bool):
-        return "true" if value else "false"
-    if isinstance(value, Mapping):
-        return _hydra_dict(value)
-    if isinstance(value, (list, tuple)):
-        return "[" + ",".join(_hydra_scalar(item) for item in value) + "]"
-    if isinstance(value, str):
-        # A trailing backslash would escape the closing quote and leave the value unterminated, and
-        # there is no spelling that avoids it — better to say so than to emit something unparseable.
-        if value.endswith("\\"):
-            raise ValueError(
-                f"Gym config override value {value!r} ends with a backslash, which Hydra's override "
-                "grammar cannot express: it escapes the closing quote."
-            )
-        return "'" + value.replace("'", "\\'") + "'"
-    return str(value)
-
-
-def _flatten_overrides(overrides: Mapping[str, Any], _prefix: str = "") -> list[str]:
-    """Flatten a nested override mapping into Hydra ``++dotted.path=value`` arguments.
-
-    Callers describe overrides as structured data — ``{"a": {"b": 1}}`` — rather than as
-    pre-serialized Hydra strings, so the config survives being sent somewhere as JSON. Hydra itself
-    only speaks the flat form, so the translation happens here, at the point of invocation.
-
-    ``++`` rather than ``+``: it sets a key whether or not it already exists, which is what an
-    override means. A bare ``+`` fails on a key the merged config already defines.
-    """
-    arguments: list[str] = []
-    for key, value in overrides.items():
-        path = f"{_prefix}{key}"
-        # An empty mapping has no leaves to descend to, so recursing would drop the override
-        # entirely. It is still a value the caller asked to set: emit it as ``++path={}``, which
-        # clears the subtree.
-        if isinstance(value, Mapping) and value:
-            arguments.extend(_flatten_overrides(value, f"{path}."))
-        else:
-            arguments.append(f"++{path}={_hydra_scalar(value)}")
-    return arguments
-
-
-def _redact_env_overrides(overrides: Mapping[str, Any], _prefix: str = "") -> dict[str, Any]:
-    """Redact credential-looking values from overrides before they are recorded as provenance.
-
-    ``env_overrides`` is a free-form escape hatch forwarded to Gym, so nothing stops a caller passing
-    ``{"model": {"api_key": "sk-..."}}``. ``RunnerInfo.config`` is persisted into the run bundle, so a
-    value that looks like a credential must not be written there.
-
-    The *key* is always kept — knowing that a run overrode ``model.api_key`` is useful provenance;
-    knowing the value is a leak. Matching is on the full dotted path, so a marker anywhere in it
-    redacts, and nesting cannot hide a credential behind an innocuous leaf name.
-
-    Lists are walked too, since a mapping inside one — ``{"models": [{"api_key": "sk-..."}]}`` —
-    reaches Gym just as a nested mapping does. The index contributes no path segment: what marks a
-    value as a credential is the key it sits under, not where in a list it happens to fall.
-    """
-    redacted: dict[str, Any] = {}
-    for key, value in overrides.items():
-        path = f"{_prefix}{key}"
-        if isinstance(value, Mapping):
-            redacted[key] = _redact_env_overrides(value, f"{path}.")
-        elif any(marker in path.casefold() for marker in _SECRET_KEY_MARKERS):
-            redacted[key] = _REDACTED
-        elif isinstance(value, (list, tuple)):
-            redacted[key] = [_redact_list_item(item, path) for item in value]
-        else:
-            redacted[key] = value
-    return redacted
-
-
-def _redact_list_item(item: Any, path: str) -> Any:
-    """Redact inside one element of a list-valued override. See :func:`_redact_env_overrides`."""
-    if isinstance(item, Mapping):
-        return _redact_env_overrides(item, f"{path}.")
-    if isinstance(item, (list, tuple)):
-        return [_redact_list_item(nested, path) for nested in item]
-    return item
-
-
-def _selection_args(config: GymRuntimeConfig, work_dir: Path) -> list[str]:
-    """The environment/agent/model selection passed to Gym.
-
-    Built once and handed verbatim to both ``gym env validate`` and ``gym env start``, so what is
-    validated is exactly what runs — a pre-flight against a different config would be worse than
-    none.
-    """
-    selection = [
-        "--config",
-        config.agent_config,
-        "--model-type",
-        config.model_type,
-        "--resources-server",
-        config.resources_server,
-    ]
-    if config.bind_resources_server:
-        # Composable (Pattern-A) agents leave resources_server.name unbound ('???'); bind it to the
-        # env we're running. Assumes the agent config's top-level key equals the agent name (the
-        # simple_agent convention) *and* that the resources-server is registered under the
-        # environment's own name — not universally true, so self-contained or differently-named
-        # servers set bind_resources_server=False and bind themselves via env_overrides.
-        selection.append(
-            f"+{config.agent}.responses_api_agents.{config.agent}.resources_server.name={config.resources_server}"
-        )
-    selection.extend(_flatten_overrides(config.env_overrides))
-    # Gym is a Hydra app, so each invocation writes a timestamped run directory — by default
-    # `outputs/<date>/<time>/` under the *current* directory. Since the subprocesses inherit this
-    # process's cwd (so Gym can find env.yaml), the default would litter whatever directory the
-    # caller happened to run from. Redirect it under the run's work dir, with the rest of the run's
-    # artifacts. Applies to every Gym entry point, `gym list` included.
-    selection.append(f"hydra.run.dir={work_dir / _HYDRA_SUBDIR}")
-    return selection
-
-
-def _canonical_row_hash(row: Mapping[str, Any]) -> str:
-    """Stable ``sha256`` of a Gym dataset row, excluding runtime-injected fields.
-
-    A given row keeps its id across dataset revisions/reorderings (so Intake
-    rollups stay consistent), and a changed row becomes a new task. No
-    dataset-revision component — identity is the row content alone.
-    """
-    payload = {key: value for key, value in row.items() if key not in _RUNTIME_KEYS}
-    encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str)
-    return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
-
-
-def _content_text(content: Any) -> str:
-    """Flatten a message ``content`` (a string, or a list of ``{type,text}`` parts) to text.
-
-    Anything else contributes nothing. A missing ``content`` is unremarkable, but a *populated* value
-    in some other shape means we're silently dropping prompt text, so warn — :func:`_render_instruction`
-    only fails loudly when the whole row renders empty, which a single skipped message wouldn't trigger.
-    """
-    if isinstance(content, str):
-        return content
-    # `str` is itself a Sequence, so this branch only ever sees non-str sequences (handled above).
-    if isinstance(content, Sequence):
-        parts = [part["text"] for part in content if isinstance(part, Mapping) and isinstance(part.get("text"), str)]
-        return "\n".join(parts)
-    if content is not None:
-        logger.warning(
-            "Ignoring message content of unsupported type %s (expected a string or a list of {type,text} parts); "
-            "any prompt text it carries will be missing from the task instruction.",
-            type(content).__name__,
-        )
-    return ""
-
-
-def _render_instruction(responses_create_params: Mapping[str, Any]) -> str:
-    """Derive a task instruction from a row's ``responses_create_params``, or ``""`` when it carries none.
-
-    Uses ``instructions`` (system prompt, when present) + ``input`` (a plain string
-    or an OpenAI message list).
-
-    An empty result is *not* an error. A whole class of Gym environments ships
-    ``"responses_create_params": {"input": []}`` and keeps the task elsewhere — ``gdpval``
-    in a top-level ``prompt``, ``legal_agent_bench`` in an ``instance_id`` its own agent
-    resolves, ``aviary``/``toolsandbox`` in nothing but a ``task_idx``. Their prompt is
-    materialized by Gym's data-preparation step or by the environment's agent, neither of
-    which this runner drives (``--no-serve --input`` hands Gym a ready-to-run dataset by
-    design). Nothing here can reconstruct it, and no single row field generalizes across
-    them, so the honest answer is no instruction rather than a guess. See
-    :func:`discover_gym_tasks` for what that means downstream.
-    """
-    parts: list[str] = []
-    instructions = responses_create_params.get("instructions")
-    if isinstance(instructions, str) and instructions.strip():
-        parts.append(instructions.strip())
-    raw_input = responses_create_params.get("input")
-    if isinstance(raw_input, str):
-        parts.append(raw_input)
-    elif isinstance(raw_input, Sequence):
-        for item in raw_input:
-            if isinstance(item, Mapping):
-                text = _content_text(item.get("content"))
-                if text:
-                    parts.append(text)
-    return "\n\n".join(part for part in parts if part).strip()
-
-
-def _read_jsonl(path: str | Path, *, tolerant: bool = False) -> list[dict[str, Any]]:
-    """Read a jsonl file. With ``tolerant=True``, skip (and log) malformed lines instead of raising —
-
-    used for Gym's ``*_failures.jsonl`` sidecar, which is written during abnormal termination and can
-    end in a truncated line; a corrupt failure record must not sink the successfully-collected trials.
-    """
-    rows: list[dict[str, Any]] = []
-    with Path(path).open(encoding="utf-8") as handle:
-        for line_no, line in enumerate(handle, 1):
-            stripped = line.strip()
-            if not stripped:
-                continue
-            try:
-                rows.append(json.loads(stripped))
-            except json.JSONDecodeError:
-                if not tolerant:
-                    raise
-                logger.warning("Skipping malformed JSON at %s:%d", path, line_no)
-    return rows
-
-
-class GymRuntimeConfig(BaseModel):
-    """Declarative config for running an existing Gym environment via the ``gym`` CLI.
-
-    Holds only plain fields; the two-step invocation is built from them at run
-    time. The dataset itself is recovered from the tasks (stamped by
-    :func:`discover_gym_tasks`), mirroring the Harbor runner.
-    """
-
-    model_config = ConfigDict(extra="forbid")
-
-    agent: str = Field(description="Agent name to collect rollouts with, e.g. 'simple_agent'.")
-    agent_config: str = Field(description="Repo-relative agent config passed to `gym env start` (--config).")
-    resources_server: str = Field(description="Resources-server (environment) name, e.g. 'mcqa' (--resources-server).")
-    model_type: str = Field(
-        default="inference_provider",
-        description="Model-type config (--model-type). `inference_provider` speaks OpenAI-compatible chat; "
-        "`openai_model` uses the OpenAI Responses API (500s against chat-only endpoints).",
-    )
-    bind_resources_server: bool = Field(
-        default=True,
-        description="Auto-bind the agent's `resources_server.name` to `resources_server` via a Hydra override "
-        "(the composable/Pattern-A agent case, e.g. simple_agent whose config leaves it '???'). Set False for "
-        "self-contained agents that already bind their own resources-server.",
-    )
-    env_overrides: dict[str, Any] = Field(
-        default_factory=dict,
-        description="Nested config overrides merged into Gym's config, applied after the auto-derived "
-        "resources-server binding. Structured rather than pre-serialized Hydra strings so the config "
-        "travels as JSON — `{'a': {'b': 1}}` becomes `++a.b=1` at invocation. This is the escape "
-        "hatch for what Gym does not standardize: an environment whose resources-server is registered "
-        "under a different name, or which references a model server no shipped config defines.",
-    )
-    num_repeats: int = Field(default=1, ge=1, description="Attempts per row; each attempt becomes one trial.")
-    concurrency: int = Field(
-        default=4,
-        ge=1,
-        description="Concurrent rollouts for `gym eval run` (the collection-phase knob, tuned to the model "
-        "endpoint's limits). Distinct from AgentEvalRunConfig.parallelism, which bounds concurrent scoring.",
-    )
-    startup_timeout_s: float = Field(default=240.0, gt=0, description="Max wait for `gym env start` readiness.")
-    collection_timeout_s: float | None = Field(
-        default=None,
-        gt=0,
-        description="Max wait for `gym eval run` collection; None = unbounded (scales with dataset x num_repeats x "
-        "model latency, so no safe fixed default). Set it to bound a hung/slow model endpoint.",
-    )
-    shutdown_grace_s: float = Field(
-        default=30.0,
-        gt=0,
-        description="Grace period for a Gym subprocess *group* to exit on SIGTERM (letting Ray shut down cleanly) "
-        "before escalating to SIGKILL.",
-    )
-    reward_key: str = Field(default=DEFAULT_REWARD_KEY, description="Key read from each rollout record.")
-
-
-def _gym_executable() -> str:
-    """Locate the ``gym`` CLI on PATH, or fail saying what to do about it.
-
-    Deliberately PATH-only, with no config field pointing at a checkout or a particular venv: these
-    runner configs become serialized job specs, and a local filesystem path cannot cross that
-    boundary. Resolving here rather than at spawn time turns a missing Gym into one legible error
-    instead of an ``ENOENT`` out of ``create_subprocess_exec`` after the run has already started.
-
-    Note that Gym generally cannot live in this SDK's own environment: it imports Ray at module load,
-    and nemo-platform excludes Ray by constraint over an unfixed CVE. Install Gym separately and put
-    its ``bin`` on PATH; in a job image, the image owns PATH and this resolves normally.
-    """
-    resolved = shutil.which(_GYM_CLI)
-    if resolved is None:
-        raise RuntimeError(
-            f"The {_GYM_CLI!r} CLI was not found on PATH. Install NeMo Gym in its own environment "
-            "(it needs Ray, which nemo-platform excludes over an unfixed CVE, so it cannot share this "
-            "one) and put that environment's `bin` directory on PATH. Each resources-server also "
-            "ships its own requirements.txt, installed from that server's directory."
-        )
-    return resolved
-
-
-class GymRewardMetric:
-    """Score the Gym verifier reward stamped onto trial metadata.
-
-    The Gym analogue of :class:`HarborRewardMetric`: reads the per-trial ``reward``
-    off the candidate metadata (populated by :class:`GymAgentTaskRunner`); a trial
-    with no reward is left **unscored** (``None`` → ``nan``), excluded from the mean
-    and surfaced as ``nan_count`` rather than counted as a spurious ``0.0``. Gym owns
-    the scoring — this metric only surfaces it (Evaluator does not re-derive the reward).
-    """
-
-    def __init__(self, *, output_name: str = "reward", metric_type: str = "gym_reward") -> None:
-        self._output_name = output_name
-        self._metric_type = metric_type
-
-    @property
-    def type(self) -> str:
-        return self._metric_type
-
-    def output_spec(self) -> list[MetricOutputSpec]:
-        return [MetricOutputSpec.continuous_score(self._output_name)]
-
-    async def compute_scores(self, input: MetricInput) -> MetricResult:
-        reward = input.candidate.metadata.get("reward")
-        value = float(reward) if reward is not None else None
-        return MetricResult(outputs=[MetricOutput(name=self._output_name, value=value)])
-
-
-def discover_gym_tasks(dataset: str | Path, *, metrics: Sequence[Any] | None = None) -> list[AgentEvalTask]:
-    """Build one :class:`AgentEvalTask` per distinct row in a Gym dataset (jsonl).
-
-    Each task's id is the content hash of the row; its ``instruction`` is rendered
-    from ``responses_create_params.input`` *when the row carries one*; the raw params
-    are stashed under ``inputs['gym_row']`` for provenance; and it is scored by a
-    :class:`GymRewardMetric`. The dataset path is stamped on
-    ``metadata['gym_dataset_path']``, and every *other* row key (the verifier's
-    ground-truth fields, ``agent_ref``, and so on) on ``metadata['gym_row_extras']``.
-    Together with ``inputs['gym_row']`` those reconstruct the complete source row,
-    which :class:`GymAgentTaskRunner` re-materializes into the dataset it hands to Gym.
-    They are kept disjoint deliberately: the task record is persisted to the run
-    bundle's ``tasks.jsonl``, and ``responses_create_params`` is usually the bulk of a
-    row, so storing it in both places would write every dataset twice per run.
-
-    **One distinct row is one task.** Duplicate rows collapse (identity is row content,
-    so they are by definition the same task) and are reported as a warning: repeated
-    attempts are a run-level concern — ``GymRuntimeConfig.num_repeats`` — not something
-    a dataset expresses by repeating a row, so duplicates almost always mean bad data.
-
-    **A row need not carry a prompt.** A sweep of the 106 built-in environments shipping example
-    data found 5 — ``aviary``, ``gdpval``, ``legal_agent_bench``, ``scicode`` and ``toolsandbox``
-    — whose every row ships ``responses_create_params.input == []``, letting Gym's data-prep step
-    or the environment's own agent materialize the prompt instead (see
-    :func:`_render_instruction`). Such a row yields a task with **no** ``inputs['instruction']``
-    key rather than a fabricated one, and is counted in a summary log line. This costs the
-    Gym path nothing: the runner never reads ``instruction`` — it re-materializes the source
-    row from ``inputs['gym_row']`` + ``metadata['gym_row_extras']`` and hands *that* to Gym,
-    and ``intent`` is a dataset label either way. It stays absent (not ``""``) so that
-    :meth:`AgentEvalTask.agent_prompt` still fails loudly for the runners that *do* need a
-    prompt — an instruction-less task must not reach an agent as an empty one.
-
-    ``responses_create_params`` itself remains required: Gym indexes into it unconditionally
-    (``rollout_collection._preprocess_rows_from_config``), so a row without it is rejected here
-    rather than crashing Gym mid-collection.
-    """
-    dataset = Path(dataset)
-    tasks: list[AgentEvalTask] = []
-    seen: set[str] = set()
-    duplicates = 0
-    promptless = 0
-    for position, row in enumerate(_read_jsonl(dataset), 1):
-        params = row.get("responses_create_params")
-        if not isinstance(params, Mapping):
-            raise ValueError(
-                f"row {position} of {dataset} has no 'responses_create_params' mapping; Gym requires that key "
-                "on every dataset row"
-            )
-        task_id = _canonical_row_hash(row)
-        if task_id in seen:
-            duplicates += 1
-            continue
-        seen.add(task_id)
-        instruction = _render_instruction(params)
-        if not instruction:
-            promptless += 1
-        tasks.append(
-            AgentEvalTask(
-                id=task_id,
-                intent=f"Gym row from {dataset.name}",
-                inputs={
-                    # Absent, not empty, when the row carries no prompt — see the docstring.
-                    **({"instruction": instruction} if instruction else {}),
-                    "gym_row": params,
-                },
-                metrics=list(metrics) if metrics is not None else [GymRewardMetric()],
-                metadata={
-                    "gym_dataset_path": str(dataset),
-                    # Everything except responses_create_params, which already lives in inputs['gym_row'].
-                    "gym_row_extras": {
-                        key: value
-                        for key, value in row.items()
-                        if key != "responses_create_params" and key not in _RUNTIME_KEYS
-                    },
-                },
-            )
-        )
-    if duplicates:
-        logger.warning(
-            "Collapsed %d duplicate row(s) in %s into %d distinct task(s): task identity is row content, so "
-            "repeating a row cannot request repeated attempts — set GymRuntimeConfig.num_repeats for that. "
-            "Duplicate rows usually indicate a data problem.",
-            duplicates,
-            dataset,
-            len(tasks),
-        )
-    if promptless:
-        logger.info(
-            "%d of %d task(s) in %s carry no prompt in responses_create_params and were discovered without an "
-            "inputs['instruction']. Expected for environments whose prompt is built by Gym's data-prep step or by "
-            "their own agent (e.g. aviary, gdpval, legal_agent_bench, scicode, toolsandbox); the Gym runner does "
-            "not read the instruction. A runner that hands the task straight to an agent will reject these tasks.",
-            promptless,
-            len(tasks),
-            dataset,
-        )
-    if not tasks:
-        raise ValueError(f"no rows found in Gym dataset {dataset}")
-    return tasks
-
-
-class GymAgentTaskRunner:
-    """An :class:`AgentTaskRunner` that runs an existing Gym env, then adapts its rollouts.
-
-    :meth:`run_tasks` materializes the requested tasks into a normalized dataset with
-    ``_ng_task_index`` stamped per row, runs the two-step Gym flow (``env start`` +
-    ``eval run --no-serve --input``) against it via subprocess, and fans the rollout
-    records out into one trial per ``_ng_rollout_index``. Because we assign the index,
-    attribution is a dict lookup rather than a positional guess, and Gym only rolls out
-    the tasks we asked for. The durable task **identity** stays our content hash.
-    """
-
-    def __init__(self, *, config: GymRuntimeConfig) -> None:
-        self._config = config
-        self._run_aggregations: dict[str, Any] | None = None
-
-    def run_aggregate_scores(self) -> Sequence[AggregateScore]:
-        """Gym's ``agent_metrics`` mapped onto typed aggregate scores, namespaced ``runner.gym.<metric>``.
-
-        Satisfies :class:`RunAggregationsProvider`. ``reward`` is skipped: the SDK already scores it
-        natively as ``gym_reward.reward``, and two differently-derived numbers under one name invites
-        exactly the confusion the namespace is there to prevent.
-        """
-        return _aggregate_scores_from_gym(self._run_aggregations)
-
-    def runner_info(self) -> RunnerInfo:
-        """Identify this runner and the Gym settings that shape its results.
-
-        Credentials normally live in the Gym checkout's gitignored ``env.yaml`` and never reach this
-        object — but ``env_overrides`` is a free-form escape hatch, so its values are redacted by key
-        (see :func:`_redact_env_overrides`) rather than trusted.
-        """
-        cfg = self._config
-        return RunnerInfo(
-            name="gym",
-            kind="runner",
-            config={
-                "resources_server": cfg.resources_server,
-                "agent": cfg.agent,
-                "agent_config": cfg.agent_config,
-                "model_type": cfg.model_type,
-                "num_repeats": cfg.num_repeats,
-                "concurrency": cfg.concurrency,
-                "bind_resources_server": cfg.bind_resources_server,
-                "env_overrides": _redact_env_overrides(cfg.env_overrides),
-                "reward_key": cfg.reward_key,
-            },
-        )
-
-    async def run_tasks(
-        self,
-        tasks: Sequence[AgentEvalTask],
-        config: AgentEvalRunConfig | None = None,
-    ) -> list[AgentEvalTrial]:
-        cfg = self._config
-        self._run_aggregations = None  # reset per run so a reused runner never leaks a prior run's numbers
-        # Provenance for the log line only — the file Gym actually reads is the normalized one we
-        # materialize below from the tasks themselves.
-        source_dataset = _source_datasets(tasks)
-
-        # config.parallelism still governs how the evaluator *scores* the trials we return (its scoring
-        # semaphore). It is deliberately not mapped onto Gym's rollout `--concurrency`: those are different
-        # phases — parallelism bounds concurrent scoring (SDK-side, cheap), while Gym's `--concurrency`
-        # bounds concurrent rollouts against the model endpoint during collection (tuned to that endpoint's
-        # limits via GymRuntimeConfig.concurrency).
-        if config is not None and config.work_dir is not None:
-            work_dir = Path(config.work_dir) / "gym_run"
-        else:
-            work_dir = Path(tempfile.mkdtemp(prefix="gym_run_"))
-        work_dir.mkdir(parents=True, exist_ok=True)
-
-        rollouts_path = work_dir / "rollouts.jsonl"
-        _ensure_fresh_output(rollouts_path)
-
-        # Hand Gym a dataset we control: one row per requested task, each carrying the _ng_task_index
-        # we assign. Gym honors a pre-stamped index, so the rollout->task join becomes a total map
-        # instead of a positional guess about Gym's raw-line dedup (see the module docstring).
-        input_path = work_dir / "gym_input.jsonl"
-        index_to_task_id = _materialize_dataset(tasks, input_path)
-        logger.info(
-            "Materialized %d task(s) from %s into %s for Gym collection.",
-            len(index_to_task_id),
-            source_dataset,
-            input_path,
-        )
-
-        await self._run_two_step(input_path, rollouts_path, work_dir)
-        self._run_aggregations = _read_run_aggregations(rollouts_path)
-        trials = _trials_from_rollouts(rollouts_path, tasks, index_to_task_id, reward_key=cfg.reward_key)
-        _require_full_coverage(tasks, covered_task_ids={trial.task_id for trial in trials}, rollouts_path=rollouts_path)
-        return trials
-
-    async def _validate_config(
-        self, gym: str, selection: Sequence[str], subprocess_env: Mapping[str, str], work_dir: Path
-    ) -> None:
-        """Pre-flight the composed Gym config with ``gym env validate`` before starting anything.
-
-        Gym does not publish what configuration an environment requires — the typed
-        ``*ResourcesServerConfig`` classes cover behavioural knobs, while the model wiring lives in
-        each environment's YAML under names that vary per environment. ``gym env validate`` is the
-        only way to find out short of running: it merges configs, flags, and overrides, then reports
-        unresolved ``???`` values, bad paths, and cross-references to servers that are not defined —
-        without Ray and without starting a server.
-
-        Running it every time is worth the second it costs. A config mistake otherwise surfaces after
-        ``gym env start`` has brought up a Ray cluster and several uvicorn servers, as a readiness
-        timeout up to ``startup_timeout_s`` (240s by default) whose message says nothing about the
-        actual problem.
-        """
-        proc = await asyncio.create_subprocess_exec(
-            gym,
-            "env",
-            "validate",
-            *selection,
-            env=dict(subprocess_env),
-            stdout=asyncio.subprocess.PIPE,
-            stderr=asyncio.subprocess.STDOUT,
-            # Its own process group, like the other two Gym invocations. `_terminate` signals the
-            # group via `killpg`, so without this a validate timeout would signal *our* group — the
-            # SDK process included.
-            start_new_session=True,
-        )
-        try:
-            stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=_VALIDATE_TIMEOUT_S)
-        except TimeoutError:
-            await _terminate(proc, grace_s=self._config.shutdown_grace_s)
-            raise RuntimeError(
-                f"`gym env validate` did not finish within {_VALIDATE_TIMEOUT_S}s for resources-server "
-                f"{self._config.resources_server!r}"
-            ) from None
-
-        report = stdout.decode("utf-8", errors="replace").strip()
-        (work_dir / "gym_validate.log").write_text(report, encoding="utf-8")
-        if proc.returncode != 0:
-            raise RuntimeError(
-                f"Gym rejected the composed config for resources-server {self._config.resources_server!r} "
-                f"before any server started. `gym env validate` said:\n\n{report}"
-            )
-        logger.debug("gym env validate: %s", report)
-
-    async def _run_two_step(self, input_path: Path, output_path: Path, work_dir: Path) -> None:
-        """Start the Gym servers, collect against them with ``--no-serve``, then tear them down."""
-        cfg = self._config
-        gym = _gym_executable()
-        env_log = work_dir / _ENV_LOG_NAME
-
-        # Gym launches each server from its own subdir with its own .venv. Ray (>=2.56) otherwise
-        # detects a `uv run` ancestor and tries to replicate that uv project onto its workers,
-        # asserting the project pyproject.toml lives in the driver's cwd — which aborts startup. That
-        # hook is wrong for Gym (servers manage their own deps), so disable it for the subprocesses.
-        subprocess_env = {**os.environ, "RAY_ENABLE_UV_RUN_RUNTIME_ENV": "0"}
-
-        selection = _selection_args(cfg, work_dir)
-
-        await self._validate_config(gym, selection, subprocess_env, work_dir)
-
-        env_cmd = [gym, "env", "start", *selection]
-        # start_new_session=True puts `gym env start` in its own process group so teardown can signal
-        # the *whole* Ray-cluster + uvicorn tree, not just the direct child (else they orphan).
-        # stderr is merged into stdout here (unlike `eval run`, which splits them) because readiness
-        # detection scans a single chronological transcript and Gym's readiness line is not guaranteed
-        # to land on a particular stream.
-        env_proc = await asyncio.create_subprocess_exec(
-            *env_cmd,
-            env=subprocess_env,
-            stdout=asyncio.subprocess.PIPE,
-            stderr=asyncio.subprocess.STDOUT,
-            start_new_session=True,
-        )
-        env_pump = asyncio.create_task(_pump_stream(env_proc.stdout, env_log, label="gym env start"))
-        try:
-            await self._wait_for_servers(env_log, env_proc)
-            await self._collect_rollouts(gym, input_path, output_path, work_dir, subprocess_env)
-        finally:
-            await _terminate(env_proc, grace_s=cfg.shutdown_grace_s)
-            # Bounded: a leaked grandchild holding the inherited pipe must not wedge teardown.
-            await _drain_pumps([env_pump], grace_s=cfg.shutdown_grace_s, what="gym env start")
-
-    async def _collect_rollouts(
-        self, gym: str, input_path: Path, output_path: Path, work_dir: Path, subprocess_env: dict[str, str]
-    ) -> None:
-        """Run ``gym eval run --no-serve`` against the live servers, bounded and self-cleaning.
-
-        Spawned in its own process group and always torn down (even on timeout/cancellation), so a hung
-        collection can't wedge the run or leak Ray workers.
-
-        Output is *streamed* to ``gym_eval.stdout.log`` / ``gym_eval.stderr.log`` in ``work_dir`` rather
-        than buffered in memory (a long collection's transcript can be large), and mirrored to this
-        module's logger at ``DEBUG`` so callers can surface it in their own terminal through ordinary
-        ``logging`` configuration. Failures point at both files and inline the last few lines.
-        """
-        cfg = self._config
-        eval_cmd = [
-            gym,
-            "eval",
-            "run",
-            "--no-serve",
-            "--agent",
-            cfg.agent,
-            "--input",
-            str(input_path),
-            "--output",
-            str(output_path),
-            "--num-repeats",
-            str(cfg.num_repeats),
-            "--concurrency",
-            str(cfg.concurrency),
-            # `gym eval run` is a Hydra app too, and unlike validate/start it does not go through
-            # _selection_args — so without this it writes `outputs/<date>/<time>/` into the caller's
-            # cwd on every single collection.
-            f"hydra.run.dir={work_dir / _HYDRA_SUBDIR}",
-        ]
-        stdout_log = work_dir / "gym_eval.stdout.log"
-        stderr_log = work_dir / "gym_eval.stderr.log"
-        eval_proc = await asyncio.create_subprocess_exec(
-            *eval_cmd,
-            env=subprocess_env,
-            stdout=asyncio.subprocess.PIPE,
-            stderr=asyncio.subprocess.PIPE,
-            start_new_session=True,
-        )
-        tails: dict[str, deque[str]] = {}
-        pumps = [
-            asyncio.create_task(
-                _pump_stream(eval_proc.stdout, stdout_log, label="gym eval", tails=tails, key="stdout")
-            ),
-            asyncio.create_task(
-                _pump_stream(eval_proc.stderr, stderr_log, label="gym eval", tails=tails, key="stderr")
-            ),
-        ]
-        try:
-            await asyncio.wait_for(eval_proc.wait(), timeout=cfg.collection_timeout_s)
-        except asyncio.TimeoutError as exc:
-            raise TimeoutError(
-                f"`gym eval run` exceeded collection_timeout_s={cfg.collection_timeout_s}s; collection aborted. "
-                f"See {stdout_log} and {stderr_log}."
-            ) from exc
-        finally:
-            # Terminate first so the pipes close, then drain (bounded) to flush whatever was buffered.
-            await _terminate(eval_proc, grace_s=cfg.shutdown_grace_s)
-            await _drain_pumps(pumps, grace_s=cfg.shutdown_grace_s, what="gym eval run")
-        if eval_proc.returncode != 0:
-            tail = "\n".join(tails.get("stderr") or tails.get("stdout") or [])
-            # A collection failure is frequently *server*-side: the resources-server raises, the
-            # agent surfaces it as a bare HTTP 500, and the traceback that explains it is only in
-            # gym_env.log. Naming the two eval logs alone sends the reader to the one place the
-            # cause is not. (Observed on wmt_translation: a 500 here, `PermissionError: /opt/Gym`
-            # there.)
-            env_log = work_dir / _ENV_LOG_NAME
-            raise RuntimeError(
-                f"`gym eval run` failed (rc={eval_proc.returncode}). Collection output: {stdout_log} "
-                f"and {stderr_log}\n"
-                f"If the tail below is an HTTP error from a Gym server, the cause is server-side: "
-                f"see {env_log}, whose lines are prefixed with the server that emitted them.\n"
-                f"--- last {_LOG_TAIL_LINES} line(s) ---\n{tail}"
-            )
-
-    async def _wait_for_servers(self, env_log: Path, env_proc: asyncio.subprocess.Process) -> None:
-        """Wait until ``gym env start`` reports every composed server ready.
-
-        Keys off Gym's own readiness line (``All N / N servers ready!``), which it prints only after
-        HTTP-health-checking each server it composed. So the caller never has to know or guess the
-        server count — Gym derives it from the config and health-checks for us.
-        """
-        cfg = self._config
-        loop = asyncio.get_running_loop()
-        ready = re.compile(r"All \d+ / \d+ servers ready")
-        deadline = loop.time() + cfg.startup_timeout_s
-        text = ""
-        while loop.time() < deadline:
-            if env_proc.returncode is not None:
-                raise RuntimeError(f"`gym env start` exited early (rc={env_proc.returncode}); see {env_log}")
-            text = env_log.read_text(encoding="utf-8", errors="replace") if env_log.exists() else ""
-            if "required uv version" in text.lower():
-                raise RuntimeError(f"`gym env start` hit the uv required-version gate; see {env_log}")
-            if ready.search(text):
-                return
-            await asyncio.sleep(2)
-        raise TimeoutError(self._startup_timeout_message(text, env_log))
-
-    def _startup_timeout_message(self, env_log_text: str, env_log: Path) -> str:
-        """Explain a startup timeout in terms of *which* server did not come up.
-
-        The bare "servers not ready" message sends you to a log that is mostly noise from the servers
-        that started fine. Gym polls and prints the outstanding set every time, so naming it costs
-        nothing and is usually the whole diagnosis: one server out of four means that environment's
-        own startup, not the platform.
-
-        The timeout is also a plausible *cause* rather than a symptom — a Docker-backed environment
-        builds its image on first run, which routinely exceeds the 240s default — so the message says
-        how to extend it instead of leaving the reader to find the knob.
-        """
-        cfg = self._config
-        pending = _pending_servers(env_log_text)
-        if pending is None:
-            detail = (
-                "Gym never reported server readiness, so it likely failed before starting them "
-                "(config composition or dependency installation)."
-            )
-        elif not pending[2]:
-            # Gym printed a readiness line but named no outstanding server. Saying "waiting on:
-            # unknown" would contradict the counts in the same sentence, so report only what is known.
-            ready_count, total, _ = pending
-            detail = (
-                f"{ready_count} of {total} server(s) started; Gym did not name which remained "
-                f"outstanding. The per-server output in {env_log.name} is prefixed with the server "
-                "that emitted each line."
-            )
-        else:
-            ready_count, total, names = pending
-            detail = (
-                f"{ready_count} of {total} server(s) started; still waiting on: {', '.join(names)}. "
-                f"Search {env_log.name} for '({names[0]})' — each line is prefixed with the server "
-                "that emitted it."
-            )
-        return (
-            f"Gym servers for resources-server {cfg.resources_server!r} were not ready within "
-            f"startup_timeout_s={cfg.startup_timeout_s}s. {detail}\n"
-            f"Full startup output: {env_log}\n"
-            "If the environment installs heavy dependencies or builds a container image on first run, "
-            "this timeout is expected on a cold cache — raise GymRuntimeConfig.startup_timeout_s and "
-            "re-run; the second run reuses what the first installed."
-        )
-
-
-#: Gym's per-poll readiness line, e.g.
-#: ``3 / 4 servers ready. Waiting for servers to spin up: ['legal_agent_bench']``.
-#: Gym knows exactly which servers are outstanding; without this we discard that and report only
-#: that *something* timed out.
-_PENDING_SERVERS_RE = re.compile(r"(\d+)\s*/\s*(\d+) servers ready\. Waiting for servers to spin up: \[([^\]]*)\]")
-
-
-def _pending_servers(text: str) -> tuple[int, int, tuple[str, ...]] | None:
-    """Recover ``(ready, total, still-pending)`` from the *last* readiness line Gym logged.
-
-    The last line is the one that matters: earlier polls list servers that have since come up.
-    Returns ``None`` when Gym never got as far as printing one (it died during composition, say),
-    which the caller must treat as "no detail available" rather than "nothing pending".
-
-    Never raises — this only ever decorates an error that is already being reported, and a parsing
-    failure here must not replace the real one. Gym's log format is not contractual.
-    """
-    matches = _PENDING_SERVERS_RE.findall(text)
-    if not matches:
-        return None
-    ready, total, names = matches[-1]
-    try:
-        return int(ready), int(total), tuple(re.findall(r"['\"]([^'\"]+)['\"]", names))
-    except ValueError:  # pragma: no cover - the regex only matches digits
-        return None
-
-
-def _agent_never_ran(record: Mapping[str, Any]) -> bool:
-    """True when a result record shows the agent produced nothing *and* never called the model.
-
-    A verifier-scored runner reports whatever score the environment computed, so an agent that
-    silently failed to start is indistinguishable from one that tried and scored 0.0 — the record
-    looks normal, no failure is reported alongside it, and the run succeeds. Observed on
-    ``legal_agent_bench``, whose agent died locating its task tree and returned an empty response:
-    two "completed" trials scoring 0.0, no model call, no failure recorded.
-
-    Deliberately conservative — **both** conditions must hold:
-
-    * no output, and
-    * a stated zero on the input side of token usage.
-
-    An empty answer alone is a legitimate result (a model can return nothing and earn 0.0), so it is
-    not sufficient. Zero *input* tokens is what says the model was never called: a call always sends
-    a prompt, whereas zero output tokens only describes an empty answer. A record carrying no
-    ``response`` key at all is left alone — some runners populate it differently, and this must not
-    reclassify trials it does not understand.
-    """
-    response = record.get("response")
-    if not isinstance(response, Mapping):
-        return False
-    if "output" not in response:
-        # Nothing to judge: some runners shape `response` differently, and absence is not emptiness.
-        return False
-    # Any non-empty value counts as output, whatever its shape. Checking `len(...) > 0` on a list
-    # alone would treat a plain string answer — or a single mapping — as "produced nothing" and fail
-    # a trial that ran perfectly well.
-    if response["output"]:
-        return False
-    usage = response.get("usage")
-    if not isinstance(usage, Mapping):
-        # No usage block to corroborate with; an empty output on its own is not enough to judge.
-        return False
-    # Zero usage has to be *stated*, not inferred from a key being absent. Two vocabularies are in
-    # play — the Responses API's `input_tokens`/`output_tokens` and Chat Completions'
-    # `prompt_tokens`/`completion_tokens` — and Gym's model servers speak both depending on the
-    # adapter. Reading only one set would make a real call reported in the other read as zero.
-    reported = [usage[key] for key in _TOKEN_USAGE_KEYS if isinstance(usage.get(key), (int, float))]
-    if any(value > 0 for value in reported):
-        return False
-    # The discriminating evidence is *input*-side, and only input-side. Calling a model always sends
-    # a prompt, so zero input tokens cannot describe a call that happened; zero **output** tokens
-    # merely describes an empty answer, which is a legitimate result. Requiring an input-side count
-    # rules out a whole class of false positives — a provider reporting only `output_tokens: 0`, or
-    # only `completion_tokens: 0` — rather than excluding those shapes one at a time.
-    input_side = [usage[key] for key in _INPUT_TOKEN_KEYS if isinstance(usage.get(key), (int, float))]
-    return bool(input_side) and all(value == 0 for value in input_side)
-
-
-def _source_datasets(tasks: Sequence[AgentEvalTask]) -> str:
-    """Human-readable summary of the dataset(s) the tasks were discovered from, for a log line.
-
-    Purely provenance. Gym reads the dataset :func:`_materialize_dataset` writes from the tasks
-    themselves, so nothing here gates the run: tasks drawn from two datasets are legitimate (the
-    materialized file is their union), and a task with no stamped path runs fine. Never raises —
-    a missing or inconsistent label is not a reason to fail an otherwise valid evaluation.
-    """
-    stamped = {
-        task.metadata["gym_dataset_path"]
-        for task in tasks
-        if isinstance(task.metadata.get("gym_dataset_path"), str) and task.metadata["gym_dataset_path"]
-    }
-    return ", ".join(sorted(stamped)) if stamped else "<tasks with no stamped dataset path>"
-
-
-async def _pump_stream(
-    stream: asyncio.StreamReader | None,
-    path: Path,
-    *,
-    label: str,
-    tails: dict[str, deque[str]] | None = None,
-    key: str = "stdout",
-) -> None:
-    """Stream a subprocess pipe to ``path`` while mirroring it to the module logger at ``DEBUG``.
-
-    Reads in chunks rather than by line so a pathologically long line can't overrun asyncio's stream
-    limit, and retains only the last :data:`_LOG_TAIL_LINES` lines in memory (via ``tails[key]``) for
-    inclusion in a failure message — the file on disk is the complete record.
-    """
-    tail: deque[str] = deque(maxlen=_LOG_TAIL_LINES)
-    if tails is not None:
-        tails[key] = tail
-    if stream is None:
-        return
-
-    def emit(raw: bytes) -> None:
-        text = raw.decode("utf-8", errors="replace").rstrip("\r")
-        logger.debug("[%s %s] %s", label, key, text)
-        tail.append(text)
-
-    with path.open("wb") as handle:
-        buffer = b""
-        while True:
-            chunk = await stream.read(65536)
-            if not chunk:
-                break
-            handle.write(chunk)
-            handle.flush()
-            buffer += chunk
-            *complete, buffer = buffer.split(b"\n")
-            for line in complete:
-                emit(line)
-        if buffer:
-            emit(buffer)
-
-
-async def _drain_pumps(pumps: Sequence[asyncio.Task[None]], *, grace_s: float, what: str) -> None:
-    """Await log pumps after teardown, but never block on them indefinitely.
-
-    A pump ends at pipe EOF, which requires *every* inheritor of the write end to close it. Gym's
-    descendants inherit it, and Ray daemonizes ``gcs_server`` into its own session where our
-    process-group signals can't reach it (see :func:`_terminate`) — so a leaked grandchild can hold
-    the pipe open forever. Waiting unconditionally would hang the whole run inside a ``finally``.
-
-    Bounded instead: give the pumps ``grace_s`` to flush, then cancel and move on. Everything read
-    before the stall is already written and flushed to the log file, so cancelling costs at most the
-    tail of a transcript belonging to a process we just killed — and it is reported, not silent.
-    """
-    if not pumps:
-        return
-    _done, pending = await asyncio.wait(pumps, timeout=grace_s)
-    if not pending:
-        return
-    logger.warning(
-        "%d `%s` log pump(s) did not finish within %.1fs; the pipe is still held open (likely a Gym "
-        "grandchild that outlived teardown, e.g. Ray's detached gcs_server). Abandoning them — the log "
-        "file holds everything read up to this point, but its tail may be truncated.",
-        len(pending),
-        what,
-        grace_s,
-    )
-    for task in pending:
-        task.cancel()
-    await asyncio.gather(*pending, return_exceptions=True)
-
-
-def _materialize_dataset(tasks: Sequence[AgentEvalTask], dest: Path) -> dict[int, str]:
-    """Write the normalized dataset Gym will read, and return its ``_ng_task_index`` → task-id map.
-
-    One line per requested task, in task order, carrying the task's full source row plus an explicitly
-    stamped ``_ng_task_index``. Gym honors a pre-supplied index instead of deriving one, so this makes
-    the rollout→task join total and order-independent, and confines the run to the tasks we asked for.
-
-    The row is reassembled from ``inputs['gym_row']`` (``responses_create_params``) and
-    ``metadata['gym_row_extras']`` (everything else), which :func:`discover_gym_tasks` keeps disjoint
-    so the run bundle doesn't store the same payload twice. Any pre-existing ``_ng_*`` fields are
-    stripped: ours is authoritative, and Gym assigns ``_ng_rollout_index`` itself per attempt.
-    """
-    index_to_task_id: dict[int, str] = {}
-    seen_task_ids: set[str] = set()
-    lines: list[str] = []
-    for index, task in enumerate(tasks):
-        # The source row is stored split across inputs/metadata so the run bundle doesn't persist
-        # responses_create_params twice; reassemble it here.
-        extras = task.metadata.get("gym_row_extras")
-        params = task.inputs.get("gym_row")
-        if not isinstance(extras, Mapping) or not isinstance(params, Mapping):
-            raise ValueError(
-                f"task {task.id!r} is missing inputs['gym_row'] and/or metadata['gym_row_extras']; build tasks "
-                "with discover_gym_tasks so the Gym dataset can be re-materialized for the run"
-            )
-        if task.id in seen_task_ids:
-            raise ValueError(
-                f"task {task.id!r} was supplied more than once; one distinct row is one task, and repeated "
-                "attempts come from GymRuntimeConfig.num_repeats"
-            )
-        seen_task_ids.add(task.id)
-        payload = {key: value for key, value in extras.items() if key not in _RUNTIME_KEYS}
-        payload["responses_create_params"] = params
-        payload[NG_TASK_INDEX] = index
-        lines.append(json.dumps(payload, default=str))
-        index_to_task_id[index] = task.id
-    if not index_to_task_id:
-        raise ValueError("GymAgentTaskRunner was given no tasks to run")
-    dest.write_text("\n".join(lines) + "\n", encoding="utf-8")
-    return index_to_task_id
-
-
-def _require_full_coverage(tasks: Sequence[AgentEvalTask], *, covered_task_ids: set[str], rollouts_path: Path) -> None:
-    """Fail the run when Gym produced no trial at all for some requested task.
-
-    :class:`AgentEvaluator` already refuses to score a task with no trial
-    (``evaluator._score_trials``), and deliberately so: an incomplete run must not read as a
-    successful one with quieter counts. We keep that contract rather than papering over it with
-    synthesized FAILED trials — an unscored trial is excluded from the mean, so fabricating them
-    would report a plausible-looking average over an unannounced subset of the dataset. A rollout
-    that failed in a *diagnosable* way already becomes a FAILED trial via the failures sidecar;
-    reaching here means the attempt is unaccounted for entirely.
-
-    What this adds over letting the evaluator raise is diagnosis: by the time the run fails the full
-    collection cost is already paid, so the error names the evidence instead of just the task ids.
-    """
-    missing = sorted({task.id for task in tasks} - covered_task_ids)
-    if not missing:
-        return
-    raise RuntimeError(
-        f"Gym produced no rollout for {len(missing)} of {len(tasks)} requested task(s), so the run is "
-        f"incomplete and will not be scored (a partial run must not be reported as a whole one).\n"
-        f"  rollouts:  {rollouts_path}\n"
-        f"  failures:  {_failures_path_for(rollouts_path)}\n"
-        f"  gym logs:  {rollouts_path.parent} (gym_env.log, gym_eval.stdout.log, gym_eval.stderr.log)\n"
-        f"  unrepresented task id(s): {missing}"
-    )
-
-
-def _failures_path_for(rollouts_path: Path) -> Path:
-    """Sidecar Gym writes failed rollouts to (mirrors Gym's own ``_failures_path_for``)."""
-    return rollouts_path.with_name(rollouts_path.stem + "_failures.jsonl")
-
-
-def _aggregate_metrics_path_for(rollouts_path: Path) -> Path:
-    """Sidecar Gym writes run-level aggregate metrics to (``<stem>_aggregate_metrics.json``)."""
-    return rollouts_path.with_name(rollouts_path.stem + "_aggregate_metrics.json")
-
-
-def _read_run_aggregations(rollouts_path: Path) -> dict[str, Any] | None:
-    """Parse Gym's ``rollouts_aggregate_metrics.json``, or ``None`` when absent/unparseable.
-
-    Gym's file is a list with one entry per agent (``agent_ref`` / ``agent_metrics`` / ``key_metrics`` /
-    ``group_level_metrics``), so it is returned keyed by agent name. Carried through as-is — Gym's schema
-    isn't contractual, so the SDK does not type it.
-    """
-    path = _aggregate_metrics_path_for(rollouts_path)
-    if not path.exists():
-        return None
-    try:
-        parsed = json.loads(path.read_text(encoding="utf-8"))
-    except (json.JSONDecodeError, OSError, UnicodeDecodeError):
-        # UnicodeDecodeError is a ValueError, not an OSError: a sidecar truncated mid-codepoint would
-        # otherwise raise straight out of run_tasks and discard a collection that already succeeded.
-        logger.warning("Could not parse Gym aggregate metrics at %s; skipping run aggregations.", path)
-        return None
-    if not isinstance(parsed, list):
-        logger.warning("Unexpected Gym aggregate-metrics shape at %s (%s); skipping.", path, type(parsed).__name__)
-        return None
-    # Gym's schema is not contractual, and this runs after a collection that already succeeded — an
-    # entry in an unexpected shape must not raise out of run_tasks and discard every trial with it.
-    aggregations: dict[str, Any] = {}
-    for entry in parsed:
-        agent_ref = entry.get("agent_ref") if isinstance(entry, Mapping) else None
-        name = agent_ref.get("name") if isinstance(agent_ref, Mapping) else None
-        if not isinstance(name, str):
-            logger.warning(
-                "Skipping a Gym aggregate-metrics entry in %s with no usable agent_ref.name (got %r).", path, agent_ref
-            )
-            continue
-        aggregations[name] = {key: value for key, value in entry.items() if key != "agent_ref"}
-    return aggregations or None
-
-
-#: Gym flattens a distribution into ``<stat>/<metric>`` keys. Its RewardProfiler emits this exact set
-#: together for every numeric column (``describe_dataframe``), minus ``histogram``, which
-#: ``prepare_for_serialization`` strips before the file is written. All five must be present before we
-#: treat a group of keys as one distribution: a resources-server is free to define a metric literally
-#: named ``mean`` (36 of Gym's ~97 servers override ``compute_metrics``), and re-assembling on a partial
-#: match would rename someone's standalone metric into a statistic of a distribution that never existed.
-_GYM_STAT_FAMILY = ("mean", "max", "min", "median", "std")
-
-#: Metric Gym reports that the SDK already computes natively from the same rollouts (``gym_reward.reward``).
-_GYM_REDUNDANT_METRICS = frozenset({"reward"})
-
-
-def _aggregate_scores_from_gym(aggregations: Mapping[str, Any] | None) -> list[AggregateScore]:
-    """Map Gym's run-level ``agent_metrics`` onto typed aggregate scores named ``runner.gym.<metric>``.
-
-    Reads ``agent_metrics``, not ``key_metrics``: ``key_metrics`` is a *subset* of it chosen by the
-    resources-server, and the default selection (``get_key_metrics``) keeps only the ``mean/*`` entries —
-    so the max/min/median/std that make a distribution never appear there, and every metric would arrive
-    as a lone ``mean/<name>`` scalar.
-
-    Keys forming a full stat family are re-assembled into one :class:`AggregateRangeScore`; every other
-    numeric key becomes an :class:`AggregateScalarScore`. Non-numeric values are skipped — they are
-    labels or notes, not measurements.
-
-    Names are namespaced by runner (not by agent): each run is instrumented with a single agent, so the
-    agent adds no disambiguation, and a reader needs to know which *backend* produced a number.
-    """
-    if not aggregations:
-        return []
-    multi_agent = len(aggregations) > 1
-    scores: list[AggregateScore] = []
-    for agent_name, payload in sorted(aggregations.items()):
-        agent_metrics = payload.get("agent_metrics") if isinstance(payload, Mapping) else None
-        if not isinstance(agent_metrics, Mapping):
-            continue
-        # One agent per run is the norm, so `runner.gym.<metric>` reads cleanly; qualify by agent only
-        # when a run really did produce several, where the unqualified names would collide.
-        prefix = f"runner.gym.{agent_name}." if multi_agent else "runner.gym."
-        scores.extend(_scores_from_agent_metrics(agent_metrics, prefix=prefix))
-    return scores
-
-
-def _scores_from_agent_metrics(agent_metrics: Mapping[str, Any], *, prefix: str) -> list[AggregateScore]:
-    families: dict[str, dict[str, float]] = {}
-    scalars: dict[str, float] = {}
-    for key, value in agent_metrics.items():
-        number = _as_float(value)
-        if number is None:
-            continue
-        stat, _, metric = key.partition("/")
-        if metric and stat in _GYM_STAT_FAMILY:
-            families.setdefault(metric, {})[stat] = number
-        else:
-            scalars[key] = number
-
-    scores: list[AggregateScore] = []
-    for metric, stats in sorted(families.items()):
-        # Redundancy is a property of the metric, not of how complete its family is. Checking after the
-        # fallback below would let a partial `reward` family survive as `mean/reward` scalars, since the
-        # scalar filter matches the bare name -- reintroducing the duplicate of `gym_reward.reward` that
-        # skipping reward exists to prevent.
-        if metric in _GYM_REDUNDANT_METRICS:
-            continue
-        if set(stats) < set(_GYM_STAT_FAMILY):
-            # Not a distribution we can vouch for; keep each key as the standalone number it may well be.
-            scalars.update({f"{stat}/{metric}": value for stat, value in stats.items()})
-            continue
-        scores.append(
-            AggregateRangeScore(
-                name=f"{prefix}{metric}",
-                # Gym reports the statistics but not the sample size behind them, and inventing one
-                # would misreport coverage. `None` says "unknown"; 0 would assert nothing was evaluated.
-                count=None,
-                nan_count=0,
-                mean=stats["mean"],
-                min=stats["min"],
-                max=stats["max"],
-                median=stats["median"],
-                # Gym computes this with pandas (ddof=1), so it is the sample standard deviation.
-                sample_std_dev=stats["std"],
-                sample_variance=stats["std"] ** 2,
-            )
-        )
-    scores.extend(
-        AggregateScalarScore(name=f"{prefix}{key}", count=None, nan_count=0, value=value)
-        for key, value in sorted(scalars.items())
-        if key not in _GYM_REDUNDANT_METRICS
-    )
-    return scores
-
-
-def _as_float(value: Any) -> float | None:
-    """``float`` for a numeric Gym metric value; None for anything else (bools included: not measurements)."""
-    if isinstance(value, bool) or not isinstance(value, (int, float)):
-        return None
-    return float(value) if math.isfinite(value) else None
-
-
-def _ensure_fresh_output(rollouts_path: Path) -> None:
-    """Enforce one Gym run per output dir (the AgentEvaluator convention).
-
-    Gym appends to the failures sidecar (``open("ab")``) and doesn't clear it between runs, so reusing
-    a populated directory would silently mix this run's failures with a prior run's. Rather than clear
-    (which would clobber an earlier run's results — infra failures are useful signal), refuse to run
-    into a directory that already holds Gym rollout output.
-    """
-    preexisting = [path for path in (rollouts_path, _failures_path_for(rollouts_path)) if path.exists()]
-    if preexisting:
-        names = ", ".join(path.name for path in preexisting)
-        raise FileExistsError(
-            f"{rollouts_path.parent} already holds Gym rollout output ({names}); give each run a fresh "
-            "output_dir (the AgentEvaluator convention). Gym appends to the failures sidecar, so reusing a "
-            "directory would mix runs and could obscure a prior run's results."
-        )
-
-
-def _coerce_reward(value: Any) -> float | None:
-    """Best-effort ``float`` for a reward; returns None (never raises) on a missing/malformed value."""
-    if value is None:
-        return None
-    try:
-        return float(value)
-    except (TypeError, ValueError):
-        logger.warning("Gym rollout carried a non-numeric reward %r; recording the trial as unscored.", value)
-        return None
-
-
-def _resolve_task_id(
-    record: Mapping[str, Any], index_to_task_id: Mapping[int, str], *, strict: bool = True
-) -> str | None:
-    """Map a rollout record to a task id via the ``_ng_task_index`` we stamped in the input dataset.
-
-    Returns None when the record carries no usable index (kill-shaped failures can lack one).
-
-    An index we never stamped is handled by ``strict``. In the **successes** file (``strict=True``)
-    it is fatal: every row Gym read came from :func:`_materialize_dataset`, so an unknown index means
-    Gym reindexed the dataset rather than honoring our stamp, and every reward attribution is
-    therefore suspect. In the **failures sidecar** (``strict=False``) it is merely unattributable and
-    is counted instead — that file is written during abnormal termination and read tolerantly by
-    design, so one odd record must not discard the successes already parsed. That relaxation cannot
-    inflate a result: failure records carry no reward, and a task left with no trial still trips
-    :func:`_require_full_coverage`.
-    """
-    index = record.get(NG_TASK_INDEX)
-    if not isinstance(index, int) or isinstance(index, bool):
-        return None
-    task_id = index_to_task_id.get(index)
-    if task_id is None:
-        if not strict:
-            logger.warning(
-                "Gym failure record carried %s=%s, which was not stamped on the materialized dataset; "
-                "counting it as unattributable rather than failing the run.",
-                NG_TASK_INDEX,
-                index,
-            )
-            return None
-        raise ValueError(
-            f"Gym emitted {NG_TASK_INDEX}={index}, which is not one of the {len(index_to_task_id)} index value(s) "
-            f"stamped on the materialized dataset. Gym appears to have reassigned {NG_TASK_INDEX} instead "
-            "of honoring the supplied one, so rollout-to-task attribution is unsafe."
-        )
-    return task_id
-
-
-def _rollout_trial_id(task_id: str, raw_rollout_index: Any, synth_seq: dict[str, int], *, missing_label: str) -> str:
-    """Per-attempt trial id: use the record's ``_ng_rollout_index`` when it's a real int, else a
-    per-task-unique ``{missing_label}{n}`` suffix so records lacking an index don't collide on one id."""
-    if isinstance(raw_rollout_index, int) and not isinstance(raw_rollout_index, bool):
-        return f"{task_id}:{raw_rollout_index}"
-    seq = synth_seq[task_id] = synth_seq.get(task_id, 0) + 1
-    return f"{task_id}:{missing_label}{seq}"
-
-
-def _trials_from_rollouts(
-    rollouts_path: Path,
-    tasks: Sequence[AgentEvalTask],
-    index_to_task_id: Mapping[int, str],
-    *,
-    reward_key: str = DEFAULT_REWARD_KEY,
-) -> list[AgentEvalTrial]:
-    """Fan Gym's rollout records out into one :class:`AgentEvalTrial` per attempt.
-
-    Reads *both* Gym output files: successes from ``rollouts.jsonl`` (COMPLETED, carrying the reward)
-    and failures from the ``*_failures.jsonl`` sidecar (FAILED — so a failed attempt is counted and
-    diagnosed rather than silently dropped). Every record is attributed to its task by
-    ``_ng_task_index`` → ``index_to_task_id``, the map we stamped in :func:`_materialize_dataset`.
-    """
-    known_task_ids = {task.id for task in tasks}
-    # Defence in depth: the index map and the task list must describe the same run. Gym is an external
-    # project on its own release cadence, and people make agent-optimization decisions from these
-    # numbers — so any disagreement about *which task a result belongs to* is a hard error, never a
-    # best-effort join. Cheap to check once; impossible to notice downstream if it silently drifts.
-    mapped_task_ids = set(index_to_task_id.values())
-    if mapped_task_ids != known_task_ids:
-        raise ValueError(
-            f"the {NG_TASK_INDEX} map does not match the task list it will be joined against "
-            f"({len(mapped_task_ids - known_task_ids)} mapped task(s) absent from the list, "
-            f"{len(known_task_ids - mapped_task_ids)} listed task(s) absent from the map); "
-            "attributing rollouts across a mismatch would report results against the wrong tasks"
-        )
-    trials: list[AgentEvalTrial] = []
-    # Shared per-task counter for records lacking a usable _ng_rollout_index (success + failure); a
-    # missing/null index must not collapse multiple attempts for one task onto the same trial id.
-    synth_seq: dict[str, int] = {}
-
-    success_evidence = CandidateEvidence(
-        descriptors={"rollouts": EvidenceDescriptor(kind="filesystem", format="file", ref=str(rollouts_path))}
-    )
-    unattributed_successes = 0
-    empty_output_count = 0
-    #: Tasks with at least one trial where the agent never ran. Tracked separately from trial
-    #: status, since the failures sidecar also produces FAILED trials for unrelated reasons.
-    empty_output_task_ids: set[str] = set()
-    for record in _read_jsonl(rollouts_path):
-        task_id = _resolve_task_id(record, index_to_task_id)
-        if task_id is None:
-            # No usable index (an unknown one raises). Skipping is right, but silence is not: with
-            # num_repeats > 1 the task keeps its other trials, so _require_full_coverage won't fire and
-            # the task would simply be scored on fewer attempts than were actually run.
-            unattributed_successes += 1
-            continue
-        raw_rollout_index = record.get(NG_ROLLOUT_INDEX)
-        reward = _coerce_reward(record.get(reward_key))
-        # An agent that never ran did not earn its reward — reporting 0.0 as a completed trial would
-        # let a broken environment read as a poorly-performing one.
-        never_ran = _agent_never_ran(record)
-        if never_ran:
-            empty_output_count += 1
-            empty_output_task_ids.add(task_id)
-            status = AgentEvalTrialStatus.FAILED
-        elif reward is not None:
-            status = AgentEvalTrialStatus.COMPLETED
-        else:
-            status = AgentEvalTrialStatus.PARTIAL
-        metadata: dict[str, Any] = {
-            "reward": None if never_ran else reward,
-            NG_TASK_INDEX: record.get(NG_TASK_INDEX),
-            NG_ROLLOUT_INDEX: raw_rollout_index,
-        }
-        if never_ran:
-            # Kept alongside the reward Gym reported, so the discarded value is still auditable.
-            metadata["gym_failure"] = (
-                "the agent produced no output and consumed no tokens, so the model was never called; "
-                f"the reported score of {reward!r} is not a measurement of this agent"
-            )
-        trials.append(
-            AgentEvalTrial(
-                id=_rollout_trial_id(task_id, raw_rollout_index, synth_seq, missing_label="noidx"),
-                task_id=task_id,
-                status=status,
-                output=AgentOutput(response=record.get("response"), metadata={"agent_ref": record.get("agent_ref")}),
-                evidence=success_evidence,
-                metadata=metadata,
-            )
-        )
-
-    failures_path = _failures_path_for(rollouts_path)
-    unattributed_failures = 0
-    if failures_path.exists():
-        failure_evidence = CandidateEvidence(
-            descriptors={
-                "rollout_failures": EvidenceDescriptor(kind="filesystem", format="file", ref=str(failures_path))
-            }
-        )
-        # tolerant: a truncated line in the abnormal-termination sidecar must not sink the good trials.
-        for record in _read_jsonl(failures_path, tolerant=True):
-            # strict=False to match this loop's tolerant read: an odd record is counted, not fatal.
-            task_id = _resolve_task_id(record, index_to_task_id, strict=False)
-            if task_id is None:
-                # kill-shaped failures (SIGTERM/OOM/actor-death) can lack a usable index — count, don't drop.
-                unattributed_failures += 1
-                continue
-            raw_rollout_index = record.get(NG_ROLLOUT_INDEX)
-            trials.append(
-                AgentEvalTrial(
-                    id=_rollout_trial_id(task_id, raw_rollout_index, synth_seq, missing_label="fail"),
-                    task_id=task_id,
-                    status=AgentEvalTrialStatus.FAILED,
-                    output=AgentOutput(
-                        response=record.get("response"), metadata={"agent_ref": record.get("agent_ref")}
-                    ),
-                    evidence=failure_evidence,
-                    metadata={
-                        "reward": None,
-                        # best-effort: Gym's failure-record schema isn't contractual, so probe common keys.
-                        "gym_failure": record.get("error") or record.get("exception") or record.get("failure_reason"),
-                        NG_TASK_INDEX: record.get(NG_TASK_INDEX),
-                        NG_ROLLOUT_INDEX: raw_rollout_index,
-                    },
-                )
-            )
-    if empty_output_count:
-        # Every trial empty is a configuration failure, not a result: nothing was measured, so there
-        # is no score to report and letting the run "succeed" would publish zeros as findings. A
-        # partial count is left as failed trials — a flaky environment is still worth the attempts
-        # that did run.
-        #
-        # Measured against *every* trial, including those the failures sidecar contributed. A run
-        # that also recorded real failures is not the uniform "nothing ran" picture this raise is
-        # for: those trials carry their own diagnosis (a verifier timeout, say, from an attempt that
-        # did reach the model), and raising would discard it for a less specific message. Likewise
-        # `unattributed_successes` never became trials and so are invisible to `len(trials)`, but
-        # each is a result the environment produced and one may have run perfectly well — claiming
-        # nothing was measured while such a record exists would be false.
-        if empty_output_count == len(trials) and not unattributed_successes:
-            affected_tasks = {trial.task_id for trial in trials}
-            raise RuntimeError(
-                f"No agent ran: all {empty_output_count} trial(s) across all {len(affected_tasks)} "
-                "task(s) produced no output and consumed no tokens, so the model was never called "
-                "and nothing was measured. The scores reported for them describe an agent that did "
-                "not run.\n"
-                "This is normally the environment failing to start its agent rather than a bad "
-                f"model: check the per-server startup output in {rollouts_path.parent / _ENV_LOG_NAME} "
-                "for a traceback from the environment's own agent server.\n"
-                f"Results as collected: {rollouts_path}"
-            )
-        logger.warning(
-            "%d of %d trial(s), across %d of %d task(s), produced no agent output and consumed no "
-            "tokens; those are marked FAILED rather than scored, since the model was never called "
-            "for them.",
-            empty_output_count,
-            len(trials),
-            len(empty_output_task_ids),
-            len({trial.task_id for trial in trials}),
-        )
-    if unattributed_successes:
-        logger.warning(
-            "%d Gym rollout record(s) in %s carried no usable %s and could not be attributed to a task; "
-            "affected tasks are scored on fewer attempts than were collected.",
-            unattributed_successes,
-            rollouts_path,
-            NG_TASK_INDEX,
-        )
-    if unattributed_failures:
-        logger.warning(
-            "%d Gym failure record(s) lacked a usable %s and could not be attributed to a task.",
-            unattributed_failures,
-            NG_TASK_INDEX,
-        )
-
-    # Reported here, enforced by _require_full_coverage in run_tasks (which knows the run's log paths).
-    missing = known_task_ids - {trial.task_id for trial in trials}
-    if missing:
-        logger.warning("No Gym rollout produced a trial for %d requested task(s): %s", len(missing), sorted(missing))
-    return trials
-
-
-def _signal_group(pgid: int, sig: int) -> None:
-    """Send ``sig`` to a process group, tolerating an already-dead group."""
-    try:
-        os.killpg(pgid, sig)
-    except ProcessLookupError:
-        pass
-
-
-async def _terminate(proc: asyncio.subprocess.Process, *, grace_s: float = 30.0) -> None:
-    """Tear down a backgrounded Gym subprocess *and its whole process group*.
-
-    ``gym env start`` fans out into a Ray cluster + uvicorn servers, and Gym stops them from a
-    ``KeyboardInterrupt`` handler (``finally: self.shutdown()``) — so we send **SIGINT** (not SIGTERM,
-    which would bypass that handler and orphan Ray's detached ``gcs_server``). The child is spawned with
-    ``start_new_session=True`` and leads its own group, so SIGINT to the group mimics Ctrl-C exactly;
-    SIGKILL to the group is the escalation if Gym's graceful shutdown overstays ``grace_s``.
-
-    Known limitation: Ray daemonizes ``gcs_server`` into its *own* session, outside this group, so the
-    escalation SIGKILL cannot reach it — clean Ray teardown depends on Gym's SIGINT ``shutdown()``
-    completing within ``grace_s``. POSIX-only (``killpg`` / ``getpgid`` / ``start_new_session`` / SIGINT).
-    """
-    if proc.returncode is not None:
-        return
-    try:
-        pgid = os.getpgid(proc.pid)
-    except ProcessLookupError:
-        return
-    _signal_group(pgid, signal.SIGINT)
-    try:
-        await asyncio.wait_for(proc.wait(), timeout=grace_s)
-    except asyncio.TimeoutError:
-        _signal_group(pgid, signal.SIGKILL)
-        await proc.wait()
-
-
-__all__ = [
-    "DEFAULT_REWARD_KEY",
-    "GymAgentTaskRunner",
-    "GymRewardMetric",
-    "GymRuntimeConfig",
-    "discover_gym_tasks",
-]
diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_gym_aggregate_scores.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_gym_aggregate_scores.py
index cbbbf63ebe..c6b86463b5 100644
--- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_gym_aggregate_scores.py
+++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_gym_aggregate_scores.py
@@ -7,7 +7,7 @@
 
 from collections.abc import Mapping, Sequence
 
-from nemo_evaluator_sdk.agent_eval.runtimes.gym_runtime import _aggregate_scores_from_gym
+from nemo_evaluator_sdk.agent_eval.runtimes.gym.results import _aggregate_scores_from_gym
 from nemo_evaluator_sdk.values.results import AggregateRangeScore, AggregateScalarScore, AggregateScore
 
 
diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_gym_environment_coverage.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_gym_environment_coverage.py
index eadace04f4..054587b24d 100644
--- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_gym_environment_coverage.py
+++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_gym_environment_coverage.py
@@ -58,12 +58,8 @@
 
 import pytest
 from nemo_evaluator_sdk.agent_eval.evaluator import AgentEvaluator
-from nemo_evaluator_sdk.agent_eval.runtimes.gym_runtime import (
-    GymAgentTaskRunner,
-    GymRuntimeConfig,
-    _flatten_overrides,
-    discover_gym_tasks,
-)
+from nemo_evaluator_sdk.agent_eval.runtimes.gym import GymAgentTaskRunner, GymRuntimeConfig, discover_gym_tasks
+from nemo_evaluator_sdk.agent_eval.runtimes.gym.config import _flatten_overrides
 from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask
 from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrialStatus
 
diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_gym_runtime.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_gym_runtime.py
index e777462e4d..b56ee9835a 100644
--- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_gym_runtime.py
+++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_gym_runtime.py
@@ -19,32 +19,37 @@
 from pathlib import Path
 
 import pytest
-from nemo_evaluator_sdk.agent_eval.runtimes.gym_runtime import (
-    _LOG_TAIL_LINES,
-    NG_ROLLOUT_INDEX,
-    NG_TASK_INDEX,
+from nemo_evaluator_sdk.agent_eval.runtimes.gym import (
     GymAgentTaskRunner,
     GymRewardMetric,
     GymRuntimeConfig,
-    _agent_never_ran,
-    _aggregate_metrics_path_for,
+    discover_gym_tasks,
+)
+from nemo_evaluator_sdk.agent_eval.runtimes.gym.config import _flatten_overrides, _hydra_scalar, _selection_args
+from nemo_evaluator_sdk.agent_eval.runtimes.gym.dataset import (
     _canonical_row_hash,
     _content_text,
+    _materialize_dataset,
+    _render_instruction,
+    _source_datasets,
+)
+from nemo_evaluator_sdk.agent_eval.runtimes.gym.process import (
+    _LOG_TAIL_LINES,
     _drain_pumps,
-    _ensure_fresh_output,
-    _flatten_overrides,
     _gym_executable,
-    _hydra_scalar,
-    _materialize_dataset,
     _pending_servers,
     _pump_stream,
+)
+from nemo_evaluator_sdk.agent_eval.runtimes.gym.records import NG_ROLLOUT_INDEX, NG_TASK_INDEX
+from nemo_evaluator_sdk.agent_eval.runtimes.gym.results import (
+    _INPUT_TOKEN_KEYS,
+    _TOKEN_USAGE_KEYS,
+    _agent_never_ran,
+    _aggregate_metrics_path_for,
+    _ensure_fresh_output,
     _read_run_aggregations,
-    _render_instruction,
     _require_full_coverage,
-    _selection_args,
-    _source_datasets,
     _trials_from_rollouts,
-    discover_gym_tasks,
 )
 from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrialStatus
 from nemo_evaluator_sdk.metrics.protocol import CandidateOutput, DatasetRow, MetricInput
@@ -492,7 +497,7 @@ async def test_pump_stream_writes_file_and_mirrors_to_logger(tmp_path: Path, cap
     stream.feed_eof()
     log_path = tmp_path / "gym_eval.stdout.log"
     tails: dict[str, deque] = {}
-    with caplog.at_level(logging.DEBUG, logger="nemo_evaluator_sdk.agent_eval.runtimes.gym_runtime"):
+    with caplog.at_level(logging.DEBUG, logger="nemo_evaluator_sdk.agent_eval.runtimes.gym.process"):
         await _pump_stream(stream, log_path, label="gym eval", tails=tails, key="stdout")
     assert log_path.read_text(encoding="utf-8") == "first line\nsecond line\nno trailing newline"
     assert list(tails["stdout"]) == ["first line", "second line", "no trailing newline"]
@@ -774,11 +779,31 @@ def test_selection_omits_the_binding_when_the_caller_binds_it_themselves(tmp_pat
     assert "++simple_agent.responses_api_agents.simple_agent.resources_server.name='other'" in selection
 
 
+@pytest.mark.parametrize(
+    "awkward",
+    [
+        pytest.param("run with spaces", id="spaces"),
+        pytest.param("run,with,commas", id="commas"),
+        pytest.param("run[1]", id="brackets"),
+    ],
+)
+def test_selection_quotes_a_work_dir_containing_hydra_grammar(tmp_path: Path, awkward: str) -> None:
+    # Raised by review on #1295. `hydra.run.dir` is a path, and Hydra's grammar reads `,` as a sweep
+    # separator and `[` as a list opener. Checked against Hydra's own parser: unquoted, `/tmp/a,b`
+    # comes back as a ChoiceSweep and `/tmp/x[1]` raises OverrideParseException. Quoting round-trips
+    # all three, so the value has to go through _hydra_scalar like every other override.
+    work_dir = tmp_path / awkward
+    arg = next(a for a in _selection_args(_config(), work_dir) if a.startswith("hydra.run.dir="))
+    value = arg.removeprefix("hydra.run.dir=")
+    assert value == _hydra_scalar(str(work_dir / "gym_hydra"))
+    assert value.startswith("'") and value.endswith("'")
+
+
 def test_selection_redirects_hydra_output_under_the_run_work_dir(tmp_path: Path) -> None:
     # Gym writes `outputs/<date>/<time>/` relative to cwd, and the subprocesses inherit ours so Gym
     # can find env.yaml — so without this every run litters the caller's directory.
     selection = _selection_args(_config(), tmp_path)
-    assert f"hydra.run.dir={tmp_path / 'gym_hydra'}" in selection
+    assert f"hydra.run.dir={_hydra_scalar(str(tmp_path / 'gym_hydra'))}" in selection
 
 
 def _stub_gym(tmp_path: Path, *, exit_code: int, message: str) -> str:
@@ -873,6 +898,35 @@ def _two_tasks_and_map(tmp_path: Path):
     return tasks, _materialize_dataset(tasks, tmp_path / "gym_input.jsonl")
 
 
+def test_token_usage_keys_match_the_openai_schemas_they_mirror() -> None:
+    """The keys are *wire-format* field names, cross-checked here against their source of truth.
+
+    Raised in review of #1295: could these come from Gym directly? No — this runtime never imports
+    `nemo_gym` (it shells out to the CLI, and nemo-platform excludes Ray by constraint), and the
+    names are not Gym's anyway. They are OpenAI's, and `openai` *is* a dependency here.
+
+    Deliberately a test rather than deriving the tuple at import time. What we match is the JSON a
+    Gym rollout record carries, which is stable regardless of how the `openai` package organises its
+    Python classes — importing `openai.types.responses.response_usage` into the runtime to obtain
+    five string literals would trade a stable thing for a fragile one, and break the whole runner if
+    that module ever moves. This way a genuinely new token field fails one test instead.
+    """
+    from openai.types.completion_usage import CompletionUsage
+    from openai.types.responses.response_usage import ResponseUsage
+
+    upstream = {field for field in CompletionUsage.model_fields if field.endswith("_tokens")} | {
+        field for field in ResponseUsage.model_fields if field.endswith("_tokens")
+    }
+    assert set(_TOKEN_USAGE_KEYS) == upstream, (
+        "the token-count keys have drifted from the OpenAI usage schemas they mirror; a field added "
+        "upstream means a real model call could be reported in a key this runtime does not read"
+    )
+    # The input-side subset is a semantic judgement neither schema encodes: these are the counts a
+    # call cannot avoid spending, because it always sends a prompt.
+    assert set(_INPUT_TOKEN_KEYS) < set(_TOKEN_USAGE_KEYS)
+    assert set(_INPUT_TOKEN_KEYS) == {"total_tokens", "input_tokens", "prompt_tokens"}
+
+
 def test_agent_never_ran_detects_the_empty_zero_token_rollout() -> None:
     assert _agent_never_ran({"response": _EMPTY_RESPONSE, "reward": 0.0}) is True
 
@@ -934,6 +988,24 @@ def test_agent_never_ran_is_false_without_usage_to_corroborate() -> None:
     assert _agent_never_ran({"response": "Lyon", "reward": 0.0}) is False
 
 
+def test_missing_results_file_names_the_logs_instead_of_raising_filenotfound(tmp_path: Path) -> None:
+    # Raised by review on #1295. `_ensure_fresh_output` deletes any stale file before the run and
+    # `_collect_rollouts` already raised on a non-zero exit, so an absent file here means Gym
+    # reported success and wrote nothing. Opening it regardless gives a bare FileNotFoundError
+    # naming a path the reader has no reason to recognise.
+    tasks, index_map = _two_tasks_and_map(tmp_path)
+    missing = tmp_path / "rollouts.jsonl"
+    assert not missing.exists()
+
+    with pytest.raises(RuntimeError) as excinfo:
+        _trials_from_rollouts(missing, tasks, index_map)
+
+    message = str(excinfo.value)
+    assert "wrote no results" in message
+    assert "gym_eval.stdout.log" in message
+    assert "gym_env.log" in message
+
+
 def test_all_rollouts_empty_fails_the_run_instead_of_reporting_zeros(tmp_path: Path) -> None:
     # The legal_agent_bench case: "completed" trials scoring 0.0 that measured nothing at all.
     # Two attempts per task, so trial and task counts differ and the message has to state both —
@@ -968,6 +1040,27 @@ def test_all_rollouts_empty_fails_the_run_instead_of_reporting_zeros(tmp_path: P
     assert "rollout" not in prose.lower()
 
 
+def test_an_unattributed_failure_prevents_the_all_empty_raise(tmp_path: Path) -> None:
+    # Raised in review of #1295: the guard covered unattributed *successes* but not unattributed
+    # *failures*. Both are invisible to len(trials) for the same reason — neither becomes a trial —
+    # and a failure record is a diagnosis in its own right (an OOM-killed agent here). Claiming
+    # "nothing was measured" while the environment recorded one is false.
+    tasks, index_map = _two_tasks_and_map(tmp_path)
+    bundle = tmp_path / "rollouts.jsonl"
+    bundle.write_text(
+        json.dumps({NG_TASK_INDEX: 0, NG_ROLLOUT_INDEX: 0, "response": _EMPTY_RESPONSE, "reward": 0.0}) + "\n",
+        encoding="utf-8",
+    )
+    # No NG_TASK_INDEX, so it is counted but never attributed to a task.
+    (tmp_path / "rollouts_failures.jsonl").write_text(
+        json.dumps({"error": "agent OOM-killed mid-episode"}) + "\n", encoding="utf-8"
+    )
+
+    trials = _trials_from_rollouts(bundle, tasks, index_map)
+
+    assert [trial.status for trial in trials] == [AgentEvalTrialStatus.FAILED]
+
+
 def test_an_unattributed_rollout_prevents_the_all_empty_raise(tmp_path: Path) -> None:
     # A record Gym collected but that carried no usable _ng_task_index never becomes a trial, so it
     # is invisible to len(trials). It is still a rollout, and it may have run perfectly well —
@@ -1115,4 +1208,4 @@ async def test_collection_redirects_hydra_output_under_the_run_work_dir(tmp_path
     await runner._collect_rollouts(gym, tmp_path / "in.jsonl", tmp_path / "out.jsonl", tmp_path, {})
 
     argv = (tmp_path / "argv.txt").read_text().splitlines()
-    assert f"hydra.run.dir={tmp_path / 'gym_hydra'}" in argv
+    assert f"hydra.run.dir={_hydra_scalar(str(tmp_path / 'gym_hydra'))}" in argv
diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_run_metadata.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_run_metadata.py
index 496d85587d..52c2ab14c9 100644
--- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_run_metadata.py
+++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_run_metadata.py
@@ -59,7 +59,7 @@ def test_every_shipped_runner_reports_a_stable_name_and_result_shaping_config()
     from nemo_evaluator_sdk.agent_eval.runtimes.docker_sandbox import DockerSandboxAgentRuntime
     from nemo_evaluator_sdk.agent_eval.runtimes.fabric.container_runtime import FabricContainerRuntime
     from nemo_evaluator_sdk.agent_eval.runtimes.fabric.runtime import FabricAgentRuntime
-    from nemo_evaluator_sdk.agent_eval.runtimes.gym_runtime import GymAgentTaskRunner, GymRuntimeConfig
+    from nemo_evaluator_sdk.agent_eval.runtimes.gym import GymAgentTaskRunner, GymRuntimeConfig
     from nemo_evaluator_sdk.agent_eval.runtimes.harbor_runtime import HarborAgentTaskRunner, HarborRuntimeConfig
 
     async def _agent_fn(task):  # pragma: no cover - never called
@@ -189,7 +189,7 @@ def _info(**kwargs):
 def test_gym_redacts_credential_looking_env_overrides() -> None:
     # env_overrides is a free-form Hydra escape hatch forwarded to `gym env start`, and RunnerInfo.config
     # is persisted into the run bundle — so a value that looks like a credential must not be written there.
-    from nemo_evaluator_sdk.agent_eval.runtimes.gym_runtime import GymAgentTaskRunner, GymRuntimeConfig
+    from nemo_evaluator_sdk.agent_eval.runtimes.gym import GymAgentTaskRunner, GymRuntimeConfig
 
     runner = GymAgentTaskRunner(
         config=GymRuntimeConfig(
diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py
index 047a9eb950..6562d52dbd 100644
--- a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py
+++ b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py
@@ -47,7 +47,7 @@
 from nemo_evaluator_sdk.agent_eval.results import AgentEvalResult
 from nemo_evaluator_sdk.agent_eval.runtimes.codex.runtime import CodexCliAgentRuntime
 from nemo_evaluator_sdk.agent_eval.runtimes.fabric.runtime import FabricAgentRuntime
-from nemo_evaluator_sdk.agent_eval.runtimes.gym_runtime import GymAgentTaskRunner, GymRuntimeConfig
+from nemo_evaluator_sdk.agent_eval.runtimes.gym import GymAgentTaskRunner, GymRuntimeConfig
 from nemo_evaluator_sdk.agent_eval.runtimes.harbor_runtime import HarborAgentTaskRunner, HarborRuntimeConfig
 from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask
 from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTarget
diff --git a/plugins/nemo-evaluator/tests/test_agent_evaluate.py b/plugins/nemo-evaluator/tests/test_agent_evaluate.py
index 248aef70ff..5fcb2101d7 100644
--- a/plugins/nemo-evaluator/tests/test_agent_evaluate.py
+++ b/plugins/nemo-evaluator/tests/test_agent_evaluate.py
@@ -41,7 +41,7 @@
 from nemo_evaluator_sdk.agent_eval.results import AgentEvalResult, AgentEvalSummary
 from nemo_evaluator_sdk.agent_eval.runtimes.codex.runtime import CodexCliAgentRuntime
 from nemo_evaluator_sdk.agent_eval.runtimes.fabric.runtime import FabricAgentRuntime
-from nemo_evaluator_sdk.agent_eval.runtimes.gym_runtime import GymAgentTaskRunner
+from nemo_evaluator_sdk.agent_eval.runtimes.gym import GymAgentTaskRunner
 from nemo_evaluator_sdk.agent_eval.runtimes.harbor_runtime import HarborAgentTaskRunner
 from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask
 from nemo_evaluator_sdk.agent_eval.trials import (
diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/__init__.py
new file mode 100644
index 0000000000..3ff68a9f8a
--- /dev/null
+++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/__init__.py
@@ -0,0 +1,84 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""NeMo Gym-backed :class:`AgentTaskRunner` for the agent-eval pipeline.
+
+Runs an *existing* NeMo Gym environment through its ``gym`` CLI and adapts the
+rollout bundle into SDK :class:`AgentEvalTrial` objects, so an
+:class:`AgentEvaluator` can score and report Gym runs through the same seam as
+Harbor/Fabric/Model. Gym owns execution *and* scoring; this runtime imports an
+existing Gym environment as-is — Gym is treated as a self-scoring engine and the
+runtime only adapts its rollout bundle, it does not re-derive rewards.
+
+**Mapping** (Gym → Evaluator): one Gym dataset → one run; each distinct row →
+one :class:`AgentEvalTask` (id = content hash of the row); each attempt
+(``_ng_rollout_index``) → one :class:`AgentEvalTrial`; the per-attempt verifier
+``reward`` → a :class:`GymRewardMetric` score. ``num_repeats=R`` therefore yields
+up to R trials per task. Row duplication is *not* a way to ask for repeated
+attempts — ``num_repeats`` is (see :func:`discover_gym_tasks`).
+
+**Attribution** is by ``_ng_task_index``, which this runtime *assigns* rather
+than infers. Gym only auto-assigns an index when a row doesn't already carry one
+(``rollout_collection._preprocess_rows_from_config``), and its own fallback
+dedup keys off the **raw jsonl line text** — a rule we cannot reproduce from
+parsed rows. So instead of guessing, :meth:`GymAgentTaskRunner.run_tasks`
+materializes a normalized dataset (one line per requested task, ``_ng_task_index``
+stamped explicitly) and feeds *that* to Gym. Gym echoes the index back on every
+rollout record, giving a total, order-independent ``index → task`` map. This also
+means a caller can run a **subset** of tasks without Gym rolling out the rest.
+
+**Execution** is the two-step Gym flow (the one that reads a dataset directly
+without triggering Gym's split-driven data-prep), preceded by a pre-flight:
+``gym env validate`` merges the composed config and reports unset ``???`` values,
+bad paths, and dangling cross-references without starting anything; then ``gym env
+start`` brings up the resources-server + agent + model servers, and ``gym eval run
+--no-serve --input <materialized dataset>`` collects rollouts against them. Both
+commands receive the identical selection arguments, so what is validated is what
+runs. The runtime shells out to the ``gym`` CLI on PATH, so this SDK never imports
+``nemo_gym``. Subprocess
+output is streamed to log files under the run's work dir *and* mirrored to this
+module's logger at ``DEBUG``, so callers choose terminal visibility through
+ordinary ``logging`` configuration.
+
+**Where Gym finds things.** NeMo Gym must be installed and its ``gym`` on PATH,
+along with the target environment's own dependencies. Generally that means a
+*separate* environment: Gym imports Ray at module load, and nemo-platform
+excludes Ray by constraint over an unfixed CVE, so the two cannot share one. In a
+job image the image owns PATH and this is unremarkable. There is deliberately no
+config field naming a checkout, a venv, or a search root — these runner configs
+become serialized job specs, and a local filesystem path means nothing on the
+other side of that boundary. Environments themselves ship in the ``nemo-gym``
+wheel (``resources_servers`` and friends install beside ``nemo_gym``, configs and
+example data included), so no checkout is needed to reach them.
+
+The subprocesses inherit this process's working directory, which is where Gym
+looks for the gitignored ``env.yaml`` holding the collector's credentials before
+falling back to its install root — so credentials never pass through this SDK.
+Run from the directory holding that file; a Gym checkout there also has its
+components take precedence, which is how you reach an environment the wheel does
+not carry.
+
+**Boundaries**: the caller is responsible for a
+Gym runtime whose deps are installed (each Gym env ships its own
+``requirements.txt``), and for handing a *ready-to-run* dataset file (``--no-serve
+--input`` bypasses Gym's prompt-templating/materialization). Service-side
+provisioning (docker/k8s, Ray) is out of scope here — that is the plugin's job.
+
+A consequence of that bypass: an environment whose rows carry no rendered prompt
+(``responses_create_params.input == []``, the prompt supplied by data-prep or by the
+environment's own agent) is still supported — the row travels through this runtime intact
+and the task simply has no ``inputs['instruction']``. See :func:`discover_gym_tasks`.
+"""
+
+from nemo_platform.beta.evaluator.agent_eval.runtimes.gym.config import DEFAULT_REWARD_KEY, GymRuntimeConfig
+from nemo_platform.beta.evaluator.agent_eval.runtimes.gym.dataset import discover_gym_tasks
+from nemo_platform.beta.evaluator.agent_eval.runtimes.gym.results import GymRewardMetric
+from nemo_platform.beta.evaluator.agent_eval.runtimes.gym.runtime import GymAgentTaskRunner
+
+__all__ = [
+    "DEFAULT_REWARD_KEY",
+    "GymAgentTaskRunner",
+    "GymRewardMetric",
+    "GymRuntimeConfig",
+    "discover_gym_tasks",
+]
diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/config.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/config.py
new file mode 100644
index 0000000000..21a3dabecd
--- /dev/null
+++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/config.py
@@ -0,0 +1,260 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Run configuration, and the Hydra grammar it is serialized into.
+
+Gym is a Hydra application, so every setting reaches it as an ``++dotted.path=value`` argument.
+That grammar is typed and unforgiving — quoting rules live here alongside the config model they
+serialize, so the two cannot drift. Redaction lives here too: these values are recorded as run
+provenance, and the override map is a free-form escape hatch a caller can put a credential into.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from collections.abc import Mapping
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+logger = logging.getLogger(__name__)
+
+
+DEFAULT_REWARD_KEY = "reward"
+#: Gym's CLI, expected on PATH. Not configurable: these runner configs become serialized job specs,
+#: and a path into somebody's venv is meaningless on the other side of that boundary. Note this name
+#: is only ever *resolved*, never executed: :func:`_gym_executable` turns it into an absolute path
+#: once, and that path is what the subprocesses run — so a child whose PATH differs from ours cannot
+#: end up executing a different Gym.
+_HYDRA_SUBDIR = "gym_hydra"
+#: `gym env start`'s combined output, under the run's work dir. Named here because a *collection*
+#: failure often has to point at it: the eval logs show the symptom, this shows the cause.
+_SECRET_KEY_MARKERS = ("api_key", "apikey", "token", "secret", "password", "passwd", "credential")
+#: Stand-in written in place of a redacted override value.
+_REDACTED = "<redacted>"
+
+
+#: Dict keys Hydra reads back unchanged. Its ``dictKey`` rule accepts no quoting, so a key is
+#: whatever the lexer makes of the bare text — this is deliberately narrower than what parses.
+_HYDRA_DICT_KEY = re.compile(r"[A-Za-z_][A-Za-z0-9_.-]*\Z")
+#: Bare words the lexer types rather than reading as text, so they cannot serve as string keys.
+_HYDRA_KEY_LITERALS = frozenset({"true", "false", "null", "inf", "nan"})
+
+
+def _hydra_dict(value: Mapping[str, Any]) -> str:
+    """Render a mapping as a Hydra dict container, ``{key:value,...}``.
+
+    Reached for a mapping nested inside a container — ``[{"b": 1}]`` — where there is no dotted path
+    to flatten onto, so the dict has to be spelled inline. Values recurse, so the typed spellings
+    below hold at any depth.
+
+    Keys are emitted bare, because Hydra's ``dictKey`` rule has no quoted form: ``{'b':1}`` does not
+    parse at all. That leaves the key at the mercy of the lexer, which types it — ``{true:1}`` keys
+    on the boolean ``True``, ``{1.5:1}`` on a float — and rejects ``:``, ``,``, brackets, and quotes
+    outright. Anything outside the conservative shape above therefore raises here rather than
+    silently keying the config on something the caller did not write.
+    """
+    rendered = []
+    for key, item in value.items():
+        if not isinstance(key, str) or not _HYDRA_DICT_KEY.match(key) or key.casefold() in _HYDRA_KEY_LITERALS:
+            raise ValueError(
+                f"Gym config override has dict key {key!r}, which Hydra's override grammar cannot "
+                "express as a string: keys are unquoted, so only a leading letter or underscore "
+                "followed by letters, digits, '_', '.', or '-' survives the round trip. Set this "
+                "key through the override path instead of nesting it inside a list."
+            )
+        rendered.append(f"{key}:{_hydra_scalar(item)}")
+    return "{" + ",".join(rendered) + "}"
+
+
+def _hydra_scalar(value: Any) -> str:
+    """Render a leaf value the way Hydra's override grammar reads it back.
+
+    Hydra's grammar is typed, so an unquoted string is not necessarily a string: ``true`` parses as a
+    boolean, ``null`` as ``None``, ``1.5`` as a float, ``a,b`` as a *sweep*, and ``A[B`` fails to
+    parse outright. Strings are therefore always single-quoted, which round-trips every one of those
+    (verified against ``hydra.core.override_parser``). Interpolations survive quoting — the override
+    sets the literal text and OmegaConf resolves it on read — so ``${policy_base_url}`` still works.
+
+    Only ``'`` is escaped. Hydra does **not** decode ``\\\\`` inside a quoted value: escaping
+    backslashes doubles them, so they are passed through raw.
+
+    ``None`` and booleans get their own spellings, since ``str()`` would emit ``"None"``/``"True"``
+    and Hydra reads those back as text. Containers recurse for the same reason: ``str()`` on a dict
+    emits Python's repr, whose quoted keys Hydra rejects outright.
+    """
+    if value is None:
+        return "null"
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    if isinstance(value, Mapping):
+        return _hydra_dict(value)
+    if isinstance(value, (list, tuple)):
+        return "[" + ",".join(_hydra_scalar(item) for item in value) + "]"
+    if isinstance(value, str):
+        # A trailing backslash would escape the closing quote and leave the value unterminated, and
+        # there is no spelling that avoids it — better to say so than to emit something unparseable.
+        if value.endswith("\\"):
+            raise ValueError(
+                f"Gym config override value {value!r} ends with a backslash, which Hydra's override "
+                "grammar cannot express: it escapes the closing quote."
+            )
+        return "'" + value.replace("'", "\\'") + "'"
+    return str(value)
+
+
+def _flatten_overrides(overrides: Mapping[str, Any], _prefix: str = "") -> list[str]:
+    """Flatten a nested override mapping into Hydra ``++dotted.path=value`` arguments.
+
+    Callers describe overrides as structured data — ``{"a": {"b": 1}}`` — rather than as
+    pre-serialized Hydra strings, so the config survives being sent somewhere as JSON. Hydra itself
+    only speaks the flat form, so the translation happens here, at the point of invocation.
+
+    ``++`` rather than ``+``: it sets a key whether or not it already exists, which is what an
+    override means. A bare ``+`` fails on a key the merged config already defines.
+    """
+    arguments: list[str] = []
+    for key, value in overrides.items():
+        path = f"{_prefix}{key}"
+        # An empty mapping has no leaves to descend to, so recursing would drop the override
+        # entirely. It is still a value the caller asked to set: emit it as ``++path={}``, which
+        # clears the subtree.
+        if isinstance(value, Mapping) and value:
+            arguments.extend(_flatten_overrides(value, f"{path}."))
+        else:
+            arguments.append(f"++{path}={_hydra_scalar(value)}")
+    return arguments
+
+
+def _redact_env_overrides(overrides: Mapping[str, Any], _prefix: str = "") -> dict[str, Any]:
+    """Redact credential-looking values from overrides before they are recorded as provenance.
+
+    ``env_overrides`` is a free-form escape hatch forwarded to Gym, so nothing stops a caller passing
+    ``{"model": {"api_key": "sk-..."}}``. ``RunnerInfo.config`` is persisted into the run bundle, so a
+    value that looks like a credential must not be written there.
+
+    The *key* is always kept — knowing that a run overrode ``model.api_key`` is useful provenance;
+    knowing the value is a leak. Matching is on the full dotted path, so a marker anywhere in it
+    redacts, and nesting cannot hide a credential behind an innocuous leaf name.
+
+    Lists are walked too, since a mapping inside one — ``{"models": [{"api_key": "sk-..."}]}`` —
+    reaches Gym just as a nested mapping does. The index contributes no path segment: what marks a
+    value as a credential is the key it sits under, not where in a list it happens to fall.
+    """
+    redacted: dict[str, Any] = {}
+    for key, value in overrides.items():
+        path = f"{_prefix}{key}"
+        if isinstance(value, Mapping):
+            redacted[key] = _redact_env_overrides(value, f"{path}.")
+        elif any(marker in path.casefold() for marker in _SECRET_KEY_MARKERS):
+            redacted[key] = _REDACTED
+        elif isinstance(value, (list, tuple)):
+            redacted[key] = [_redact_list_item(item, path) for item in value]
+        else:
+            redacted[key] = value
+    return redacted
+
+
+def _redact_list_item(item: Any, path: str) -> Any:
+    """Redact inside one element of a list-valued override. See :func:`_redact_env_overrides`."""
+    if isinstance(item, Mapping):
+        return _redact_env_overrides(item, f"{path}.")
+    if isinstance(item, (list, tuple)):
+        return [_redact_list_item(nested, path) for nested in item]
+    return item
+
+
+def _selection_args(config: GymRuntimeConfig, work_dir: Path) -> list[str]:
+    """The environment/agent/model selection passed to Gym.
+
+    Built once and handed verbatim to both ``gym env validate`` and ``gym env start``, so what is
+    validated is exactly what runs — a pre-flight against a different config would be worse than
+    none.
+    """
+    selection = [
+        "--config",
+        config.agent_config,
+        "--model-type",
+        config.model_type,
+        "--resources-server",
+        config.resources_server,
+    ]
+    if config.bind_resources_server:
+        # Composable (Pattern-A) agents leave resources_server.name unbound ('???'); bind it to the
+        # env we're running. Assumes the agent config's top-level key equals the agent name (the
+        # simple_agent convention) *and* that the resources-server is registered under the
+        # environment's own name — not universally true, so self-contained or differently-named
+        # servers set bind_resources_server=False and bind themselves via env_overrides.
+        selection.append(
+            f"+{config.agent}.responses_api_agents.{config.agent}.resources_server.name={config.resources_server}"
+        )
+    selection.extend(_flatten_overrides(config.env_overrides))
+    # Gym is a Hydra app, so each invocation writes a timestamped run directory — by default
+    # `outputs/<date>/<time>/` under the *current* directory. Since the subprocesses inherit this
+    # process's cwd (so Gym can find env.yaml), the default would litter whatever directory the
+    # caller happened to run from. Redirect it under the run's work dir, with the rest of the run's
+    # artifacts. Applies to every Gym entry point, `gym list` included.
+    #
+    # Quoted like any other override value: this one is a *path*, and Hydra's grammar reads `,` as a
+    # sweep separator and `[` as a list opener, so an unquoted work dir containing either is
+    # silently misread or rejected outright. Verified against Hydra's own parser — unquoted,
+    # `/tmp/a,b` becomes a ChoiceSweep and `/tmp/x[1]` raises OverrideParseException.
+    selection.append(f"hydra.run.dir={_hydra_scalar(str(work_dir / _HYDRA_SUBDIR))}")
+    return selection
+
+
+class GymRuntimeConfig(BaseModel):
+    """Declarative config for running an existing Gym environment via the ``gym`` CLI.
+
+    Holds only plain fields; the two-step invocation is built from them at run
+    time. The dataset itself is recovered from the tasks (stamped by
+    :func:`discover_gym_tasks`), mirroring the Harbor runner.
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    agent: str = Field(description="Agent name to collect rollouts with, e.g. 'simple_agent'.")
+    agent_config: str = Field(description="Repo-relative agent config passed to `gym env start` (--config).")
+    resources_server: str = Field(description="Resources-server (environment) name, e.g. 'mcqa' (--resources-server).")
+    model_type: str = Field(
+        default="inference_provider",
+        description="Model-type config (--model-type). `inference_provider` speaks OpenAI-compatible chat; "
+        "`openai_model` uses the OpenAI Responses API (500s against chat-only endpoints).",
+    )
+    bind_resources_server: bool = Field(
+        default=True,
+        description="Auto-bind the agent's `resources_server.name` to `resources_server` via a Hydra override "
+        "(the composable/Pattern-A agent case, e.g. simple_agent whose config leaves it '???'). Set False for "
+        "self-contained agents that already bind their own resources-server.",
+    )
+    env_overrides: dict[str, Any] = Field(
+        default_factory=dict,
+        description="Nested config overrides merged into Gym's config, applied after the auto-derived "
+        "resources-server binding. Structured rather than pre-serialized Hydra strings so the config "
+        "travels as JSON — `{'a': {'b': 1}}` becomes `++a.b=1` at invocation. This is the escape "
+        "hatch for what Gym does not standardize: an environment whose resources-server is registered "
+        "under a different name, or which references a model server no shipped config defines.",
+    )
+    num_repeats: int = Field(default=1, ge=1, description="Attempts per row; each attempt becomes one trial.")
+    concurrency: int = Field(
+        default=4,
+        ge=1,
+        description="Concurrent rollouts for `gym eval run` (the collection-phase knob, tuned to the model "
+        "endpoint's limits). Distinct from AgentEvalRunConfig.parallelism, which bounds concurrent scoring.",
+    )
+    startup_timeout_s: float = Field(default=240.0, gt=0, description="Max wait for `gym env start` readiness.")
+    collection_timeout_s: float | None = Field(
+        default=None,
+        gt=0,
+        description="Max wait for `gym eval run` collection; None = unbounded (scales with dataset x num_repeats x "
+        "model latency, so no safe fixed default). Set it to bound a hung/slow model endpoint.",
+    )
+    shutdown_grace_s: float = Field(
+        default=30.0,
+        gt=0,
+        description="Grace period for a Gym subprocess *group* to exit on SIGTERM (letting Ray shut down cleanly) "
+        "before escalating to SIGKILL.",
+    )
+    reward_key: str = Field(default=DEFAULT_REWARD_KEY, description="Key read from each rollout record.")
diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/dataset.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/dataset.py
new file mode 100644
index 0000000000..4cf0f3524c
--- /dev/null
+++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/dataset.py
@@ -0,0 +1,249 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Gym rows to Evaluator tasks, and back to a dataset Gym will collect against.
+
+Task identity is the content hash of a row, and attribution is an index this module *assigns*
+rather than infers — see :func:`_materialize_dataset`. Both halves live together because they are
+two directions of one translation: what breaks one silently breaks the other.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+from collections.abc import Mapping, Sequence
+from pathlib import Path
+from typing import Any
+
+from nemo_platform.beta.evaluator.agent_eval.runtimes.gym.records import _RUNTIME_KEYS, NG_TASK_INDEX, _read_jsonl
+from nemo_platform.beta.evaluator.agent_eval.runtimes.gym.results import GymRewardMetric
+from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalTask
+
+logger = logging.getLogger(__name__)
+
+
+def _canonical_row_hash(row: Mapping[str, Any]) -> str:
+    """Stable ``sha256`` of a Gym dataset row, excluding runtime-injected fields.
+
+    A given row keeps its id across dataset revisions/reorderings (so Intake
+    rollups stay consistent), and a changed row becomes a new task. No
+    dataset-revision component — identity is the row content alone.
+    """
+    payload = {key: value for key, value in row.items() if key not in _RUNTIME_KEYS}
+    encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str)
+    return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
+
+
+def _content_text(content: Any) -> str:
+    """Flatten a message ``content`` (a string, or a list of ``{type,text}`` parts) to text.
+
+    Anything else contributes nothing. A missing ``content`` is unremarkable, but a *populated* value
+    in some other shape means we're silently dropping prompt text, so warn — :func:`_render_instruction`
+    only fails loudly when the whole row renders empty, which a single skipped message wouldn't trigger.
+    """
+    if isinstance(content, str):
+        return content
+    # `str` is itself a Sequence, so this branch only ever sees non-str sequences (handled above).
+    if isinstance(content, Sequence):
+        parts = [part["text"] for part in content if isinstance(part, Mapping) and isinstance(part.get("text"), str)]
+        return "\n".join(parts)
+    if content is not None:
+        logger.warning(
+            "Ignoring message content of unsupported type %s (expected a string or a list of {type,text} parts); "
+            "any prompt text it carries will be missing from the task instruction.",
+            type(content).__name__,
+        )
+    return ""
+
+
+def _render_instruction(responses_create_params: Mapping[str, Any]) -> str:
+    """Derive a task instruction from a row's ``responses_create_params``, or ``""`` when it carries none.
+
+    Uses ``instructions`` (system prompt, when present) + ``input`` (a plain string
+    or an OpenAI message list).
+
+    An empty result is *not* an error. A whole class of Gym environments ships
+    ``"responses_create_params": {"input": []}`` and keeps the task elsewhere — ``gdpval``
+    in a top-level ``prompt``, ``legal_agent_bench`` in an ``instance_id`` its own agent
+    resolves, ``aviary``/``toolsandbox`` in nothing but a ``task_idx``. Their prompt is
+    materialized by Gym's data-preparation step or by the environment's agent, neither of
+    which this runner drives (``--no-serve --input`` hands Gym a ready-to-run dataset by
+    design). Nothing here can reconstruct it, and no single row field generalizes across
+    them, so the honest answer is no instruction rather than a guess. See
+    :func:`discover_gym_tasks` for what that means downstream.
+    """
+    parts: list[str] = []
+    instructions = responses_create_params.get("instructions")
+    if isinstance(instructions, str) and instructions.strip():
+        parts.append(instructions.strip())
+    raw_input = responses_create_params.get("input")
+    if isinstance(raw_input, str):
+        parts.append(raw_input)
+    elif isinstance(raw_input, Sequence):
+        for item in raw_input:
+            if isinstance(item, Mapping):
+                text = _content_text(item.get("content"))
+                if text:
+                    parts.append(text)
+    return "\n\n".join(part for part in parts if part).strip()
+
+
+def discover_gym_tasks(dataset: str | Path, *, metrics: Sequence[Any] | None = None) -> list[AgentEvalTask]:
+    """Build one :class:`AgentEvalTask` per distinct row in a Gym dataset (jsonl).
+
+    Each task's id is the content hash of the row; its ``instruction`` is rendered
+    from ``responses_create_params.input`` *when the row carries one*; the raw params
+    are stashed under ``inputs['gym_row']`` for provenance; and it is scored by a
+    :class:`GymRewardMetric`. The dataset path is stamped on
+    ``metadata['gym_dataset_path']``, and every *other* row key (the verifier's
+    ground-truth fields, ``agent_ref``, and so on) on ``metadata['gym_row_extras']``.
+    Together with ``inputs['gym_row']`` those reconstruct the complete source row,
+    which :class:`GymAgentTaskRunner` re-materializes into the dataset it hands to Gym.
+    They are kept disjoint deliberately: the task record is persisted to the run
+    bundle's ``tasks.jsonl``, and ``responses_create_params`` is usually the bulk of a
+    row, so storing it in both places would write every dataset twice per run.
+
+    **One distinct row is one task.** Duplicate rows collapse (identity is row content,
+    so they are by definition the same task) and are reported as a warning: repeated
+    attempts are a run-level concern — ``GymRuntimeConfig.num_repeats`` — not something
+    a dataset expresses by repeating a row, so duplicates almost always mean bad data.
+
+    **A row need not carry a prompt.** A sweep of the 106 built-in environments shipping example
+    data found 5 — ``aviary``, ``gdpval``, ``legal_agent_bench``, ``scicode`` and ``toolsandbox``
+    — whose every row ships ``responses_create_params.input == []``, letting Gym's data-prep step
+    or the environment's own agent materialize the prompt instead (see
+    :func:`_render_instruction`). Such a row yields a task with **no** ``inputs['instruction']``
+    key rather than a fabricated one, and is counted in a summary log line. This costs the
+    Gym path nothing: the runner never reads ``instruction`` — it re-materializes the source
+    row from ``inputs['gym_row']`` + ``metadata['gym_row_extras']`` and hands *that* to Gym,
+    and ``intent`` is a dataset label either way. It stays absent (not ``""``) so that
+    :meth:`AgentEvalTask.agent_prompt` still fails loudly for the runners that *do* need a
+    prompt — an instruction-less task must not reach an agent as an empty one.
+
+    ``responses_create_params`` itself remains required: Gym indexes into it unconditionally
+    (``rollout_collection._preprocess_rows_from_config``), so a row without it is rejected here
+    rather than crashing Gym mid-collection.
+    """
+    dataset = Path(dataset)
+    tasks: list[AgentEvalTask] = []
+    seen: set[str] = set()
+    duplicates = 0
+    promptless = 0
+    for position, row in enumerate(_read_jsonl(dataset), 1):
+        params = row.get("responses_create_params")
+        if not isinstance(params, Mapping):
+            raise ValueError(
+                f"row {position} of {dataset} has no 'responses_create_params' mapping; Gym requires that key "
+                "on every dataset row"
+            )
+        task_id = _canonical_row_hash(row)
+        if task_id in seen:
+            duplicates += 1
+            continue
+        seen.add(task_id)
+        instruction = _render_instruction(params)
+        if not instruction:
+            promptless += 1
+        tasks.append(
+            AgentEvalTask(
+                id=task_id,
+                intent=f"Gym row from {dataset.name}",
+                inputs={
+                    # Absent, not empty, when the row carries no prompt — see the docstring.
+                    **({"instruction": instruction} if instruction else {}),
+                    "gym_row": params,
+                },
+                metrics=list(metrics) if metrics is not None else [GymRewardMetric()],
+                metadata={
+                    "gym_dataset_path": str(dataset),
+                    # Everything except responses_create_params, which already lives in inputs['gym_row'].
+                    "gym_row_extras": {
+                        key: value
+                        for key, value in row.items()
+                        if key != "responses_create_params" and key not in _RUNTIME_KEYS
+                    },
+                },
+            )
+        )
+    if duplicates:
+        logger.warning(
+            "Collapsed %d duplicate row(s) in %s into %d distinct task(s): task identity is row content, so "
+            "repeating a row cannot request repeated attempts — set GymRuntimeConfig.num_repeats for that. "
+            "Duplicate rows usually indicate a data problem.",
+            duplicates,
+            dataset,
+            len(tasks),
+        )
+    if promptless:
+        logger.info(
+            "%d of %d task(s) in %s carry no prompt in responses_create_params and were discovered without an "
+            "inputs['instruction']. Expected for environments whose prompt is built by Gym's data-prep step or by "
+            "their own agent (e.g. aviary, gdpval, legal_agent_bench, scicode, toolsandbox); the Gym runner does "
+            "not read the instruction. A runner that hands the task straight to an agent will reject these tasks.",
+            promptless,
+            len(tasks),
+            dataset,
+        )
+    if not tasks:
+        raise ValueError(f"no rows found in Gym dataset {dataset}")
+    return tasks
+
+
+def _source_datasets(tasks: Sequence[AgentEvalTask]) -> str:
+    """Human-readable summary of the dataset(s) the tasks were discovered from, for a log line.
+
+    Purely provenance. Gym reads the dataset :func:`_materialize_dataset` writes from the tasks
+    themselves, so nothing here gates the run: tasks drawn from two datasets are legitimate (the
+    materialized file is their union), and a task with no stamped path runs fine. Never raises —
+    a missing or inconsistent label is not a reason to fail an otherwise valid evaluation.
+    """
+    stamped = {
+        task.metadata["gym_dataset_path"]
+        for task in tasks
+        if isinstance(task.metadata.get("gym_dataset_path"), str) and task.metadata["gym_dataset_path"]
+    }
+    return ", ".join(sorted(stamped)) if stamped else "<tasks with no stamped dataset path>"
+
+
+def _materialize_dataset(tasks: Sequence[AgentEvalTask], dest: Path) -> dict[int, str]:
+    """Write the normalized dataset Gym will read, and return its ``_ng_task_index`` → task-id map.
+
+    One line per requested task, in task order, carrying the task's full source row plus an explicitly
+    stamped ``_ng_task_index``. Gym honors a pre-supplied index instead of deriving one, so this makes
+    the rollout→task join total and order-independent, and confines the run to the tasks we asked for.
+
+    The row is reassembled from ``inputs['gym_row']`` (``responses_create_params``) and
+    ``metadata['gym_row_extras']`` (everything else), which :func:`discover_gym_tasks` keeps disjoint
+    so the run bundle doesn't store the same payload twice. Any pre-existing ``_ng_*`` fields are
+    stripped: ours is authoritative, and Gym assigns ``_ng_rollout_index`` itself per attempt.
+    """
+    index_to_task_id: dict[int, str] = {}
+    seen_task_ids: set[str] = set()
+    lines: list[str] = []
+    for index, task in enumerate(tasks):
+        # The source row is stored split across inputs/metadata so the run bundle doesn't persist
+        # responses_create_params twice; reassemble it here.
+        extras = task.metadata.get("gym_row_extras")
+        params = task.inputs.get("gym_row")
+        if not isinstance(extras, Mapping) or not isinstance(params, Mapping):
+            raise ValueError(
+                f"task {task.id!r} is missing inputs['gym_row'] and/or metadata['gym_row_extras']; build tasks "
+                "with discover_gym_tasks so the Gym dataset can be re-materialized for the run"
+            )
+        if task.id in seen_task_ids:
+            raise ValueError(
+                f"task {task.id!r} was supplied more than once; one distinct row is one task, and repeated "
+                "attempts come from GymRuntimeConfig.num_repeats"
+            )
+        seen_task_ids.add(task.id)
+        payload = {key: value for key, value in extras.items() if key not in _RUNTIME_KEYS}
+        payload["responses_create_params"] = params
+        payload[NG_TASK_INDEX] = index
+        lines.append(json.dumps(payload, default=str))
+        index_to_task_id[index] = task.id
+    if not index_to_task_id:
+        raise ValueError("GymAgentTaskRunner was given no tasks to run")
+    dest.write_text("\n".join(lines) + "\n", encoding="utf-8")
+    return index_to_task_id
diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/process.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/process.py
new file mode 100644
index 0000000000..b4871a9891
--- /dev/null
+++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/process.py
@@ -0,0 +1,188 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Running the ``gym`` CLI: locating it, watching it, and tearing it down.
+
+Gym spawns a Ray cluster and several uvicorn servers, so a subprocess here is a process *tree*.
+Everything about that lifecycle — resolution, streamed logs, readiness parsing, group teardown —
+lives here so the runtime module can read as orchestration rather than process management.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+import re
+import shutil
+import signal
+from collections import deque
+from collections.abc import Sequence
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+_GYM_CLI = "gym"
+#: Bound on `gym env validate`. It merges config without starting anything and returns in about a
+#: second; this only exists so a wedged invocation cannot stall the run before it begins.
+_VALIDATE_TIMEOUT_S = 120.0
+#: Where Gym's Hydra run directories are redirected, relative to the run's work dir.
+#: Token-count keys read to decide whether a model was called at all. Both vocabularies, because
+#: Gym's model servers report in either depending on the adapter: the Responses API spells them
+#: `input_tokens`/`output_tokens`, Chat Completions `prompt_tokens`/`completion_tokens`.
+_LOG_TAIL_LINES = 40
+
+
+#: Substrings that mark an override as carrying a credential. Matched case-insensitively against the
+#: full dotted path, so nesting cannot hide one behind an innocuous leaf name.
+def _gym_executable() -> str:
+    """Locate the ``gym`` CLI on PATH, or fail saying what to do about it.
+
+    Deliberately PATH-only, with no config field pointing at a checkout or a particular venv: these
+    runner configs become serialized job specs, and a local filesystem path cannot cross that
+    boundary. Resolving here rather than at spawn time turns a missing Gym into one legible error
+    instead of an ``ENOENT`` out of ``create_subprocess_exec`` after the run has already started.
+
+    Note that Gym generally cannot live in this SDK's own environment: it imports Ray at module load,
+    and nemo-platform excludes Ray by constraint over an unfixed CVE. Install Gym separately and put
+    its ``bin`` on PATH; in a job image, the image owns PATH and this resolves normally.
+    """
+    resolved = shutil.which(_GYM_CLI)
+    if resolved is None:
+        raise RuntimeError(
+            f"The {_GYM_CLI!r} CLI was not found on PATH. Install NeMo Gym in its own environment "
+            "(it needs Ray, which nemo-platform excludes over an unfixed CVE, so it cannot share this "
+            "one) and put that environment's `bin` directory on PATH. Each resources-server also "
+            "ships its own requirements.txt, installed from that server's directory."
+        )
+    return resolved
+
+
+_PENDING_SERVERS_RE = re.compile(r"(\d+)\s*/\s*(\d+) servers ready\. Waiting for servers to spin up: \[([^\]]*)\]")
+
+
+def _pending_servers(text: str) -> tuple[int, int, tuple[str, ...]] | None:
+    """Recover ``(ready, total, still-pending)`` from the *last* readiness line Gym logged.
+
+    The last line is the one that matters: earlier polls list servers that have since come up.
+    Returns ``None`` when Gym never got as far as printing one (it died during composition, say),
+    which the caller must treat as "no detail available" rather than "nothing pending".
+
+    Never raises — this only ever decorates an error that is already being reported, and a parsing
+    failure here must not replace the real one. Gym's log format is not contractual.
+    """
+    matches = _PENDING_SERVERS_RE.findall(text)
+    if not matches:
+        return None
+    ready, total, names = matches[-1]
+    try:
+        return int(ready), int(total), tuple(re.findall(r"['\"]([^'\"]+)['\"]", names))
+    except ValueError:  # pragma: no cover - the regex only matches digits
+        return None
+
+
+async def _pump_stream(
+    stream: asyncio.StreamReader | None,
+    path: Path,
+    *,
+    label: str,
+    tails: dict[str, deque[str]] | None = None,
+    key: str = "stdout",
+) -> None:
+    """Stream a subprocess pipe to ``path`` while mirroring it to the module logger at ``DEBUG``.
+
+    Reads in chunks rather than by line so a pathologically long line can't overrun asyncio's stream
+    limit, and retains only the last :data:`_LOG_TAIL_LINES` lines in memory (via ``tails[key]``) for
+    inclusion in a failure message — the file on disk is the complete record.
+    """
+    tail: deque[str] = deque(maxlen=_LOG_TAIL_LINES)
+    if tails is not None:
+        tails[key] = tail
+    if stream is None:
+        return
+
+    def emit(raw: bytes) -> None:
+        text = raw.decode("utf-8", errors="replace").rstrip("\r")
+        logger.debug("[%s %s] %s", label, key, text)
+        tail.append(text)
+
+    with path.open("wb") as handle:
+        buffer = b""
+        while True:
+            chunk = await stream.read(65536)
+            if not chunk:
+                break
+            handle.write(chunk)
+            handle.flush()
+            buffer += chunk
+            *complete, buffer = buffer.split(b"\n")
+            for line in complete:
+                emit(line)
+        if buffer:
+            emit(buffer)
+
+
+async def _drain_pumps(pumps: Sequence[asyncio.Task[None]], *, grace_s: float, what: str) -> None:
+    """Await log pumps after teardown, but never block on them indefinitely.
+
+    A pump ends at pipe EOF, which requires *every* inheritor of the write end to close it. Gym's
+    descendants inherit it, and Ray daemonizes ``gcs_server`` into its own session where our
+    process-group signals can't reach it (see :func:`_terminate`) — so a leaked grandchild can hold
+    the pipe open forever. Waiting unconditionally would hang the whole run inside a ``finally``.
+
+    Bounded instead: give the pumps ``grace_s`` to flush, then cancel and move on. Everything read
+    before the stall is already written and flushed to the log file, so cancelling costs at most the
+    tail of a transcript belonging to a process we just killed — and it is reported, not silent.
+    """
+    if not pumps:
+        return
+    _done, pending = await asyncio.wait(pumps, timeout=grace_s)
+    if not pending:
+        return
+    logger.warning(
+        "%d `%s` log pump(s) did not finish within %.1fs; the pipe is still held open (likely a Gym "
+        "grandchild that outlived teardown, e.g. Ray's detached gcs_server). Abandoning them — the log "
+        "file holds everything read up to this point, but its tail may be truncated.",
+        len(pending),
+        what,
+        grace_s,
+    )
+    for task in pending:
+        task.cancel()
+    await asyncio.gather(*pending, return_exceptions=True)
+
+
+def _signal_group(pgid: int, sig: int) -> None:
+    """Send ``sig`` to a process group, tolerating an already-dead group."""
+    try:
+        os.killpg(pgid, sig)
+    except ProcessLookupError:
+        pass
+
+
+async def _terminate(proc: asyncio.subprocess.Process, *, grace_s: float = 30.0) -> None:
+    """Tear down a backgrounded Gym subprocess *and its whole process group*.
+
+    ``gym env start`` fans out into a Ray cluster + uvicorn servers, and Gym stops them from a
+    ``KeyboardInterrupt`` handler (``finally: self.shutdown()``) — so we send **SIGINT** (not SIGTERM,
+    which would bypass that handler and orphan Ray's detached ``gcs_server``). The child is spawned with
+    ``start_new_session=True`` and leads its own group, so SIGINT to the group mimics Ctrl-C exactly;
+    SIGKILL to the group is the escalation if Gym's graceful shutdown overstays ``grace_s``.
+
+    Known limitation: Ray daemonizes ``gcs_server`` into its *own* session, outside this group, so the
+    escalation SIGKILL cannot reach it — clean Ray teardown depends on Gym's SIGINT ``shutdown()``
+    completing within ``grace_s``. POSIX-only (``killpg`` / ``getpgid`` / ``start_new_session`` / SIGINT).
+    """
+    if proc.returncode is not None:
+        return
+    try:
+        pgid = os.getpgid(proc.pid)
+    except ProcessLookupError:
+        return
+    _signal_group(pgid, signal.SIGINT)
+    try:
+        await asyncio.wait_for(proc.wait(), timeout=grace_s)
+    except asyncio.TimeoutError:
+        _signal_group(pgid, signal.SIGKILL)
+        await proc.wait()
diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/records.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/records.py
new file mode 100644
index 0000000000..6fec4fd4a0
--- /dev/null
+++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/records.py
@@ -0,0 +1,52 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Gym's on-disk record format: the artifacts it writes, and the key that joins them.
+
+Shared deliberately between :mod:`dataset` (which *writes* the index onto a materialized row) and
+:mod:`results` (which *reads* it back off a rollout). The attribution contract is exactly this
+pair of key names, so one definition means the writer and the reader cannot disagree.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+
+NG_TASK_INDEX = "_ng_task_index"
+NG_ROLLOUT_INDEX = "_ng_rollout_index"
+#: Fields excluded from a row's content hash (runtime-injected, not task-defining).
+_RUNTIME_KEYS = frozenset({NG_TASK_INDEX, NG_ROLLOUT_INDEX})
+
+
+#: Lines of subprocess output retained in memory for inclusion in a failure message.
+def _read_jsonl(path: str | Path, *, tolerant: bool = False) -> list[dict[str, Any]]:
+    """Read a jsonl file. With ``tolerant=True``, skip (and log) malformed lines instead of raising —
+
+    used for Gym's ``*_failures.jsonl`` sidecar, which is written during abnormal termination and can
+    end in a truncated line; a corrupt failure record must not sink the successfully-collected trials.
+    """
+    rows: list[dict[str, Any]] = []
+    with Path(path).open(encoding="utf-8") as handle:
+        for line_no, line in enumerate(handle, 1):
+            stripped = line.strip()
+            if not stripped:
+                continue
+            try:
+                rows.append(json.loads(stripped))
+            except json.JSONDecodeError:
+                if not tolerant:
+                    raise
+                logger.warning("Skipping malformed JSON at %s:%d", path, line_no)
+    return rows
+
+
+#: `gym env start`'s combined output, under the run's work dir. Named here with the other
+#: artifacts because a *collection* failure often has to point at it: the eval logs show the
+#: symptom, this shows the cause.
+_ENV_LOG_NAME = "gym_env.log"
diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/results.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/results.py
new file mode 100644
index 0000000000..6c5c9c09ff
--- /dev/null
+++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/results.py
@@ -0,0 +1,556 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""What Gym produced, read back as trials and scores.
+
+Covers the rollout and failure records, the aggregate metrics Gym computes for itself, and the
+judgement calls in between: which records represent an agent that actually ran, and which reward
+is a measurement rather than an artefact of one that did not.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import math
+from collections.abc import Mapping, Sequence
+from pathlib import Path
+from typing import Any
+
+from nemo_platform.beta.evaluator.agent_eval.runtimes.gym.config import DEFAULT_REWARD_KEY
+from nemo_platform.beta.evaluator.agent_eval.runtimes.gym.records import (
+    _ENV_LOG_NAME,
+    NG_ROLLOUT_INDEX,
+    NG_TASK_INDEX,
+    _read_jsonl,
+)
+from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalTask
+from nemo_platform.beta.evaluator.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput
+from nemo_platform.beta.evaluator.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult
+from nemo_platform.beta.evaluator.values.evidence import CandidateEvidence, EvidenceDescriptor
+from nemo_platform.beta.evaluator.values.results import AggregateRangeScore, AggregateScalarScore, AggregateScore
+
+logger = logging.getLogger(__name__)
+
+
+_TOKEN_USAGE_KEYS = ("total_tokens", "input_tokens", "output_tokens", "prompt_tokens", "completion_tokens")
+#: The subset that proves a call happened. A model call always consumes input tokens, so zero on any
+#: of these is evidence the model was never reached; zero output tokens only means an empty answer.
+_INPUT_TOKEN_KEYS = ("total_tokens", "input_tokens", "prompt_tokens")
+
+
+#: Gym's index fields on each rollout record. ``_ng_task_index`` is the only join back to the input
+#: rows that survives a round-trip: Gym mutates ``responses_create_params`` (even the prompt) and
+#: copies only a fixed allowlist of row keys onto the result, so no field we invent comes back. Gym
+#: *honors* a caller-supplied ``_ng_task_index``, which is what makes the join here deterministic.
+class GymRewardMetric:
+    """Score the Gym verifier reward stamped onto trial metadata.
+
+    The Gym analogue of :class:`HarborRewardMetric`: reads the per-trial ``reward``
+    off the candidate metadata (populated by :class:`GymAgentTaskRunner`); a trial
+    with no reward is left **unscored** (``None`` → ``nan``), excluded from the mean
+    and surfaced as ``nan_count`` rather than counted as a spurious ``0.0``. Gym owns
+    the scoring — this metric only surfaces it (Evaluator does not re-derive the reward).
+    """
+
+    def __init__(self, *, output_name: str = "reward", metric_type: str = "gym_reward") -> None:
+        self._output_name = output_name
+        self._metric_type = metric_type
+
+    @property
+    def type(self) -> str:
+        return self._metric_type
+
+    def output_spec(self) -> list[MetricOutputSpec]:
+        return [MetricOutputSpec.continuous_score(self._output_name)]
+
+    async def compute_scores(self, input: MetricInput) -> MetricResult:
+        reward = input.candidate.metadata.get("reward")
+        value = float(reward) if reward is not None else None
+        return MetricResult(outputs=[MetricOutput(name=self._output_name, value=value)])
+
+
+def _agent_never_ran(record: Mapping[str, Any]) -> bool:
+    """True when a result record shows the agent produced nothing *and* never called the model.
+
+    A verifier-scored runner reports whatever score the environment computed, so an agent that
+    silently failed to start is indistinguishable from one that tried and scored 0.0 — the record
+    looks normal, no failure is reported alongside it, and the run succeeds. Observed on
+    ``legal_agent_bench``, whose agent died locating its task tree and returned an empty response:
+    two "completed" trials scoring 0.0, no model call, no failure recorded.
+
+    Deliberately conservative — **both** conditions must hold:
+
+    * no output, and
+    * a stated zero on the input side of token usage.
+
+    An empty answer alone is a legitimate result (a model can return nothing and earn 0.0), so it is
+    not sufficient. Zero *input* tokens is what says the model was never called: a call always sends
+    a prompt, whereas zero output tokens only describes an empty answer. A record carrying no
+    ``response`` key at all is left alone — some runners populate it differently, and this must not
+    reclassify trials it does not understand.
+    """
+    response = record.get("response")
+    if not isinstance(response, Mapping):
+        return False
+    if "output" not in response:
+        # Nothing to judge: some runners shape `response` differently, and absence is not emptiness.
+        return False
+    # Any non-empty value counts as output, whatever its shape. Checking `len(...) > 0` on a list
+    # alone would treat a plain string answer — or a single mapping — as "produced nothing" and fail
+    # a trial that ran perfectly well.
+    if response["output"]:
+        return False
+    usage = response.get("usage")
+    if not isinstance(usage, Mapping):
+        # No usage block to corroborate with; an empty output on its own is not enough to judge.
+        return False
+    # Zero usage has to be *stated*, not inferred from a key being absent. Two vocabularies are in
+    # play — the Responses API's `input_tokens`/`output_tokens` and Chat Completions'
+    # `prompt_tokens`/`completion_tokens` — and Gym's model servers speak both depending on the
+    # adapter. Reading only one set would make a real call reported in the other read as zero.
+    reported = [usage[key] for key in _TOKEN_USAGE_KEYS if isinstance(usage.get(key), (int, float))]
+    if any(value > 0 for value in reported):
+        return False
+    # The discriminating evidence is *input*-side, and only input-side. Calling a model always sends
+    # a prompt, so zero input tokens cannot describe a call that happened; zero **output** tokens
+    # merely describes an empty answer, which is a legitimate result. Requiring an input-side count
+    # rules out a whole class of false positives — a provider reporting only `output_tokens: 0`, or
+    # only `completion_tokens: 0` — rather than excluding those shapes one at a time.
+    input_side = [usage[key] for key in _INPUT_TOKEN_KEYS if isinstance(usage.get(key), (int, float))]
+    return bool(input_side) and all(value == 0 for value in input_side)
+
+
+def _require_full_coverage(tasks: Sequence[AgentEvalTask], *, covered_task_ids: set[str], rollouts_path: Path) -> None:
+    """Fail the run when Gym produced no trial at all for some requested task.
+
+    :class:`AgentEvaluator` already refuses to score a task with no trial
+    (``evaluator._score_trials``), and deliberately so: an incomplete run must not read as a
+    successful one with quieter counts. We keep that contract rather than papering over it with
+    synthesized FAILED trials — an unscored trial is excluded from the mean, so fabricating them
+    would report a plausible-looking average over an unannounced subset of the dataset. A rollout
+    that failed in a *diagnosable* way already becomes a FAILED trial via the failures sidecar;
+    reaching here means the attempt is unaccounted for entirely.
+
+    What this adds over letting the evaluator raise is diagnosis: by the time the run fails the full
+    collection cost is already paid, so the error names the evidence instead of just the task ids.
+    """
+    missing = sorted({task.id for task in tasks} - covered_task_ids)
+    if not missing:
+        return
+    raise RuntimeError(
+        f"Gym produced no rollout for {len(missing)} of {len(tasks)} requested task(s), so the run is "
+        f"incomplete and will not be scored (a partial run must not be reported as a whole one).\n"
+        f"  rollouts:  {rollouts_path}\n"
+        f"  failures:  {_failures_path_for(rollouts_path)}\n"
+        f"  gym logs:  {rollouts_path.parent} (gym_env.log, gym_eval.stdout.log, gym_eval.stderr.log)\n"
+        f"  unrepresented task id(s): {missing}"
+    )
+
+
+def _failures_path_for(rollouts_path: Path) -> Path:
+    """Sidecar Gym writes failed rollouts to (mirrors Gym's own ``_failures_path_for``)."""
+    return rollouts_path.with_name(rollouts_path.stem + "_failures.jsonl")
+
+
+def _aggregate_metrics_path_for(rollouts_path: Path) -> Path:
+    """Sidecar Gym writes run-level aggregate metrics to (``<stem>_aggregate_metrics.json``)."""
+    return rollouts_path.with_name(rollouts_path.stem + "_aggregate_metrics.json")
+
+
+def _read_run_aggregations(rollouts_path: Path) -> dict[str, Any] | None:
+    """Parse Gym's ``rollouts_aggregate_metrics.json``, or ``None`` when absent/unparseable.
+
+    Gym's file is a list with one entry per agent (``agent_ref`` / ``agent_metrics`` / ``key_metrics`` /
+    ``group_level_metrics``), so it is returned keyed by agent name. Carried through as-is — Gym's schema
+    isn't contractual, so the SDK does not type it.
+    """
+    path = _aggregate_metrics_path_for(rollouts_path)
+    if not path.exists():
+        return None
+    try:
+        parsed = json.loads(path.read_text(encoding="utf-8"))
+    except (json.JSONDecodeError, OSError, UnicodeDecodeError):
+        # UnicodeDecodeError is a ValueError, not an OSError: a sidecar truncated mid-codepoint would
+        # otherwise raise straight out of run_tasks and discard a collection that already succeeded.
+        logger.warning("Could not parse Gym aggregate metrics at %s; skipping run aggregations.", path)
+        return None
+    if not isinstance(parsed, list):
+        logger.warning("Unexpected Gym aggregate-metrics shape at %s (%s); skipping.", path, type(parsed).__name__)
+        return None
+    # Gym's schema is not contractual, and this runs after a collection that already succeeded — an
+    # entry in an unexpected shape must not raise out of run_tasks and discard every trial with it.
+    aggregations: dict[str, Any] = {}
+    for entry in parsed:
+        agent_ref = entry.get("agent_ref") if isinstance(entry, Mapping) else None
+        name = agent_ref.get("name") if isinstance(agent_ref, Mapping) else None
+        if not isinstance(name, str):
+            logger.warning(
+                "Skipping a Gym aggregate-metrics entry in %s with no usable agent_ref.name (got %r).", path, agent_ref
+            )
+            continue
+        aggregations[name] = {key: value for key, value in entry.items() if key != "agent_ref"}
+    return aggregations or None
+
+
+#: Gym flattens a distribution into ``<stat>/<metric>`` keys. Its RewardProfiler emits this exact set
+#: together for every numeric column (``describe_dataframe``), minus ``histogram``, which
+#: ``prepare_for_serialization`` strips before the file is written. All five must be present before we
+#: treat a group of keys as one distribution: a resources-server is free to define a metric literally
+#: named ``mean`` (36 of Gym's ~97 servers override ``compute_metrics``), and re-assembling on a partial
+#: match would rename someone's standalone metric into a statistic of a distribution that never existed.
+_GYM_STAT_FAMILY = ("mean", "max", "min", "median", "std")
+
+#: Metric Gym reports that the SDK already computes natively from the same rollouts (``gym_reward.reward``).
+_GYM_REDUNDANT_METRICS = frozenset({"reward"})
+
+
+def _aggregate_scores_from_gym(aggregations: Mapping[str, Any] | None) -> list[AggregateScore]:
+    """Map Gym's run-level ``agent_metrics`` onto typed aggregate scores named ``runner.gym.<metric>``.
+
+    Reads ``agent_metrics``, not ``key_metrics``: ``key_metrics`` is a *subset* of it chosen by the
+    resources-server, and the default selection (``get_key_metrics``) keeps only the ``mean/*`` entries —
+    so the max/min/median/std that make a distribution never appear there, and every metric would arrive
+    as a lone ``mean/<name>`` scalar.
+
+    Keys forming a full stat family are re-assembled into one :class:`AggregateRangeScore`; every other
+    numeric key becomes an :class:`AggregateScalarScore`. Non-numeric values are skipped — they are
+    labels or notes, not measurements.
+
+    Names are namespaced by runner (not by agent): each run is instrumented with a single agent, so the
+    agent adds no disambiguation, and a reader needs to know which *backend* produced a number.
+    """
+    if not aggregations:
+        return []
+    multi_agent = len(aggregations) > 1
+    scores: list[AggregateScore] = []
+    for agent_name, payload in sorted(aggregations.items()):
+        agent_metrics = payload.get("agent_metrics") if isinstance(payload, Mapping) else None
+        if not isinstance(agent_metrics, Mapping):
+            continue
+        # One agent per run is the norm, so `runner.gym.<metric>` reads cleanly; qualify by agent only
+        # when a run really did produce several, where the unqualified names would collide.
+        prefix = f"runner.gym.{agent_name}." if multi_agent else "runner.gym."
+        scores.extend(_scores_from_agent_metrics(agent_metrics, prefix=prefix))
+    return scores
+
+
+def _scores_from_agent_metrics(agent_metrics: Mapping[str, Any], *, prefix: str) -> list[AggregateScore]:
+    families: dict[str, dict[str, float]] = {}
+    scalars: dict[str, float] = {}
+    for key, value in agent_metrics.items():
+        number = _as_float(value)
+        if number is None:
+            continue
+        stat, _, metric = key.partition("/")
+        if metric and stat in _GYM_STAT_FAMILY:
+            families.setdefault(metric, {})[stat] = number
+        else:
+            scalars[key] = number
+
+    scores: list[AggregateScore] = []
+    for metric, stats in sorted(families.items()):
+        # Redundancy is a property of the metric, not of how complete its family is. Checking after the
+        # fallback below would let a partial `reward` family survive as `mean/reward` scalars, since the
+        # scalar filter matches the bare name -- reintroducing the duplicate of `gym_reward.reward` that
+        # skipping reward exists to prevent.
+        if metric in _GYM_REDUNDANT_METRICS:
+            continue
+        if set(stats) < set(_GYM_STAT_FAMILY):
+            # Not a distribution we can vouch for; keep each key as the standalone number it may well be.
+            scalars.update({f"{stat}/{metric}": value for stat, value in stats.items()})
+            continue
+        scores.append(
+            AggregateRangeScore(
+                name=f"{prefix}{metric}",
+                # Gym reports the statistics but not the sample size behind them, and inventing one
+                # would misreport coverage. `None` says "unknown"; 0 would assert nothing was evaluated.
+                count=None,
+                nan_count=0,
+                mean=stats["mean"],
+                min=stats["min"],
+                max=stats["max"],
+                median=stats["median"],
+                # Gym computes this with pandas (ddof=1), so it is the sample standard deviation.
+                sample_std_dev=stats["std"],
+                sample_variance=stats["std"] ** 2,
+            )
+        )
+    scores.extend(
+        AggregateScalarScore(name=f"{prefix}{key}", count=None, nan_count=0, value=value)
+        for key, value in sorted(scalars.items())
+        if key not in _GYM_REDUNDANT_METRICS
+    )
+    return scores
+
+
+def _as_float(value: Any) -> float | None:
+    """``float`` for a numeric Gym metric value; None for anything else (bools included: not measurements)."""
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        return None
+    return float(value) if math.isfinite(value) else None
+
+
+def _ensure_fresh_output(rollouts_path: Path) -> None:
+    """Enforce one Gym run per output dir (the AgentEvaluator convention).
+
+    Gym appends to the failures sidecar (``open("ab")``) and doesn't clear it between runs, so reusing
+    a populated directory would silently mix this run's failures with a prior run's. Rather than clear
+    (which would clobber an earlier run's results — infra failures are useful signal), refuse to run
+    into a directory that already holds Gym rollout output.
+    """
+    preexisting = [path for path in (rollouts_path, _failures_path_for(rollouts_path)) if path.exists()]
+    if preexisting:
+        names = ", ".join(path.name for path in preexisting)
+        raise FileExistsError(
+            f"{rollouts_path.parent} already holds Gym rollout output ({names}); give each run a fresh "
+            "output_dir (the AgentEvaluator convention). Gym appends to the failures sidecar, so reusing a "
+            "directory would mix runs and could obscure a prior run's results."
+        )
+
+
+def _coerce_reward(value: Any) -> float | None:
+    """Best-effort ``float`` for a reward; returns None (never raises) on a missing/malformed value."""
+    if value is None:
+        return None
+    try:
+        return float(value)
+    except (TypeError, ValueError):
+        logger.warning("Gym rollout carried a non-numeric reward %r; recording the trial as unscored.", value)
+        return None
+
+
+def _resolve_task_id(
+    record: Mapping[str, Any], index_to_task_id: Mapping[int, str], *, strict: bool = True
+) -> str | None:
+    """Map a rollout record to a task id via the ``_ng_task_index`` we stamped in the input dataset.
+
+    Returns None when the record carries no usable index (kill-shaped failures can lack one).
+
+    An index we never stamped is handled by ``strict``. In the **successes** file (``strict=True``)
+    it is fatal: every row Gym read came from :func:`_materialize_dataset`, so an unknown index means
+    Gym reindexed the dataset rather than honoring our stamp, and every reward attribution is
+    therefore suspect. In the **failures sidecar** (``strict=False``) it is merely unattributable and
+    is counted instead — that file is written during abnormal termination and read tolerantly by
+    design, so one odd record must not discard the successes already parsed. That relaxation cannot
+    inflate a result: failure records carry no reward, and a task left with no trial still trips
+    :func:`_require_full_coverage`.
+    """
+    index = record.get(NG_TASK_INDEX)
+    if not isinstance(index, int) or isinstance(index, bool):
+        return None
+    task_id = index_to_task_id.get(index)
+    if task_id is None:
+        if not strict:
+            logger.warning(
+                "Gym failure record carried %s=%s, which was not stamped on the materialized dataset; "
+                "counting it as unattributable rather than failing the run.",
+                NG_TASK_INDEX,
+                index,
+            )
+            return None
+        raise ValueError(
+            f"Gym emitted {NG_TASK_INDEX}={index}, which is not one of the {len(index_to_task_id)} index value(s) "
+            f"stamped on the materialized dataset. Gym appears to have reassigned {NG_TASK_INDEX} instead "
+            "of honoring the supplied one, so rollout-to-task attribution is unsafe."
+        )
+    return task_id
+
+
+def _rollout_trial_id(task_id: str, raw_rollout_index: Any, synth_seq: dict[str, int], *, missing_label: str) -> str:
+    """Per-attempt trial id: use the record's ``_ng_rollout_index`` when it's a real int, else a
+    per-task-unique ``{missing_label}{n}`` suffix so records lacking an index don't collide on one id."""
+    if isinstance(raw_rollout_index, int) and not isinstance(raw_rollout_index, bool):
+        return f"{task_id}:{raw_rollout_index}"
+    seq = synth_seq[task_id] = synth_seq.get(task_id, 0) + 1
+    return f"{task_id}:{missing_label}{seq}"
+
+
+def _trials_from_rollouts(
+    rollouts_path: Path,
+    tasks: Sequence[AgentEvalTask],
+    index_to_task_id: Mapping[int, str],
+    *,
+    reward_key: str = DEFAULT_REWARD_KEY,
+) -> list[AgentEvalTrial]:
+    """Fan Gym's rollout records out into one :class:`AgentEvalTrial` per attempt.
+
+    Reads *both* Gym output files: successes from ``rollouts.jsonl`` (COMPLETED, carrying the reward)
+    and failures from the ``*_failures.jsonl`` sidecar (FAILED — so a failed attempt is counted and
+    diagnosed rather than silently dropped). Every record is attributed to its task by
+    ``_ng_task_index`` → ``index_to_task_id``, the map we stamped in :func:`_materialize_dataset`.
+    """
+    if not rollouts_path.exists():
+        # `_ensure_fresh_output` removed any stale file before the run, and `_collect_rollouts`
+        # already raised on a non-zero exit — so reaching here means Gym reported success and wrote
+        # nothing. Opening the path regardless would surface that as a bare FileNotFoundError
+        # naming a file the reader has no reason to know about, which is the same illegibility the
+        # rest of this change set exists to remove.
+        raise RuntimeError(
+            f"`gym eval run` reported success but wrote no results to {rollouts_path}, so there is "
+            "nothing to score. This usually means collection stopped before any attempt completed.\n"
+            f"See {rollouts_path.parent / 'gym_eval.stdout.log'} for what collection did, and "
+            f"{rollouts_path.parent / _ENV_LOG_NAME} for a traceback from the environment's own "
+            "servers."
+        )
+    known_task_ids = {task.id for task in tasks}
+    # Defence in depth: the index map and the task list must describe the same run. Gym is an external
+    # project on its own release cadence, and people make agent-optimization decisions from these
+    # numbers — so any disagreement about *which task a result belongs to* is a hard error, never a
+    # best-effort join. Cheap to check once; impossible to notice downstream if it silently drifts.
+    mapped_task_ids = set(index_to_task_id.values())
+    if mapped_task_ids != known_task_ids:
+        raise ValueError(
+            f"the {NG_TASK_INDEX} map does not match the task list it will be joined against "
+            f"({len(mapped_task_ids - known_task_ids)} mapped task(s) absent from the list, "
+            f"{len(known_task_ids - mapped_task_ids)} listed task(s) absent from the map); "
+            "attributing rollouts across a mismatch would report results against the wrong tasks"
+        )
+    trials: list[AgentEvalTrial] = []
+    # Shared per-task counter for records lacking a usable _ng_rollout_index (success + failure); a
+    # missing/null index must not collapse multiple attempts for one task onto the same trial id.
+    synth_seq: dict[str, int] = {}
+
+    success_evidence = CandidateEvidence(
+        descriptors={"rollouts": EvidenceDescriptor(kind="filesystem", format="file", ref=str(rollouts_path))}
+    )
+    unattributed_successes = 0
+    empty_output_count = 0
+    #: Tasks with at least one trial where the agent never ran. Tracked separately from trial
+    #: status, since the failures sidecar also produces FAILED trials for unrelated reasons.
+    empty_output_task_ids: set[str] = set()
+    for record in _read_jsonl(rollouts_path):
+        task_id = _resolve_task_id(record, index_to_task_id)
+        if task_id is None:
+            # No usable index (an unknown one raises). Skipping is right, but silence is not: with
+            # num_repeats > 1 the task keeps its other trials, so _require_full_coverage won't fire and
+            # the task would simply be scored on fewer attempts than were actually run.
+            unattributed_successes += 1
+            continue
+        raw_rollout_index = record.get(NG_ROLLOUT_INDEX)
+        reward = _coerce_reward(record.get(reward_key))
+        # An agent that never ran did not earn its reward — reporting 0.0 as a completed trial would
+        # let a broken environment read as a poorly-performing one.
+        never_ran = _agent_never_ran(record)
+        if never_ran:
+            empty_output_count += 1
+            empty_output_task_ids.add(task_id)
+            status = AgentEvalTrialStatus.FAILED
+        elif reward is not None:
+            status = AgentEvalTrialStatus.COMPLETED
+        else:
+            status = AgentEvalTrialStatus.PARTIAL
+        metadata: dict[str, Any] = {
+            "reward": None if never_ran else reward,
+            NG_TASK_INDEX: record.get(NG_TASK_INDEX),
+            NG_ROLLOUT_INDEX: raw_rollout_index,
+        }
+        if never_ran:
+            # Kept alongside the reward Gym reported, so the discarded value is still auditable.
+            metadata["gym_failure"] = (
+                "the agent produced no output and consumed no tokens, so the model was never called; "
+                f"the reported score of {reward!r} is not a measurement of this agent"
+            )
+        trials.append(
+            AgentEvalTrial(
+                id=_rollout_trial_id(task_id, raw_rollout_index, synth_seq, missing_label="noidx"),
+                task_id=task_id,
+                status=status,
+                output=AgentOutput(response=record.get("response"), metadata={"agent_ref": record.get("agent_ref")}),
+                evidence=success_evidence,
+                metadata=metadata,
+            )
+        )
+
+    failures_path = _failures_path_for(rollouts_path)
+    unattributed_failures = 0
+    if failures_path.exists():
+        failure_evidence = CandidateEvidence(
+            descriptors={
+                "rollout_failures": EvidenceDescriptor(kind="filesystem", format="file", ref=str(failures_path))
+            }
+        )
+        # tolerant: a truncated line in the abnormal-termination sidecar must not sink the good trials.
+        for record in _read_jsonl(failures_path, tolerant=True):
+            # strict=False to match this loop's tolerant read: an odd record is counted, not fatal.
+            task_id = _resolve_task_id(record, index_to_task_id, strict=False)
+            if task_id is None:
+                # kill-shaped failures (SIGTERM/OOM/actor-death) can lack a usable index — count, don't drop.
+                unattributed_failures += 1
+                continue
+            raw_rollout_index = record.get(NG_ROLLOUT_INDEX)
+            trials.append(
+                AgentEvalTrial(
+                    id=_rollout_trial_id(task_id, raw_rollout_index, synth_seq, missing_label="fail"),
+                    task_id=task_id,
+                    status=AgentEvalTrialStatus.FAILED,
+                    output=AgentOutput(
+                        response=record.get("response"), metadata={"agent_ref": record.get("agent_ref")}
+                    ),
+                    evidence=failure_evidence,
+                    metadata={
+                        "reward": None,
+                        # best-effort: Gym's failure-record schema isn't contractual, so probe common keys.
+                        "gym_failure": record.get("error") or record.get("exception") or record.get("failure_reason"),
+                        NG_TASK_INDEX: record.get(NG_TASK_INDEX),
+                        NG_ROLLOUT_INDEX: raw_rollout_index,
+                    },
+                )
+            )
+    if empty_output_count:
+        # Every trial empty is a configuration failure, not a result: nothing was measured, so there
+        # is no score to report and letting the run "succeed" would publish zeros as findings. A
+        # partial count is left as failed trials — a flaky environment is still worth the attempts
+        # that did run.
+        #
+        # Measured against *every* trial, including those the failures sidecar contributed. A run
+        # that also recorded real failures is not the uniform "nothing ran" picture this raise is
+        # for: those trials carry their own diagnosis (a verifier timeout, say, from an attempt that
+        # did reach the model), and raising would discard it for a less specific message.
+        #
+        # The unattributed counts need the same treatment and are easy to miss, because those
+        # records never became trials and so are invisible to `len(trials)`. Each is still a result
+        # the environment produced — an unattributed *success* may have run perfectly well, and an
+        # unattributed *failure* is a diagnosis of its own (an OOM-killed agent, say). Claiming
+        # nothing was measured while either exists would be false.
+        if empty_output_count == len(trials) and not unattributed_successes and not unattributed_failures:
+            affected_tasks = {trial.task_id for trial in trials}
+            raise RuntimeError(
+                f"No agent ran: all {empty_output_count} trial(s) across all {len(affected_tasks)} "
+                "task(s) produced no output and consumed no tokens, so the model was never called "
+                "and nothing was measured. The scores reported for them describe an agent that did "
+                "not run.\n"
+                "This is normally the environment failing to start its agent rather than a bad "
+                f"model: check the per-server startup output in {rollouts_path.parent / _ENV_LOG_NAME} "
+                "for a traceback from the environment's own agent server.\n"
+                f"Results as collected: {rollouts_path}"
+            )
+        logger.warning(
+            "%d of %d trial(s), across %d of %d task(s), produced no agent output and consumed no "
+            "tokens; those are marked FAILED rather than scored, since the model was never called "
+            "for them.",
+            empty_output_count,
+            len(trials),
+            len(empty_output_task_ids),
+            len({trial.task_id for trial in trials}),
+        )
+    if unattributed_successes:
+        logger.warning(
+            "%d Gym rollout record(s) in %s carried no usable %s and could not be attributed to a task; "
+            "affected tasks are scored on fewer attempts than were collected.",
+            unattributed_successes,
+            rollouts_path,
+            NG_TASK_INDEX,
+        )
+    if unattributed_failures:
+        logger.warning(
+            "%d Gym failure record(s) lacked a usable %s and could not be attributed to a task.",
+            unattributed_failures,
+            NG_TASK_INDEX,
+        )
+
+    # Reported here, enforced by _require_full_coverage in run_tasks (which knows the run's log paths).
+    missing = known_task_ids - {trial.task_id for trial in trials}
+    if missing:
+        logger.warning("No Gym rollout produced a trial for %d requested task(s): %s", len(missing), sorted(missing))
+    return trials
diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/runtime.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/runtime.py
new file mode 100644
index 0000000000..252dc003d6
--- /dev/null
+++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym/runtime.py
@@ -0,0 +1,383 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""The runner itself: the two-step Gym invocation, start to scored trials.
+
+Orchestration only. The stages it drives live in sibling modules — :mod:`config` for what Gym is
+told, :mod:`dataset` for what it is given, :mod:`process` for how it is run, and :mod:`results`
+for what comes back.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+import re
+import tempfile
+from collections import deque
+from collections.abc import Mapping, Sequence
+from pathlib import Path
+from typing import Any
+
+from nemo_platform.beta.evaluator.agent_eval.runtimes.gym.config import (
+    _HYDRA_SUBDIR,
+    GymRuntimeConfig,
+    _hydra_scalar,
+    _redact_env_overrides,
+    _selection_args,
+)
+from nemo_platform.beta.evaluator.agent_eval.runtimes.gym.dataset import _materialize_dataset, _source_datasets
+from nemo_platform.beta.evaluator.agent_eval.runtimes.gym.process import (
+    _LOG_TAIL_LINES,
+    _VALIDATE_TIMEOUT_S,
+    _drain_pumps,
+    _gym_executable,
+    _pending_servers,
+    _pump_stream,
+    _terminate,
+)
+from nemo_platform.beta.evaluator.agent_eval.runtimes.gym.records import _ENV_LOG_NAME
+from nemo_platform.beta.evaluator.agent_eval.runtimes.gym.results import (
+    _aggregate_scores_from_gym,
+    _ensure_fresh_output,
+    _read_run_aggregations,
+    _require_full_coverage,
+    _trials_from_rollouts,
+)
+from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask
+from nemo_platform.beta.evaluator.agent_eval.trials import AgentEvalTrial, RunnerInfo
+from nemo_platform.beta.evaluator.values.results import AggregateScore
+
+logger = logging.getLogger(__name__)
+
+
+class GymAgentTaskRunner:
+    """An :class:`AgentTaskRunner` that runs an existing Gym env, then adapts its rollouts.
+
+    :meth:`run_tasks` materializes the requested tasks into a normalized dataset with
+    ``_ng_task_index`` stamped per row, runs the two-step Gym flow (``env start`` +
+    ``eval run --no-serve --input``) against it via subprocess, and fans the rollout
+    records out into one trial per ``_ng_rollout_index``. Because we assign the index,
+    attribution is a dict lookup rather than a positional guess, and Gym only rolls out
+    the tasks we asked for. The durable task **identity** stays our content hash.
+    """
+
+    def __init__(self, *, config: GymRuntimeConfig) -> None:
+        self._config = config
+        self._run_aggregations: dict[str, Any] | None = None
+
+    def run_aggregate_scores(self) -> Sequence[AggregateScore]:
+        """Gym's ``agent_metrics`` mapped onto typed aggregate scores, namespaced ``runner.gym.<metric>``.
+
+        Satisfies :class:`RunAggregationsProvider`. ``reward`` is skipped: the SDK already scores it
+        natively as ``gym_reward.reward``, and two differently-derived numbers under one name invites
+        exactly the confusion the namespace is there to prevent.
+        """
+        return _aggregate_scores_from_gym(self._run_aggregations)
+
+    def runner_info(self) -> RunnerInfo:
+        """Identify this runner and the Gym settings that shape its results.
+
+        Credentials normally live in the Gym checkout's gitignored ``env.yaml`` and never reach this
+        object — but ``env_overrides`` is a free-form escape hatch, so its values are redacted by key
+        (see :func:`_redact_env_overrides`) rather than trusted.
+        """
+        cfg = self._config
+        return RunnerInfo(
+            name="gym",
+            kind="runner",
+            config={
+                "resources_server": cfg.resources_server,
+                "agent": cfg.agent,
+                "agent_config": cfg.agent_config,
+                "model_type": cfg.model_type,
+                "num_repeats": cfg.num_repeats,
+                "concurrency": cfg.concurrency,
+                "bind_resources_server": cfg.bind_resources_server,
+                "env_overrides": _redact_env_overrides(cfg.env_overrides),
+                "reward_key": cfg.reward_key,
+            },
+        )
+
+    async def run_tasks(
+        self,
+        tasks: Sequence[AgentEvalTask],
+        config: AgentEvalRunConfig | None = None,
+    ) -> list[AgentEvalTrial]:
+        cfg = self._config
+        self._run_aggregations = None  # reset per run so a reused runner never leaks a prior run's numbers
+        # Provenance for the log line only — the file Gym actually reads is the normalized one we
+        # materialize below from the tasks themselves.
+        source_dataset = _source_datasets(tasks)
+
+        # config.parallelism still governs how the evaluator *scores* the trials we return (its scoring
+        # semaphore). It is deliberately not mapped onto Gym's rollout `--concurrency`: those are different
+        # phases — parallelism bounds concurrent scoring (SDK-side, cheap), while Gym's `--concurrency`
+        # bounds concurrent rollouts against the model endpoint during collection (tuned to that endpoint's
+        # limits via GymRuntimeConfig.concurrency).
+        if config is not None and config.work_dir is not None:
+            work_dir = Path(config.work_dir) / "gym_run"
+        else:
+            work_dir = Path(tempfile.mkdtemp(prefix="gym_run_"))
+        work_dir.mkdir(parents=True, exist_ok=True)
+
+        rollouts_path = work_dir / "rollouts.jsonl"
+        _ensure_fresh_output(rollouts_path)
+
+        # Hand Gym a dataset we control: one row per requested task, each carrying the _ng_task_index
+        # we assign. Gym honors a pre-stamped index, so the rollout->task join becomes a total map
+        # instead of a positional guess about Gym's raw-line dedup (see the module docstring).
+        input_path = work_dir / "gym_input.jsonl"
+        index_to_task_id = _materialize_dataset(tasks, input_path)
+        logger.info(
+            "Materialized %d task(s) from %s into %s for Gym collection.",
+            len(index_to_task_id),
+            source_dataset,
+            input_path,
+        )
+
+        await self._run_two_step(input_path, rollouts_path, work_dir)
+        self._run_aggregations = _read_run_aggregations(rollouts_path)
+        trials = _trials_from_rollouts(rollouts_path, tasks, index_to_task_id, reward_key=cfg.reward_key)
+        _require_full_coverage(tasks, covered_task_ids={trial.task_id for trial in trials}, rollouts_path=rollouts_path)
+        return trials
+
+    async def _validate_config(
+        self, gym: str, selection: Sequence[str], subprocess_env: Mapping[str, str], work_dir: Path
+    ) -> None:
+        """Pre-flight the composed Gym config with ``gym env validate`` before starting anything.
+
+        Gym does not publish what configuration an environment requires — the typed
+        ``*ResourcesServerConfig`` classes cover behavioural knobs, while the model wiring lives in
+        each environment's YAML under names that vary per environment. ``gym env validate`` is the
+        only way to find out short of running: it merges configs, flags, and overrides, then reports
+        unresolved ``???`` values, bad paths, and cross-references to servers that are not defined —
+        without Ray and without starting a server.
+
+        Running it every time is worth the second it costs. A config mistake otherwise surfaces after
+        ``gym env start`` has brought up a Ray cluster and several uvicorn servers, as a readiness
+        timeout up to ``startup_timeout_s`` (240s by default) whose message says nothing about the
+        actual problem.
+        """
+        proc = await asyncio.create_subprocess_exec(
+            gym,
+            "env",
+            "validate",
+            *selection,
+            env=dict(subprocess_env),
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.STDOUT,
+            # Its own process group, like the other two Gym invocations. `_terminate` signals the
+            # group via `killpg`, so without this a validate timeout would signal *our* group — the
+            # SDK process included.
+            start_new_session=True,
+        )
+        try:
+            stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=_VALIDATE_TIMEOUT_S)
+        except TimeoutError:
+            await _terminate(proc, grace_s=self._config.shutdown_grace_s)
+            raise RuntimeError(
+                f"`gym env validate` did not finish within {_VALIDATE_TIMEOUT_S}s for resources-server "
+                f"{self._config.resources_server!r}"
+            ) from None
+
+        report = stdout.decode("utf-8", errors="replace").strip()
+        (work_dir / "gym_validate.log").write_text(report, encoding="utf-8")
+        if proc.returncode != 0:
+            raise RuntimeError(
+                f"Gym rejected the composed config for resources-server {self._config.resources_server!r} "
+                f"before any server started. `gym env validate` said:\n\n{report}"
+            )
+        logger.debug("gym env validate: %s", report)
+
+    async def _run_two_step(self, input_path: Path, output_path: Path, work_dir: Path) -> None:
+        """Start the Gym servers, collect against them with ``--no-serve``, then tear them down."""
+        cfg = self._config
+        gym = _gym_executable()
+        env_log = work_dir / _ENV_LOG_NAME
+
+        # Gym launches each server from its own subdir with its own .venv. Ray (>=2.56) otherwise
+        # detects a `uv run` ancestor and tries to replicate that uv project onto its workers,
+        # asserting the project pyproject.toml lives in the driver's cwd — which aborts startup. That
+        # hook is wrong for Gym (servers manage their own deps), so disable it for the subprocesses.
+        subprocess_env = {**os.environ, "RAY_ENABLE_UV_RUN_RUNTIME_ENV": "0"}
+
+        selection = _selection_args(cfg, work_dir)
+
+        await self._validate_config(gym, selection, subprocess_env, work_dir)
+
+        env_cmd = [gym, "env", "start", *selection]
+        # start_new_session=True puts `gym env start` in its own process group so teardown can signal
+        # the *whole* Ray-cluster + uvicorn tree, not just the direct child (else they orphan).
+        # stderr is merged into stdout here (unlike `eval run`, which splits them) because readiness
+        # detection scans a single chronological transcript and Gym's readiness line is not guaranteed
+        # to land on a particular stream.
+        env_proc = await asyncio.create_subprocess_exec(
+            *env_cmd,
+            env=subprocess_env,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.STDOUT,
+            start_new_session=True,
+        )
+        env_pump = asyncio.create_task(_pump_stream(env_proc.stdout, env_log, label="gym env start"))
+        try:
+            await self._wait_for_servers(env_log, env_proc)
+            await self._collect_rollouts(gym, input_path, output_path, work_dir, subprocess_env)
+        finally:
+            await _terminate(env_proc, grace_s=cfg.shutdown_grace_s)
+            # Bounded: a leaked grandchild holding the inherited pipe must not wedge teardown.
+            await _drain_pumps([env_pump], grace_s=cfg.shutdown_grace_s, what="gym env start")
+
+    async def _collect_rollouts(
+        self, gym: str, input_path: Path, output_path: Path, work_dir: Path, subprocess_env: dict[str, str]
+    ) -> None:
+        """Run ``gym eval run --no-serve`` against the live servers, bounded and self-cleaning.
+
+        Spawned in its own process group and always torn down (even on timeout/cancellation), so a hung
+        collection can't wedge the run or leak Ray workers.
+
+        Output is *streamed* to ``gym_eval.stdout.log`` / ``gym_eval.stderr.log`` in ``work_dir`` rather
+        than buffered in memory (a long collection's transcript can be large), and mirrored to this
+        module's logger at ``DEBUG`` so callers can surface it in their own terminal through ordinary
+        ``logging`` configuration. Failures point at both files and inline the last few lines.
+        """
+        cfg = self._config
+        eval_cmd = [
+            gym,
+            "eval",
+            "run",
+            "--no-serve",
+            "--agent",
+            cfg.agent,
+            "--input",
+            str(input_path),
+            "--output",
+            str(output_path),
+            "--num-repeats",
+            str(cfg.num_repeats),
+            "--concurrency",
+            str(cfg.concurrency),
+            # `gym eval run` is a Hydra app too, and unlike validate/start it does not go through
+            # _selection_args — so without this it writes `outputs/<date>/<time>/` into the caller's
+            # cwd on every single collection. Quoted for the same reason as there: a work dir
+            # containing `,` or `[` is otherwise misread as sweep syntax.
+            f"hydra.run.dir={_hydra_scalar(str(work_dir / _HYDRA_SUBDIR))}",
+        ]
+        stdout_log = work_dir / "gym_eval.stdout.log"
+        stderr_log = work_dir / "gym_eval.stderr.log"
+        eval_proc = await asyncio.create_subprocess_exec(
+            *eval_cmd,
+            env=subprocess_env,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+            start_new_session=True,
+        )
+        tails: dict[str, deque[str]] = {}
+        pumps = [
+            asyncio.create_task(
+                _pump_stream(eval_proc.stdout, stdout_log, label="gym eval", tails=tails, key="stdout")
+            ),
+            asyncio.create_task(
+                _pump_stream(eval_proc.stderr, stderr_log, label="gym eval", tails=tails, key="stderr")
+            ),
+        ]
+        try:
+            await asyncio.wait_for(eval_proc.wait(), timeout=cfg.collection_timeout_s)
+        except asyncio.TimeoutError as exc:
+            raise TimeoutError(
+                f"`gym eval run` exceeded collection_timeout_s={cfg.collection_timeout_s}s; collection aborted. "
+                f"See {stdout_log} and {stderr_log}."
+            ) from exc
+        finally:
+            # Terminate first so the pipes close, then drain (bounded) to flush whatever was buffered.
+            await _terminate(eval_proc, grace_s=cfg.shutdown_grace_s)
+            await _drain_pumps(pumps, grace_s=cfg.shutdown_grace_s, what="gym eval run")
+        if eval_proc.returncode != 0:
+            tail = "\n".join(tails.get("stderr") or tails.get("stdout") or [])
+            # A collection failure is frequently *server*-side: the resources-server raises, the
+            # agent surfaces it as a bare HTTP 500, and the traceback that explains it is only in
+            # gym_env.log. Naming the two eval logs alone sends the reader to the one place the
+            # cause is not. (Observed on wmt_translation: a 500 here, `PermissionError: /opt/Gym`
+            # there.)
+            env_log = work_dir / _ENV_LOG_NAME
+            raise RuntimeError(
+                f"`gym eval run` failed (rc={eval_proc.returncode}). Collection output: {stdout_log} "
+                f"and {stderr_log}\n"
+                f"If the tail below is an HTTP error from a Gym server, the cause is server-side: "
+                f"see {env_log}, whose lines are prefixed with the server that emitted them.\n"
+                f"--- last {_LOG_TAIL_LINES} line(s) ---\n{tail}"
+            )
+
+    async def _wait_for_servers(self, env_log: Path, env_proc: asyncio.subprocess.Process) -> None:
+        """Wait until ``gym env start`` reports every composed server ready.
+
+        Keys off Gym's own readiness line (``All N / N servers ready!``), which it prints only after
+        HTTP-health-checking each server it composed. So the caller never has to know or guess the
+        server count — Gym derives it from the config and health-checks for us.
+        """
+        cfg = self._config
+        loop = asyncio.get_running_loop()
+        ready = re.compile(r"All \d+ / \d+ servers ready")
+        deadline = loop.time() + cfg.startup_timeout_s
+        text = ""
+        while loop.time() < deadline:
+            if env_proc.returncode is not None:
+                raise RuntimeError(f"`gym env start` exited early (rc={env_proc.returncode}); see {env_log}")
+            text = env_log.read_text(encoding="utf-8", errors="replace") if env_log.exists() else ""
+            if "required uv version" in text.lower():
+                raise RuntimeError(f"`gym env start` hit the uv required-version gate; see {env_log}")
+            if ready.search(text):
+                return
+            await asyncio.sleep(2)
+        raise TimeoutError(self._startup_timeout_message(text, env_log))
+
+    def _startup_timeout_message(self, env_log_text: str, env_log: Path) -> str:
+        """Explain a startup timeout in terms of *which* server did not come up.
+
+        The bare "servers not ready" message sends you to a log that is mostly noise from the servers
+        that started fine. Gym polls and prints the outstanding set every time, so naming it costs
+        nothing and is usually the whole diagnosis: one server out of four means that environment's
+        own startup, not the platform.
+
+        The timeout is also a plausible *cause* rather than a symptom — a Docker-backed environment
+        builds its image on first run, which routinely exceeds the 240s default — so the message says
+        how to extend it instead of leaving the reader to find the knob.
+        """
+        cfg = self._config
+        pending = _pending_servers(env_log_text)
+        if pending is None:
+            detail = (
+                "Gym never reported server readiness, so it likely failed before starting them "
+                "(config composition or dependency installation)."
+            )
+        elif not pending[2]:
+            # Gym printed a readiness line but named no outstanding server. Saying "waiting on:
+            # unknown" would contradict the counts in the same sentence, so report only what is known.
+            ready_count, total, _ = pending
+            detail = (
+                f"{ready_count} of {total} server(s) started; Gym did not name which remained "
+                f"outstanding. The per-server output in {env_log.name} is prefixed with the server "
+                "that emitted each line."
+            )
+        else:
+            ready_count, total, names = pending
+            detail = (
+                f"{ready_count} of {total} server(s) started; still waiting on: {', '.join(names)}. "
+                f"Search {env_log.name} for '({names[0]})' — each line is prefixed with the server "
+                "that emitted it."
+            )
+        return (
+            f"Gym servers for resources-server {cfg.resources_server!r} were not ready within "
+            f"startup_timeout_s={cfg.startup_timeout_s}s. {detail}\n"
+            f"Full startup output: {env_log}\n"
+            "If the environment installs heavy dependencies or builds a container image on first run, "
+            "this timeout is expected on a cold cache — raise GymRuntimeConfig.startup_timeout_s and "
+            "re-run; the second run reuses what the first installed."
+        )
+
+
+#: Gym's per-poll readiness line, e.g.
+#: ``3 / 4 servers ready. Waiting for servers to spin up: ['legal_agent_bench']``.
+#: Gym knows exactly which servers are outstanding; without this we discard that and report only
+#: that *something* timed out.
diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym_runtime.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym_runtime.py
deleted file mode 100644
index af55a29af6..0000000000
--- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym_runtime.py
+++ /dev/null
@@ -1,1589 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""NeMo Gym-backed :class:`AgentTaskRunner` for the agent-eval pipeline.
-
-Runs an *existing* NeMo Gym environment through its ``gym`` CLI and adapts the
-rollout bundle into SDK :class:`AgentEvalTrial` objects, so an
-:class:`AgentEvaluator` can score and report Gym runs through the same seam as
-Harbor/Fabric/Model. Gym owns execution *and* scoring; this runtime imports an
-existing Gym environment as-is — Gym is treated as a self-scoring engine and the
-runtime only adapts its rollout bundle, it does not re-derive rewards.
-
-**Mapping** (Gym → Evaluator): one Gym dataset → one run; each distinct row →
-one :class:`AgentEvalTask` (id = content hash of the row); each attempt
-(``_ng_rollout_index``) → one :class:`AgentEvalTrial`; the per-attempt verifier
-``reward`` → a :class:`GymRewardMetric` score. ``num_repeats=R`` therefore yields
-up to R trials per task. Row duplication is *not* a way to ask for repeated
-attempts — ``num_repeats`` is (see :func:`discover_gym_tasks`).
-
-**Attribution** is by ``_ng_task_index``, which this runtime *assigns* rather
-than infers. Gym only auto-assigns an index when a row doesn't already carry one
-(``rollout_collection._preprocess_rows_from_config``), and its own fallback
-dedup keys off the **raw jsonl line text** — a rule we cannot reproduce from
-parsed rows. So instead of guessing, :meth:`GymAgentTaskRunner.run_tasks`
-materializes a normalized dataset (one line per requested task, ``_ng_task_index``
-stamped explicitly) and feeds *that* to Gym. Gym echoes the index back on every
-rollout record, giving a total, order-independent ``index → task`` map. This also
-means a caller can run a **subset** of tasks without Gym rolling out the rest.
-
-**Execution** is the two-step Gym flow (the one that reads a dataset directly
-without triggering Gym's split-driven data-prep), preceded by a pre-flight:
-``gym env validate`` merges the composed config and reports unset ``???`` values,
-bad paths, and dangling cross-references without starting anything; then ``gym env
-start`` brings up the resources-server + agent + model servers, and ``gym eval run
---no-serve --input <materialized dataset>`` collects rollouts against them. Both
-commands receive the identical selection arguments, so what is validated is what
-runs. The runtime shells out to the ``gym`` CLI on PATH, so this SDK never imports
-``nemo_gym``. Subprocess
-output is streamed to log files under the run's work dir *and* mirrored to this
-module's logger at ``DEBUG``, so callers choose terminal visibility through
-ordinary ``logging`` configuration.
-
-**Where Gym finds things.** NeMo Gym must be installed and its ``gym`` on PATH,
-along with the target environment's own dependencies. Generally that means a
-*separate* environment: Gym imports Ray at module load, and nemo-platform
-excludes Ray by constraint over an unfixed CVE, so the two cannot share one. In a
-job image the image owns PATH and this is unremarkable. There is deliberately no
-config field naming a checkout, a venv, or a search root — these runner configs
-become serialized job specs, and a local filesystem path means nothing on the
-other side of that boundary. Environments themselves ship in the ``nemo-gym``
-wheel (``resources_servers`` and friends install beside ``nemo_gym``, configs and
-example data included), so no checkout is needed to reach them.
-
-The subprocesses inherit this process's working directory, which is where Gym
-looks for the gitignored ``env.yaml`` holding the collector's credentials before
-falling back to its install root — so credentials never pass through this SDK.
-Run from the directory holding that file; a Gym checkout there also has its
-components take precedence, which is how you reach an environment the wheel does
-not carry.
-
-**Boundaries**: the caller is responsible for a
-Gym runtime whose deps are installed (each Gym env ships its own
-``requirements.txt``), and for handing a *ready-to-run* dataset file (``--no-serve
---input`` bypasses Gym's prompt-templating/materialization). Service-side
-provisioning (docker/k8s, Ray) is out of scope here — that is the plugin's job.
-
-A consequence of that bypass: an environment whose rows carry no rendered prompt
-(``responses_create_params.input == []``, the prompt supplied by data-prep or by the
-environment's own agent) is still supported — the row travels through this runtime intact
-and the task simply has no ``inputs['instruction']``. See :func:`discover_gym_tasks`.
-"""
-
-from __future__ import annotations
-
-import asyncio
-import hashlib
-import json
-import logging
-import math
-import os
-import re
-import shutil
-import signal
-import tempfile
-from collections import deque
-from collections.abc import Mapping, Sequence
-from pathlib import Path
-from typing import Any
-
-from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask
-from nemo_platform.beta.evaluator.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput, RunnerInfo
-from nemo_platform.beta.evaluator.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult
-from nemo_platform.beta.evaluator.values.evidence import CandidateEvidence, EvidenceDescriptor
-from nemo_platform.beta.evaluator.values.results import AggregateRangeScore, AggregateScalarScore, AggregateScore
-from pydantic import BaseModel, ConfigDict, Field
-
-logger = logging.getLogger(__name__)
-
-#: Reward key read from each Gym rollout record.
-DEFAULT_REWARD_KEY = "reward"
-#: Gym's CLI, expected on PATH. Not configurable: these runner configs become serialized job specs,
-#: and a path into somebody's venv is meaningless on the other side of that boundary. Note this name
-#: is only ever *resolved*, never executed: :func:`_gym_executable` turns it into an absolute path
-#: once, and that path is what the subprocesses run — so a child whose PATH differs from ours cannot
-#: end up executing a different Gym.
-_GYM_CLI = "gym"
-#: Bound on `gym env validate`. It merges config without starting anything and returns in about a
-#: second; this only exists so a wedged invocation cannot stall the run before it begins.
-_VALIDATE_TIMEOUT_S = 120.0
-#: Where Gym's Hydra run directories are redirected, relative to the run's work dir.
-_HYDRA_SUBDIR = "gym_hydra"
-#: `gym env start`'s combined output, under the run's work dir. Named here because a *collection*
-#: failure often has to point at it: the eval logs show the symptom, this shows the cause.
-_ENV_LOG_NAME = "gym_env.log"
-#: Token-count keys read to decide whether a model was called at all. Both vocabularies, because
-#: Gym's model servers report in either depending on the adapter: the Responses API spells them
-#: `input_tokens`/`output_tokens`, Chat Completions `prompt_tokens`/`completion_tokens`.
-_TOKEN_USAGE_KEYS = ("total_tokens", "input_tokens", "output_tokens", "prompt_tokens", "completion_tokens")
-#: The subset that proves a call happened. A model call always consumes input tokens, so zero on any
-#: of these is evidence the model was never reached; zero output tokens only means an empty answer.
-_INPUT_TOKEN_KEYS = ("total_tokens", "input_tokens", "prompt_tokens")
-#: Gym's index fields on each rollout record. ``_ng_task_index`` is the only join back to the input
-#: rows that survives a round-trip: Gym mutates ``responses_create_params`` (even the prompt) and
-#: copies only a fixed allowlist of row keys onto the result, so no field we invent comes back. Gym
-#: *honors* a caller-supplied ``_ng_task_index``, which is what makes the join here deterministic.
-NG_TASK_INDEX = "_ng_task_index"
-NG_ROLLOUT_INDEX = "_ng_rollout_index"
-#: Fields excluded from a row's content hash (runtime-injected, not task-defining).
-_RUNTIME_KEYS = frozenset({NG_TASK_INDEX, NG_ROLLOUT_INDEX})
-#: Lines of subprocess output retained in memory for inclusion in a failure message.
-_LOG_TAIL_LINES = 40
-
-
-#: Substrings that mark an override as carrying a credential. Matched case-insensitively against the
-#: full dotted path, so nesting cannot hide one behind an innocuous leaf name.
-_SECRET_KEY_MARKERS = ("api_key", "apikey", "token", "secret", "password", "passwd", "credential")
-#: Stand-in written in place of a redacted override value.
-_REDACTED = "<redacted>"
-
-
-#: Dict keys Hydra reads back unchanged. Its ``dictKey`` rule accepts no quoting, so a key is
-#: whatever the lexer makes of the bare text — this is deliberately narrower than what parses.
-_HYDRA_DICT_KEY = re.compile(r"[A-Za-z_][A-Za-z0-9_.-]*\Z")
-#: Bare words the lexer types rather than reading as text, so they cannot serve as string keys.
-_HYDRA_KEY_LITERALS = frozenset({"true", "false", "null", "inf", "nan"})
-
-
-def _hydra_dict(value: Mapping[str, Any]) -> str:
-    """Render a mapping as a Hydra dict container, ``{key:value,...}``.
-
-    Reached for a mapping nested inside a container — ``[{"b": 1}]`` — where there is no dotted path
-    to flatten onto, so the dict has to be spelled inline. Values recurse, so the typed spellings
-    below hold at any depth.
-
-    Keys are emitted bare, because Hydra's ``dictKey`` rule has no quoted form: ``{'b':1}`` does not
-    parse at all. That leaves the key at the mercy of the lexer, which types it — ``{true:1}`` keys
-    on the boolean ``True``, ``{1.5:1}`` on a float — and rejects ``:``, ``,``, brackets, and quotes
-    outright. Anything outside the conservative shape above therefore raises here rather than
-    silently keying the config on something the caller did not write.
-    """
-    rendered = []
-    for key, item in value.items():
-        if not isinstance(key, str) or not _HYDRA_DICT_KEY.match(key) or key.casefold() in _HYDRA_KEY_LITERALS:
-            raise ValueError(
-                f"Gym config override has dict key {key!r}, which Hydra's override grammar cannot "
-                "express as a string: keys are unquoted, so only a leading letter or underscore "
-                "followed by letters, digits, '_', '.', or '-' survives the round trip. Set this "
-                "key through the override path instead of nesting it inside a list."
-            )
-        rendered.append(f"{key}:{_hydra_scalar(item)}")
-    return "{" + ",".join(rendered) + "}"
-
-
-def _hydra_scalar(value: Any) -> str:
-    """Render a leaf value the way Hydra's override grammar reads it back.
-
-    Hydra's grammar is typed, so an unquoted string is not necessarily a string: ``true`` parses as a
-    boolean, ``null`` as ``None``, ``1.5`` as a float, ``a,b`` as a *sweep*, and ``A[B`` fails to
-    parse outright. Strings are therefore always single-quoted, which round-trips every one of those
-    (verified against ``hydra.core.override_parser``). Interpolations survive quoting — the override
-    sets the literal text and OmegaConf resolves it on read — so ``${policy_base_url}`` still works.
-
-    Only ``'`` is escaped. Hydra does **not** decode ``\\\\`` inside a quoted value: escaping
-    backslashes doubles them, so they are passed through raw.
-
-    ``None`` and booleans get their own spellings, since ``str()`` would emit ``"None"``/``"True"``
-    and Hydra reads those back as text. Containers recurse for the same reason: ``str()`` on a dict
-    emits Python's repr, whose quoted keys Hydra rejects outright.
-    """
-    if value is None:
-        return "null"
-    if isinstance(value, bool):
-        return "true" if value else "false"
-    if isinstance(value, Mapping):
-        return _hydra_dict(value)
-    if isinstance(value, (list, tuple)):
-        return "[" + ",".join(_hydra_scalar(item) for item in value) + "]"
-    if isinstance(value, str):
-        # A trailing backslash would escape the closing quote and leave the value unterminated, and
-        # there is no spelling that avoids it — better to say so than to emit something unparseable.
-        if value.endswith("\\"):
-            raise ValueError(
-                f"Gym config override value {value!r} ends with a backslash, which Hydra's override "
-                "grammar cannot express: it escapes the closing quote."
-            )
-        return "'" + value.replace("'", "\\'") + "'"
-    return str(value)
-
-
-def _flatten_overrides(overrides: Mapping[str, Any], _prefix: str = "") -> list[str]:
-    """Flatten a nested override mapping into Hydra ``++dotted.path=value`` arguments.
-
-    Callers describe overrides as structured data — ``{"a": {"b": 1}}`` — rather than as
-    pre-serialized Hydra strings, so the config survives being sent somewhere as JSON. Hydra itself
-    only speaks the flat form, so the translation happens here, at the point of invocation.
-
-    ``++`` rather than ``+``: it sets a key whether or not it already exists, which is what an
-    override means. A bare ``+`` fails on a key the merged config already defines.
-    """
-    arguments: list[str] = []
-    for key, value in overrides.items():
-        path = f"{_prefix}{key}"
-        # An empty mapping has no leaves to descend to, so recursing would drop the override
-        # entirely. It is still a value the caller asked to set: emit it as ``++path={}``, which
-        # clears the subtree.
-        if isinstance(value, Mapping) and value:
-            arguments.extend(_flatten_overrides(value, f"{path}."))
-        else:
-            arguments.append(f"++{path}={_hydra_scalar(value)}")
-    return arguments
-
-
-def _redact_env_overrides(overrides: Mapping[str, Any], _prefix: str = "") -> dict[str, Any]:
-    """Redact credential-looking values from overrides before they are recorded as provenance.
-
-    ``env_overrides`` is a free-form escape hatch forwarded to Gym, so nothing stops a caller passing
-    ``{"model": {"api_key": "sk-..."}}``. ``RunnerInfo.config`` is persisted into the run bundle, so a
-    value that looks like a credential must not be written there.
-
-    The *key* is always kept — knowing that a run overrode ``model.api_key`` is useful provenance;
-    knowing the value is a leak. Matching is on the full dotted path, so a marker anywhere in it
-    redacts, and nesting cannot hide a credential behind an innocuous leaf name.
-
-    Lists are walked too, since a mapping inside one — ``{"models": [{"api_key": "sk-..."}]}`` —
-    reaches Gym just as a nested mapping does. The index contributes no path segment: what marks a
-    value as a credential is the key it sits under, not where in a list it happens to fall.
-    """
-    redacted: dict[str, Any] = {}
-    for key, value in overrides.items():
-        path = f"{_prefix}{key}"
-        if isinstance(value, Mapping):
-            redacted[key] = _redact_env_overrides(value, f"{path}.")
-        elif any(marker in path.casefold() for marker in _SECRET_KEY_MARKERS):
-            redacted[key] = _REDACTED
-        elif isinstance(value, (list, tuple)):
-            redacted[key] = [_redact_list_item(item, path) for item in value]
-        else:
-            redacted[key] = value
-    return redacted
-
-
-def _redact_list_item(item: Any, path: str) -> Any:
-    """Redact inside one element of a list-valued override. See :func:`_redact_env_overrides`."""
-    if isinstance(item, Mapping):
-        return _redact_env_overrides(item, f"{path}.")
-    if isinstance(item, (list, tuple)):
-        return [_redact_list_item(nested, path) for nested in item]
-    return item
-
-
-def _selection_args(config: GymRuntimeConfig, work_dir: Path) -> list[str]:
-    """The environment/agent/model selection passed to Gym.
-
-    Built once and handed verbatim to both ``gym env validate`` and ``gym env start``, so what is
-    validated is exactly what runs — a pre-flight against a different config would be worse than
-    none.
-    """
-    selection = [
-        "--config",
-        config.agent_config,
-        "--model-type",
-        config.model_type,
-        "--resources-server",
-        config.resources_server,
-    ]
-    if config.bind_resources_server:
-        # Composable (Pattern-A) agents leave resources_server.name unbound ('???'); bind it to the
-        # env we're running. Assumes the agent config's top-level key equals the agent name (the
-        # simple_agent convention) *and* that the resources-server is registered under the
-        # environment's own name — not universally true, so self-contained or differently-named
-        # servers set bind_resources_server=False and bind themselves via env_overrides.
-        selection.append(
-            f"+{config.agent}.responses_api_agents.{config.agent}.resources_server.name={config.resources_server}"
-        )
-    selection.extend(_flatten_overrides(config.env_overrides))
-    # Gym is a Hydra app, so each invocation writes a timestamped run directory — by default
-    # `outputs/<date>/<time>/` under the *current* directory. Since the subprocesses inherit this
-    # process's cwd (so Gym can find env.yaml), the default would litter whatever directory the
-    # caller happened to run from. Redirect it under the run's work dir, with the rest of the run's
-    # artifacts. Applies to every Gym entry point, `gym list` included.
-    selection.append(f"hydra.run.dir={work_dir / _HYDRA_SUBDIR}")
-    return selection
-
-
-def _canonical_row_hash(row: Mapping[str, Any]) -> str:
-    """Stable ``sha256`` of a Gym dataset row, excluding runtime-injected fields.
-
-    A given row keeps its id across dataset revisions/reorderings (so Intake
-    rollups stay consistent), and a changed row becomes a new task. No
-    dataset-revision component — identity is the row content alone.
-    """
-    payload = {key: value for key, value in row.items() if key not in _RUNTIME_KEYS}
-    encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str)
-    return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
-
-
-def _content_text(content: Any) -> str:
-    """Flatten a message ``content`` (a string, or a list of ``{type,text}`` parts) to text.
-
-    Anything else contributes nothing. A missing ``content`` is unremarkable, but a *populated* value
-    in some other shape means we're silently dropping prompt text, so warn — :func:`_render_instruction`
-    only fails loudly when the whole row renders empty, which a single skipped message wouldn't trigger.
-    """
-    if isinstance(content, str):
-        return content
-    # `str` is itself a Sequence, so this branch only ever sees non-str sequences (handled above).
-    if isinstance(content, Sequence):
-        parts = [part["text"] for part in content if isinstance(part, Mapping) and isinstance(part.get("text"), str)]
-        return "\n".join(parts)
-    if content is not None:
-        logger.warning(
-            "Ignoring message content of unsupported type %s (expected a string or a list of {type,text} parts); "
-            "any prompt text it carries will be missing from the task instruction.",
-            type(content).__name__,
-        )
-    return ""
-
-
-def _render_instruction(responses_create_params: Mapping[str, Any]) -> str:
-    """Derive a task instruction from a row's ``responses_create_params``, or ``""`` when it carries none.
-
-    Uses ``instructions`` (system prompt, when present) + ``input`` (a plain string
-    or an OpenAI message list).
-
-    An empty result is *not* an error. A whole class of Gym environments ships
-    ``"responses_create_params": {"input": []}`` and keeps the task elsewhere — ``gdpval``
-    in a top-level ``prompt``, ``legal_agent_bench`` in an ``instance_id`` its own agent
-    resolves, ``aviary``/``toolsandbox`` in nothing but a ``task_idx``. Their prompt is
-    materialized by Gym's data-preparation step or by the environment's agent, neither of
-    which this runner drives (``--no-serve --input`` hands Gym a ready-to-run dataset by
-    design). Nothing here can reconstruct it, and no single row field generalizes across
-    them, so the honest answer is no instruction rather than a guess. See
-    :func:`discover_gym_tasks` for what that means downstream.
-    """
-    parts: list[str] = []
-    instructions = responses_create_params.get("instructions")
-    if isinstance(instructions, str) and instructions.strip():
-        parts.append(instructions.strip())
-    raw_input = responses_create_params.get("input")
-    if isinstance(raw_input, str):
-        parts.append(raw_input)
-    elif isinstance(raw_input, Sequence):
-        for item in raw_input:
-            if isinstance(item, Mapping):
-                text = _content_text(item.get("content"))
-                if text:
-                    parts.append(text)
-    return "\n\n".join(part for part in parts if part).strip()
-
-
-def _read_jsonl(path: str | Path, *, tolerant: bool = False) -> list[dict[str, Any]]:
-    """Read a jsonl file. With ``tolerant=True``, skip (and log) malformed lines instead of raising —
-
-    used for Gym's ``*_failures.jsonl`` sidecar, which is written during abnormal termination and can
-    end in a truncated line; a corrupt failure record must not sink the successfully-collected trials.
-    """
-    rows: list[dict[str, Any]] = []
-    with Path(path).open(encoding="utf-8") as handle:
-        for line_no, line in enumerate(handle, 1):
-            stripped = line.strip()
-            if not stripped:
-                continue
-            try:
-                rows.append(json.loads(stripped))
-            except json.JSONDecodeError:
-                if not tolerant:
-                    raise
-                logger.warning("Skipping malformed JSON at %s:%d", path, line_no)
-    return rows
-
-
-class GymRuntimeConfig(BaseModel):
-    """Declarative config for running an existing Gym environment via the ``gym`` CLI.
-
-    Holds only plain fields; the two-step invocation is built from them at run
-    time. The dataset itself is recovered from the tasks (stamped by
-    :func:`discover_gym_tasks`), mirroring the Harbor runner.
-    """
-
-    model_config = ConfigDict(extra="forbid")
-
-    agent: str = Field(description="Agent name to collect rollouts with, e.g. 'simple_agent'.")
-    agent_config: str = Field(description="Repo-relative agent config passed to `gym env start` (--config).")
-    resources_server: str = Field(description="Resources-server (environment) name, e.g. 'mcqa' (--resources-server).")
-    model_type: str = Field(
-        default="inference_provider",
-        description="Model-type config (--model-type). `inference_provider` speaks OpenAI-compatible chat; "
-        "`openai_model` uses the OpenAI Responses API (500s against chat-only endpoints).",
-    )
-    bind_resources_server: bool = Field(
-        default=True,
-        description="Auto-bind the agent's `resources_server.name` to `resources_server` via a Hydra override "
-        "(the composable/Pattern-A agent case, e.g. simple_agent whose config leaves it '???'). Set False for "
-        "self-contained agents that already bind their own resources-server.",
-    )
-    env_overrides: dict[str, Any] = Field(
-        default_factory=dict,
-        description="Nested config overrides merged into Gym's config, applied after the auto-derived "
-        "resources-server binding. Structured rather than pre-serialized Hydra strings so the config "
-        "travels as JSON — `{'a': {'b': 1}}` becomes `++a.b=1` at invocation. This is the escape "
-        "hatch for what Gym does not standardize: an environment whose resources-server is registered "
-        "under a different name, or which references a model server no shipped config defines.",
-    )
-    num_repeats: int = Field(default=1, ge=1, description="Attempts per row; each attempt becomes one trial.")
-    concurrency: int = Field(
-        default=4,
-        ge=1,
-        description="Concurrent rollouts for `gym eval run` (the collection-phase knob, tuned to the model "
-        "endpoint's limits). Distinct from AgentEvalRunConfig.parallelism, which bounds concurrent scoring.",
-    )
-    startup_timeout_s: float = Field(default=240.0, gt=0, description="Max wait for `gym env start` readiness.")
-    collection_timeout_s: float | None = Field(
-        default=None,
-        gt=0,
-        description="Max wait for `gym eval run` collection; None = unbounded (scales with dataset x num_repeats x "
-        "model latency, so no safe fixed default). Set it to bound a hung/slow model endpoint.",
-    )
-    shutdown_grace_s: float = Field(
-        default=30.0,
-        gt=0,
-        description="Grace period for a Gym subprocess *group* to exit on SIGTERM (letting Ray shut down cleanly) "
-        "before escalating to SIGKILL.",
-    )
-    reward_key: str = Field(default=DEFAULT_REWARD_KEY, description="Key read from each rollout record.")
-
-
-def _gym_executable() -> str:
-    """Locate the ``gym`` CLI on PATH, or fail saying what to do about it.
-
-    Deliberately PATH-only, with no config field pointing at a checkout or a particular venv: these
-    runner configs become serialized job specs, and a local filesystem path cannot cross that
-    boundary. Resolving here rather than at spawn time turns a missing Gym into one legible error
-    instead of an ``ENOENT`` out of ``create_subprocess_exec`` after the run has already started.
-
-    Note that Gym generally cannot live in this SDK's own environment: it imports Ray at module load,
-    and nemo-platform excludes Ray by constraint over an unfixed CVE. Install Gym separately and put
-    its ``bin`` on PATH; in a job image, the image owns PATH and this resolves normally.
-    """
-    resolved = shutil.which(_GYM_CLI)
-    if resolved is None:
-        raise RuntimeError(
-            f"The {_GYM_CLI!r} CLI was not found on PATH. Install NeMo Gym in its own environment "
-            "(it needs Ray, which nemo-platform excludes over an unfixed CVE, so it cannot share this "
-            "one) and put that environment's `bin` directory on PATH. Each resources-server also "
-            "ships its own requirements.txt, installed from that server's directory."
-        )
-    return resolved
-
-
-class GymRewardMetric:
-    """Score the Gym verifier reward stamped onto trial metadata.
-
-    The Gym analogue of :class:`HarborRewardMetric`: reads the per-trial ``reward``
-    off the candidate metadata (populated by :class:`GymAgentTaskRunner`); a trial
-    with no reward is left **unscored** (``None`` → ``nan``), excluded from the mean
-    and surfaced as ``nan_count`` rather than counted as a spurious ``0.0``. Gym owns
-    the scoring — this metric only surfaces it (Evaluator does not re-derive the reward).
-    """
-
-    def __init__(self, *, output_name: str = "reward", metric_type: str = "gym_reward") -> None:
-        self._output_name = output_name
-        self._metric_type = metric_type
-
-    @property
-    def type(self) -> str:
-        return self._metric_type
-
-    def output_spec(self) -> list[MetricOutputSpec]:
-        return [MetricOutputSpec.continuous_score(self._output_name)]
-
-    async def compute_scores(self, input: MetricInput) -> MetricResult:
-        reward = input.candidate.metadata.get("reward")
-        value = float(reward) if reward is not None else None
-        return MetricResult(outputs=[MetricOutput(name=self._output_name, value=value)])
-
-
-def discover_gym_tasks(dataset: str | Path, *, metrics: Sequence[Any] | None = None) -> list[AgentEvalTask]:
-    """Build one :class:`AgentEvalTask` per distinct row in a Gym dataset (jsonl).
-
-    Each task's id is the content hash of the row; its ``instruction`` is rendered
-    from ``responses_create_params.input`` *when the row carries one*; the raw params
-    are stashed under ``inputs['gym_row']`` for provenance; and it is scored by a
-    :class:`GymRewardMetric`. The dataset path is stamped on
-    ``metadata['gym_dataset_path']``, and every *other* row key (the verifier's
-    ground-truth fields, ``agent_ref``, and so on) on ``metadata['gym_row_extras']``.
-    Together with ``inputs['gym_row']`` those reconstruct the complete source row,
-    which :class:`GymAgentTaskRunner` re-materializes into the dataset it hands to Gym.
-    They are kept disjoint deliberately: the task record is persisted to the run
-    bundle's ``tasks.jsonl``, and ``responses_create_params`` is usually the bulk of a
-    row, so storing it in both places would write every dataset twice per run.
-
-    **One distinct row is one task.** Duplicate rows collapse (identity is row content,
-    so they are by definition the same task) and are reported as a warning: repeated
-    attempts are a run-level concern — ``GymRuntimeConfig.num_repeats`` — not something
-    a dataset expresses by repeating a row, so duplicates almost always mean bad data.
-
-    **A row need not carry a prompt.** A sweep of the 106 built-in environments shipping example
-    data found 5 — ``aviary``, ``gdpval``, ``legal_agent_bench``, ``scicode`` and ``toolsandbox``
-    — whose every row ships ``responses_create_params.input == []``, letting Gym's data-prep step
-    or the environment's own agent materialize the prompt instead (see
-    :func:`_render_instruction`). Such a row yields a task with **no** ``inputs['instruction']``
-    key rather than a fabricated one, and is counted in a summary log line. This costs the
-    Gym path nothing: the runner never reads ``instruction`` — it re-materializes the source
-    row from ``inputs['gym_row']`` + ``metadata['gym_row_extras']`` and hands *that* to Gym,
-    and ``intent`` is a dataset label either way. It stays absent (not ``""``) so that
-    :meth:`AgentEvalTask.agent_prompt` still fails loudly for the runners that *do* need a
-    prompt — an instruction-less task must not reach an agent as an empty one.
-
-    ``responses_create_params`` itself remains required: Gym indexes into it unconditionally
-    (``rollout_collection._preprocess_rows_from_config``), so a row without it is rejected here
-    rather than crashing Gym mid-collection.
-    """
-    dataset = Path(dataset)
-    tasks: list[AgentEvalTask] = []
-    seen: set[str] = set()
-    duplicates = 0
-    promptless = 0
-    for position, row in enumerate(_read_jsonl(dataset), 1):
-        params = row.get("responses_create_params")
-        if not isinstance(params, Mapping):
-            raise ValueError(
-                f"row {position} of {dataset} has no 'responses_create_params' mapping; Gym requires that key "
-                "on every dataset row"
-            )
-        task_id = _canonical_row_hash(row)
-        if task_id in seen:
-            duplicates += 1
-            continue
-        seen.add(task_id)
-        instruction = _render_instruction(params)
-        if not instruction:
-            promptless += 1
-        tasks.append(
-            AgentEvalTask(
-                id=task_id,
-                intent=f"Gym row from {dataset.name}",
-                inputs={
-                    # Absent, not empty, when the row carries no prompt — see the docstring.
-                    **({"instruction": instruction} if instruction else {}),
-                    "gym_row": params,
-                },
-                metrics=list(metrics) if metrics is not None else [GymRewardMetric()],
-                metadata={
-                    "gym_dataset_path": str(dataset),
-                    # Everything except responses_create_params, which already lives in inputs['gym_row'].
-                    "gym_row_extras": {
-                        key: value
-                        for key, value in row.items()
-                        if key != "responses_create_params" and key not in _RUNTIME_KEYS
-                    },
-                },
-            )
-        )
-    if duplicates:
-        logger.warning(
-            "Collapsed %d duplicate row(s) in %s into %d distinct task(s): task identity is row content, so "
-            "repeating a row cannot request repeated attempts — set GymRuntimeConfig.num_repeats for that. "
-            "Duplicate rows usually indicate a data problem.",
-            duplicates,
-            dataset,
-            len(tasks),
-        )
-    if promptless:
-        logger.info(
-            "%d of %d task(s) in %s carry no prompt in responses_create_params and were discovered without an "
-            "inputs['instruction']. Expected for environments whose prompt is built by Gym's data-prep step or by "
-            "their own agent (e.g. aviary, gdpval, legal_agent_bench, scicode, toolsandbox); the Gym runner does "
-            "not read the instruction. A runner that hands the task straight to an agent will reject these tasks.",
-            promptless,
-            len(tasks),
-            dataset,
-        )
-    if not tasks:
-        raise ValueError(f"no rows found in Gym dataset {dataset}")
-    return tasks
-
-
-class GymAgentTaskRunner:
-    """An :class:`AgentTaskRunner` that runs an existing Gym env, then adapts its rollouts.
-
-    :meth:`run_tasks` materializes the requested tasks into a normalized dataset with
-    ``_ng_task_index`` stamped per row, runs the two-step Gym flow (``env start`` +
-    ``eval run --no-serve --input``) against it via subprocess, and fans the rollout
-    records out into one trial per ``_ng_rollout_index``. Because we assign the index,
-    attribution is a dict lookup rather than a positional guess, and Gym only rolls out
-    the tasks we asked for. The durable task **identity** stays our content hash.
-    """
-
-    def __init__(self, *, config: GymRuntimeConfig) -> None:
-        self._config = config
-        self._run_aggregations: dict[str, Any] | None = None
-
-    def run_aggregate_scores(self) -> Sequence[AggregateScore]:
-        """Gym's ``agent_metrics`` mapped onto typed aggregate scores, namespaced ``runner.gym.<metric>``.
-
-        Satisfies :class:`RunAggregationsProvider`. ``reward`` is skipped: the SDK already scores it
-        natively as ``gym_reward.reward``, and two differently-derived numbers under one name invites
-        exactly the confusion the namespace is there to prevent.
-        """
-        return _aggregate_scores_from_gym(self._run_aggregations)
-
-    def runner_info(self) -> RunnerInfo:
-        """Identify this runner and the Gym settings that shape its results.
-
-        Credentials normally live in the Gym checkout's gitignored ``env.yaml`` and never reach this
-        object — but ``env_overrides`` is a free-form escape hatch, so its values are redacted by key
-        (see :func:`_redact_env_overrides`) rather than trusted.
-        """
-        cfg = self._config
-        return RunnerInfo(
-            name="gym",
-            kind="runner",
-            config={
-                "resources_server": cfg.resources_server,
-                "agent": cfg.agent,
-                "agent_config": cfg.agent_config,
-                "model_type": cfg.model_type,
-                "num_repeats": cfg.num_repeats,
-                "concurrency": cfg.concurrency,
-                "bind_resources_server": cfg.bind_resources_server,
-                "env_overrides": _redact_env_overrides(cfg.env_overrides),
-                "reward_key": cfg.reward_key,
-            },
-        )
-
-    async def run_tasks(
-        self,
-        tasks: Sequence[AgentEvalTask],
-        config: AgentEvalRunConfig | None = None,
-    ) -> list[AgentEvalTrial]:
-        cfg = self._config
-        self._run_aggregations = None  # reset per run so a reused runner never leaks a prior run's numbers
-        # Provenance for the log line only — the file Gym actually reads is the normalized one we
-        # materialize below from the tasks themselves.
-        source_dataset = _source_datasets(tasks)
-
-        # config.parallelism still governs how the evaluator *scores* the trials we return (its scoring
-        # semaphore). It is deliberately not mapped onto Gym's rollout `--concurrency`: those are different
-        # phases — parallelism bounds concurrent scoring (SDK-side, cheap), while Gym's `--concurrency`
-        # bounds concurrent rollouts against the model endpoint during collection (tuned to that endpoint's
-        # limits via GymRuntimeConfig.concurrency).
-        if config is not None and config.work_dir is not None:
-            work_dir = Path(config.work_dir) / "gym_run"
-        else:
-            work_dir = Path(tempfile.mkdtemp(prefix="gym_run_"))
-        work_dir.mkdir(parents=True, exist_ok=True)
-
-        rollouts_path = work_dir / "rollouts.jsonl"
-        _ensure_fresh_output(rollouts_path)
-
-        # Hand Gym a dataset we control: one row per requested task, each carrying the _ng_task_index
-        # we assign. Gym honors a pre-stamped index, so the rollout->task join becomes a total map
-        # instead of a positional guess about Gym's raw-line dedup (see the module docstring).
-        input_path = work_dir / "gym_input.jsonl"
-        index_to_task_id = _materialize_dataset(tasks, input_path)
-        logger.info(
-            "Materialized %d task(s) from %s into %s for Gym collection.",
-            len(index_to_task_id),
-            source_dataset,
-            input_path,
-        )
-
-        await self._run_two_step(input_path, rollouts_path, work_dir)
-        self._run_aggregations = _read_run_aggregations(rollouts_path)
-        trials = _trials_from_rollouts(rollouts_path, tasks, index_to_task_id, reward_key=cfg.reward_key)
-        _require_full_coverage(tasks, covered_task_ids={trial.task_id for trial in trials}, rollouts_path=rollouts_path)
-        return trials
-
-    async def _validate_config(
-        self, gym: str, selection: Sequence[str], subprocess_env: Mapping[str, str], work_dir: Path
-    ) -> None:
-        """Pre-flight the composed Gym config with ``gym env validate`` before starting anything.
-
-        Gym does not publish what configuration an environment requires — the typed
-        ``*ResourcesServerConfig`` classes cover behavioural knobs, while the model wiring lives in
-        each environment's YAML under names that vary per environment. ``gym env validate`` is the
-        only way to find out short of running: it merges configs, flags, and overrides, then reports
-        unresolved ``???`` values, bad paths, and cross-references to servers that are not defined —
-        without Ray and without starting a server.
-
-        Running it every time is worth the second it costs. A config mistake otherwise surfaces after
-        ``gym env start`` has brought up a Ray cluster and several uvicorn servers, as a readiness
-        timeout up to ``startup_timeout_s`` (240s by default) whose message says nothing about the
-        actual problem.
-        """
-        proc = await asyncio.create_subprocess_exec(
-            gym,
-            "env",
-            "validate",
-            *selection,
-            env=dict(subprocess_env),
-            stdout=asyncio.subprocess.PIPE,
-            stderr=asyncio.subprocess.STDOUT,
-            # Its own process group, like the other two Gym invocations. `_terminate` signals the
-            # group via `killpg`, so without this a validate timeout would signal *our* group — the
-            # SDK process included.
-            start_new_session=True,
-        )
-        try:
-            stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=_VALIDATE_TIMEOUT_S)
-        except TimeoutError:
-            await _terminate(proc, grace_s=self._config.shutdown_grace_s)
-            raise RuntimeError(
-                f"`gym env validate` did not finish within {_VALIDATE_TIMEOUT_S}s for resources-server "
-                f"{self._config.resources_server!r}"
-            ) from None
-
-        report = stdout.decode("utf-8", errors="replace").strip()
-        (work_dir / "gym_validate.log").write_text(report, encoding="utf-8")
-        if proc.returncode != 0:
-            raise RuntimeError(
-                f"Gym rejected the composed config for resources-server {self._config.resources_server!r} "
-                f"before any server started. `gym env validate` said:\n\n{report}"
-            )
-        logger.debug("gym env validate: %s", report)
-
-    async def _run_two_step(self, input_path: Path, output_path: Path, work_dir: Path) -> None:
-        """Start the Gym servers, collect against them with ``--no-serve``, then tear them down."""
-        cfg = self._config
-        gym = _gym_executable()
-        env_log = work_dir / _ENV_LOG_NAME
-
-        # Gym launches each server from its own subdir with its own .venv. Ray (>=2.56) otherwise
-        # detects a `uv run` ancestor and tries to replicate that uv project onto its workers,
-        # asserting the project pyproject.toml lives in the driver's cwd — which aborts startup. That
-        # hook is wrong for Gym (servers manage their own deps), so disable it for the subprocesses.
-        subprocess_env = {**os.environ, "RAY_ENABLE_UV_RUN_RUNTIME_ENV": "0"}
-
-        selection = _selection_args(cfg, work_dir)
-
-        await self._validate_config(gym, selection, subprocess_env, work_dir)
-
-        env_cmd = [gym, "env", "start", *selection]
-        # start_new_session=True puts `gym env start` in its own process group so teardown can signal
-        # the *whole* Ray-cluster + uvicorn tree, not just the direct child (else they orphan).
-        # stderr is merged into stdout here (unlike `eval run`, which splits them) because readiness
-        # detection scans a single chronological transcript and Gym's readiness line is not guaranteed
-        # to land on a particular stream.
-        env_proc = await asyncio.create_subprocess_exec(
-            *env_cmd,
-            env=subprocess_env,
-            stdout=asyncio.subprocess.PIPE,
-            stderr=asyncio.subprocess.STDOUT,
-            start_new_session=True,
-        )
-        env_pump = asyncio.create_task(_pump_stream(env_proc.stdout, env_log, label="gym env start"))
-        try:
-            await self._wait_for_servers(env_log, env_proc)
-            await self._collect_rollouts(gym, input_path, output_path, work_dir, subprocess_env)
-        finally:
-            await _terminate(env_proc, grace_s=cfg.shutdown_grace_s)
-            # Bounded: a leaked grandchild holding the inherited pipe must not wedge teardown.
-            await _drain_pumps([env_pump], grace_s=cfg.shutdown_grace_s, what="gym env start")
-
-    async def _collect_rollouts(
-        self, gym: str, input_path: Path, output_path: Path, work_dir: Path, subprocess_env: dict[str, str]
-    ) -> None:
-        """Run ``gym eval run --no-serve`` against the live servers, bounded and self-cleaning.
-
-        Spawned in its own process group and always torn down (even on timeout/cancellation), so a hung
-        collection can't wedge the run or leak Ray workers.
-
-        Output is *streamed* to ``gym_eval.stdout.log`` / ``gym_eval.stderr.log`` in ``work_dir`` rather
-        than buffered in memory (a long collection's transcript can be large), and mirrored to this
-        module's logger at ``DEBUG`` so callers can surface it in their own terminal through ordinary
-        ``logging`` configuration. Failures point at both files and inline the last few lines.
-        """
-        cfg = self._config
-        eval_cmd = [
-            gym,
-            "eval",
-            "run",
-            "--no-serve",
-            "--agent",
-            cfg.agent,
-            "--input",
-            str(input_path),
-            "--output",
-            str(output_path),
-            "--num-repeats",
-            str(cfg.num_repeats),
-            "--concurrency",
-            str(cfg.concurrency),
-            # `gym eval run` is a Hydra app too, and unlike validate/start it does not go through
-            # _selection_args — so without this it writes `outputs/<date>/<time>/` into the caller's
-            # cwd on every single collection.
-            f"hydra.run.dir={work_dir / _HYDRA_SUBDIR}",
-        ]
-        stdout_log = work_dir / "gym_eval.stdout.log"
-        stderr_log = work_dir / "gym_eval.stderr.log"
-        eval_proc = await asyncio.create_subprocess_exec(
-            *eval_cmd,
-            env=subprocess_env,
-            stdout=asyncio.subprocess.PIPE,
-            stderr=asyncio.subprocess.PIPE,
-            start_new_session=True,
-        )
-        tails: dict[str, deque[str]] = {}
-        pumps = [
-            asyncio.create_task(
-                _pump_stream(eval_proc.stdout, stdout_log, label="gym eval", tails=tails, key="stdout")
-            ),
-            asyncio.create_task(
-                _pump_stream(eval_proc.stderr, stderr_log, label="gym eval", tails=tails, key="stderr")
-            ),
-        ]
-        try:
-            await asyncio.wait_for(eval_proc.wait(), timeout=cfg.collection_timeout_s)
-        except asyncio.TimeoutError as exc:
-            raise TimeoutError(
-                f"`gym eval run` exceeded collection_timeout_s={cfg.collection_timeout_s}s; collection aborted. "
-                f"See {stdout_log} and {stderr_log}."
-            ) from exc
-        finally:
-            # Terminate first so the pipes close, then drain (bounded) to flush whatever was buffered.
-            await _terminate(eval_proc, grace_s=cfg.shutdown_grace_s)
-            await _drain_pumps(pumps, grace_s=cfg.shutdown_grace_s, what="gym eval run")
-        if eval_proc.returncode != 0:
-            tail = "\n".join(tails.get("stderr") or tails.get("stdout") or [])
-            # A collection failure is frequently *server*-side: the resources-server raises, the
-            # agent surfaces it as a bare HTTP 500, and the traceback that explains it is only in
-            # gym_env.log. Naming the two eval logs alone sends the reader to the one place the
-            # cause is not. (Observed on wmt_translation: a 500 here, `PermissionError: /opt/Gym`
-            # there.)
-            env_log = work_dir / _ENV_LOG_NAME
-            raise RuntimeError(
-                f"`gym eval run` failed (rc={eval_proc.returncode}). Collection output: {stdout_log} "
-                f"and {stderr_log}\n"
-                f"If the tail below is an HTTP error from a Gym server, the cause is server-side: "
-                f"see {env_log}, whose lines are prefixed with the server that emitted them.\n"
-                f"--- last {_LOG_TAIL_LINES} line(s) ---\n{tail}"
-            )
-
-    async def _wait_for_servers(self, env_log: Path, env_proc: asyncio.subprocess.Process) -> None:
-        """Wait until ``gym env start`` reports every composed server ready.
-
-        Keys off Gym's own readiness line (``All N / N servers ready!``), which it prints only after
-        HTTP-health-checking each server it composed. So the caller never has to know or guess the
-        server count — Gym derives it from the config and health-checks for us.
-        """
-        cfg = self._config
-        loop = asyncio.get_running_loop()
-        ready = re.compile(r"All \d+ / \d+ servers ready")
-        deadline = loop.time() + cfg.startup_timeout_s
-        text = ""
-        while loop.time() < deadline:
-            if env_proc.returncode is not None:
-                raise RuntimeError(f"`gym env start` exited early (rc={env_proc.returncode}); see {env_log}")
-            text = env_log.read_text(encoding="utf-8", errors="replace") if env_log.exists() else ""
-            if "required uv version" in text.lower():
-                raise RuntimeError(f"`gym env start` hit the uv required-version gate; see {env_log}")
-            if ready.search(text):
-                return
-            await asyncio.sleep(2)
-        raise TimeoutError(self._startup_timeout_message(text, env_log))
-
-    def _startup_timeout_message(self, env_log_text: str, env_log: Path) -> str:
-        """Explain a startup timeout in terms of *which* server did not come up.
-
-        The bare "servers not ready" message sends you to a log that is mostly noise from the servers
-        that started fine. Gym polls and prints the outstanding set every time, so naming it costs
-        nothing and is usually the whole diagnosis: one server out of four means that environment's
-        own startup, not the platform.
-
-        The timeout is also a plausible *cause* rather than a symptom — a Docker-backed environment
-        builds its image on first run, which routinely exceeds the 240s default — so the message says
-        how to extend it instead of leaving the reader to find the knob.
-        """
-        cfg = self._config
-        pending = _pending_servers(env_log_text)
-        if pending is None:
-            detail = (
-                "Gym never reported server readiness, so it likely failed before starting them "
-                "(config composition or dependency installation)."
-            )
-        elif not pending[2]:
-            # Gym printed a readiness line but named no outstanding server. Saying "waiting on:
-            # unknown" would contradict the counts in the same sentence, so report only what is known.
-            ready_count, total, _ = pending
-            detail = (
-                f"{ready_count} of {total} server(s) started; Gym did not name which remained "
-                f"outstanding. The per-server output in {env_log.name} is prefixed with the server "
-                "that emitted each line."
-            )
-        else:
-            ready_count, total, names = pending
-            detail = (
-                f"{ready_count} of {total} server(s) started; still waiting on: {', '.join(names)}. "
-                f"Search {env_log.name} for '({names[0]})' — each line is prefixed with the server "
-                "that emitted it."
-            )
-        return (
-            f"Gym servers for resources-server {cfg.resources_server!r} were not ready within "
-            f"startup_timeout_s={cfg.startup_timeout_s}s. {detail}\n"
-            f"Full startup output: {env_log}\n"
-            "If the environment installs heavy dependencies or builds a container image on first run, "
-            "this timeout is expected on a cold cache — raise GymRuntimeConfig.startup_timeout_s and "
-            "re-run; the second run reuses what the first installed."
-        )
-
-
-#: Gym's per-poll readiness line, e.g.
-#: ``3 / 4 servers ready. Waiting for servers to spin up: ['legal_agent_bench']``.
-#: Gym knows exactly which servers are outstanding; without this we discard that and report only
-#: that *something* timed out.
-_PENDING_SERVERS_RE = re.compile(r"(\d+)\s*/\s*(\d+) servers ready\. Waiting for servers to spin up: \[([^\]]*)\]")
-
-
-def _pending_servers(text: str) -> tuple[int, int, tuple[str, ...]] | None:
-    """Recover ``(ready, total, still-pending)`` from the *last* readiness line Gym logged.
-
-    The last line is the one that matters: earlier polls list servers that have since come up.
-    Returns ``None`` when Gym never got as far as printing one (it died during composition, say),
-    which the caller must treat as "no detail available" rather than "nothing pending".
-
-    Never raises — this only ever decorates an error that is already being reported, and a parsing
-    failure here must not replace the real one. Gym's log format is not contractual.
-    """
-    matches = _PENDING_SERVERS_RE.findall(text)
-    if not matches:
-        return None
-    ready, total, names = matches[-1]
-    try:
-        return int(ready), int(total), tuple(re.findall(r"['\"]([^'\"]+)['\"]", names))
-    except ValueError:  # pragma: no cover - the regex only matches digits
-        return None
-
-
-def _agent_never_ran(record: Mapping[str, Any]) -> bool:
-    """True when a result record shows the agent produced nothing *and* never called the model.
-
-    A verifier-scored runner reports whatever score the environment computed, so an agent that
-    silently failed to start is indistinguishable from one that tried and scored 0.0 — the record
-    looks normal, no failure is reported alongside it, and the run succeeds. Observed on
-    ``legal_agent_bench``, whose agent died locating its task tree and returned an empty response:
-    two "completed" trials scoring 0.0, no model call, no failure recorded.
-
-    Deliberately conservative — **both** conditions must hold:
-
-    * no output, and
-    * a stated zero on the input side of token usage.
-
-    An empty answer alone is a legitimate result (a model can return nothing and earn 0.0), so it is
-    not sufficient. Zero *input* tokens is what says the model was never called: a call always sends
-    a prompt, whereas zero output tokens only describes an empty answer. A record carrying no
-    ``response`` key at all is left alone — some runners populate it differently, and this must not
-    reclassify trials it does not understand.
-    """
-    response = record.get("response")
-    if not isinstance(response, Mapping):
-        return False
-    if "output" not in response:
-        # Nothing to judge: some runners shape `response` differently, and absence is not emptiness.
-        return False
-    # Any non-empty value counts as output, whatever its shape. Checking `len(...) > 0` on a list
-    # alone would treat a plain string answer — or a single mapping — as "produced nothing" and fail
-    # a trial that ran perfectly well.
-    if response["output"]:
-        return False
-    usage = response.get("usage")
-    if not isinstance(usage, Mapping):
-        # No usage block to corroborate with; an empty output on its own is not enough to judge.
-        return False
-    # Zero usage has to be *stated*, not inferred from a key being absent. Two vocabularies are in
-    # play — the Responses API's `input_tokens`/`output_tokens` and Chat Completions'
-    # `prompt_tokens`/`completion_tokens` — and Gym's model servers speak both depending on the
-    # adapter. Reading only one set would make a real call reported in the other read as zero.
-    reported = [usage[key] for key in _TOKEN_USAGE_KEYS if isinstance(usage.get(key), (int, float))]
-    if any(value > 0 for value in reported):
-        return False
-    # The discriminating evidence is *input*-side, and only input-side. Calling a model always sends
-    # a prompt, so zero input tokens cannot describe a call that happened; zero **output** tokens
-    # merely describes an empty answer, which is a legitimate result. Requiring an input-side count
-    # rules out a whole class of false positives — a provider reporting only `output_tokens: 0`, or
-    # only `completion_tokens: 0` — rather than excluding those shapes one at a time.
-    input_side = [usage[key] for key in _INPUT_TOKEN_KEYS if isinstance(usage.get(key), (int, float))]
-    return bool(input_side) and all(value == 0 for value in input_side)
-
-
-def _source_datasets(tasks: Sequence[AgentEvalTask]) -> str:
-    """Human-readable summary of the dataset(s) the tasks were discovered from, for a log line.
-
-    Purely provenance. Gym reads the dataset :func:`_materialize_dataset` writes from the tasks
-    themselves, so nothing here gates the run: tasks drawn from two datasets are legitimate (the
-    materialized file is their union), and a task with no stamped path runs fine. Never raises —
-    a missing or inconsistent label is not a reason to fail an otherwise valid evaluation.
-    """
-    stamped = {
-        task.metadata["gym_dataset_path"]
-        for task in tasks
-        if isinstance(task.metadata.get("gym_dataset_path"), str) and task.metadata["gym_dataset_path"]
-    }
-    return ", ".join(sorted(stamped)) if stamped else "<tasks with no stamped dataset path>"
-
-
-async def _pump_stream(
-    stream: asyncio.StreamReader | None,
-    path: Path,
-    *,
-    label: str,
-    tails: dict[str, deque[str]] | None = None,
-    key: str = "stdout",
-) -> None:
-    """Stream a subprocess pipe to ``path`` while mirroring it to the module logger at ``DEBUG``.
-
-    Reads in chunks rather than by line so a pathologically long line can't overrun asyncio's stream
-    limit, and retains only the last :data:`_LOG_TAIL_LINES` lines in memory (via ``tails[key]``) for
-    inclusion in a failure message — the file on disk is the complete record.
-    """
-    tail: deque[str] = deque(maxlen=_LOG_TAIL_LINES)
-    if tails is not None:
-        tails[key] = tail
-    if stream is None:
-        return
-
-    def emit(raw: bytes) -> None:
-        text = raw.decode("utf-8", errors="replace").rstrip("\r")
-        logger.debug("[%s %s] %s", label, key, text)
-        tail.append(text)
-
-    with path.open("wb") as handle:
-        buffer = b""
-        while True:
-            chunk = await stream.read(65536)
-            if not chunk:
-                break
-            handle.write(chunk)
-            handle.flush()
-            buffer += chunk
-            *complete, buffer = buffer.split(b"\n")
-            for line in complete:
-                emit(line)
-        if buffer:
-            emit(buffer)
-
-
-async def _drain_pumps(pumps: Sequence[asyncio.Task[None]], *, grace_s: float, what: str) -> None:
-    """Await log pumps after teardown, but never block on them indefinitely.
-
-    A pump ends at pipe EOF, which requires *every* inheritor of the write end to close it. Gym's
-    descendants inherit it, and Ray daemonizes ``gcs_server`` into its own session where our
-    process-group signals can't reach it (see :func:`_terminate`) — so a leaked grandchild can hold
-    the pipe open forever. Waiting unconditionally would hang the whole run inside a ``finally``.
-
-    Bounded instead: give the pumps ``grace_s`` to flush, then cancel and move on. Everything read
-    before the stall is already written and flushed to the log file, so cancelling costs at most the
-    tail of a transcript belonging to a process we just killed — and it is reported, not silent.
-    """
-    if not pumps:
-        return
-    _done, pending = await asyncio.wait(pumps, timeout=grace_s)
-    if not pending:
-        return
-    logger.warning(
-        "%d `%s` log pump(s) did not finish within %.1fs; the pipe is still held open (likely a Gym "
-        "grandchild that outlived teardown, e.g. Ray's detached gcs_server). Abandoning them — the log "
-        "file holds everything read up to this point, but its tail may be truncated.",
-        len(pending),
-        what,
-        grace_s,
-    )
-    for task in pending:
-        task.cancel()
-    await asyncio.gather(*pending, return_exceptions=True)
-
-
-def _materialize_dataset(tasks: Sequence[AgentEvalTask], dest: Path) -> dict[int, str]:
-    """Write the normalized dataset Gym will read, and return its ``_ng_task_index`` → task-id map.
-
-    One line per requested task, in task order, carrying the task's full source row plus an explicitly
-    stamped ``_ng_task_index``. Gym honors a pre-supplied index instead of deriving one, so this makes
-    the rollout→task join total and order-independent, and confines the run to the tasks we asked for.
-
-    The row is reassembled from ``inputs['gym_row']`` (``responses_create_params``) and
-    ``metadata['gym_row_extras']`` (everything else), which :func:`discover_gym_tasks` keeps disjoint
-    so the run bundle doesn't store the same payload twice. Any pre-existing ``_ng_*`` fields are
-    stripped: ours is authoritative, and Gym assigns ``_ng_rollout_index`` itself per attempt.
-    """
-    index_to_task_id: dict[int, str] = {}
-    seen_task_ids: set[str] = set()
-    lines: list[str] = []
-    for index, task in enumerate(tasks):
-        # The source row is stored split across inputs/metadata so the run bundle doesn't persist
-        # responses_create_params twice; reassemble it here.
-        extras = task.metadata.get("gym_row_extras")
-        params = task.inputs.get("gym_row")
-        if not isinstance(extras, Mapping) or not isinstance(params, Mapping):
-            raise ValueError(
-                f"task {task.id!r} is missing inputs['gym_row'] and/or metadata['gym_row_extras']; build tasks "
-                "with discover_gym_tasks so the Gym dataset can be re-materialized for the run"
-            )
-        if task.id in seen_task_ids:
-            raise ValueError(
-                f"task {task.id!r} was supplied more than once; one distinct row is one task, and repeated "
-                "attempts come from GymRuntimeConfig.num_repeats"
-            )
-        seen_task_ids.add(task.id)
-        payload = {key: value for key, value in extras.items() if key not in _RUNTIME_KEYS}
-        payload["responses_create_params"] = params
-        payload[NG_TASK_INDEX] = index
-        lines.append(json.dumps(payload, default=str))
-        index_to_task_id[index] = task.id
-    if not index_to_task_id:
-        raise ValueError("GymAgentTaskRunner was given no tasks to run")
-    dest.write_text("\n".join(lines) + "\n", encoding="utf-8")
-    return index_to_task_id
-
-
-def _require_full_coverage(tasks: Sequence[AgentEvalTask], *, covered_task_ids: set[str], rollouts_path: Path) -> None:
-    """Fail the run when Gym produced no trial at all for some requested task.
-
-    :class:`AgentEvaluator` already refuses to score a task with no trial
-    (``evaluator._score_trials``), and deliberately so: an incomplete run must not read as a
-    successful one with quieter counts. We keep that contract rather than papering over it with
-    synthesized FAILED trials — an unscored trial is excluded from the mean, so fabricating them
-    would report a plausible-looking average over an unannounced subset of the dataset. A rollout
-    that failed in a *diagnosable* way already becomes a FAILED trial via the failures sidecar;
-    reaching here means the attempt is unaccounted for entirely.
-
-    What this adds over letting the evaluator raise is diagnosis: by the time the run fails the full
-    collection cost is already paid, so the error names the evidence instead of just the task ids.
-    """
-    missing = sorted({task.id for task in tasks} - covered_task_ids)
-    if not missing:
-        return
-    raise RuntimeError(
-        f"Gym produced no rollout for {len(missing)} of {len(tasks)} requested task(s), so the run is "
-        f"incomplete and will not be scored (a partial run must not be reported as a whole one).\n"
-        f"  rollouts:  {rollouts_path}\n"
-        f"  failures:  {_failures_path_for(rollouts_path)}\n"
-        f"  gym logs:  {rollouts_path.parent} (gym_env.log, gym_eval.stdout.log, gym_eval.stderr.log)\n"
-        f"  unrepresented task id(s): {missing}"
-    )
-
-
-def _failures_path_for(rollouts_path: Path) -> Path:
-    """Sidecar Gym writes failed rollouts to (mirrors Gym's own ``_failures_path_for``)."""
-    return rollouts_path.with_name(rollouts_path.stem + "_failures.jsonl")
-
-
-def _aggregate_metrics_path_for(rollouts_path: Path) -> Path:
-    """Sidecar Gym writes run-level aggregate metrics to (``<stem>_aggregate_metrics.json``)."""
-    return rollouts_path.with_name(rollouts_path.stem + "_aggregate_metrics.json")
-
-
-def _read_run_aggregations(rollouts_path: Path) -> dict[str, Any] | None:
-    """Parse Gym's ``rollouts_aggregate_metrics.json``, or ``None`` when absent/unparseable.
-
-    Gym's file is a list with one entry per agent (``agent_ref`` / ``agent_metrics`` / ``key_metrics`` /
-    ``group_level_metrics``), so it is returned keyed by agent name. Carried through as-is — Gym's schema
-    isn't contractual, so the SDK does not type it.
-    """
-    path = _aggregate_metrics_path_for(rollouts_path)
-    if not path.exists():
-        return None
-    try:
-        parsed = json.loads(path.read_text(encoding="utf-8"))
-    except (json.JSONDecodeError, OSError, UnicodeDecodeError):
-        # UnicodeDecodeError is a ValueError, not an OSError: a sidecar truncated mid-codepoint would
-        # otherwise raise straight out of run_tasks and discard a collection that already succeeded.
-        logger.warning("Could not parse Gym aggregate metrics at %s; skipping run aggregations.", path)
-        return None
-    if not isinstance(parsed, list):
-        logger.warning("Unexpected Gym aggregate-metrics shape at %s (%s); skipping.", path, type(parsed).__name__)
-        return None
-    # Gym's schema is not contractual, and this runs after a collection that already succeeded — an
-    # entry in an unexpected shape must not raise out of run_tasks and discard every trial with it.
-    aggregations: dict[str, Any] = {}
-    for entry in parsed:
-        agent_ref = entry.get("agent_ref") if isinstance(entry, Mapping) else None
-        name = agent_ref.get("name") if isinstance(agent_ref, Mapping) else None
-        if not isinstance(name, str):
-            logger.warning(
-                "Skipping a Gym aggregate-metrics entry in %s with no usable agent_ref.name (got %r).", path, agent_ref
-            )
-            continue
-        aggregations[name] = {key: value for key, value in entry.items() if key != "agent_ref"}
-    return aggregations or None
-
-
-#: Gym flattens a distribution into ``<stat>/<metric>`` keys. Its RewardProfiler emits this exact set
-#: together for every numeric column (``describe_dataframe``), minus ``histogram``, which
-#: ``prepare_for_serialization`` strips before the file is written. All five must be present before we
-#: treat a group of keys as one distribution: a resources-server is free to define a metric literally
-#: named ``mean`` (36 of Gym's ~97 servers override ``compute_metrics``), and re-assembling on a partial
-#: match would rename someone's standalone metric into a statistic of a distribution that never existed.
-_GYM_STAT_FAMILY = ("mean", "max", "min", "median", "std")
-
-#: Metric Gym reports that the SDK already computes natively from the same rollouts (``gym_reward.reward``).
-_GYM_REDUNDANT_METRICS = frozenset({"reward"})
-
-
-def _aggregate_scores_from_gym(aggregations: Mapping[str, Any] | None) -> list[AggregateScore]:
-    """Map Gym's run-level ``agent_metrics`` onto typed aggregate scores named ``runner.gym.<metric>``.
-
-    Reads ``agent_metrics``, not ``key_metrics``: ``key_metrics`` is a *subset* of it chosen by the
-    resources-server, and the default selection (``get_key_metrics``) keeps only the ``mean/*`` entries —
-    so the max/min/median/std that make a distribution never appear there, and every metric would arrive
-    as a lone ``mean/<name>`` scalar.
-
-    Keys forming a full stat family are re-assembled into one :class:`AggregateRangeScore`; every other
-    numeric key becomes an :class:`AggregateScalarScore`. Non-numeric values are skipped — they are
-    labels or notes, not measurements.
-
-    Names are namespaced by runner (not by agent): each run is instrumented with a single agent, so the
-    agent adds no disambiguation, and a reader needs to know which *backend* produced a number.
-    """
-    if not aggregations:
-        return []
-    multi_agent = len(aggregations) > 1
-    scores: list[AggregateScore] = []
-    for agent_name, payload in sorted(aggregations.items()):
-        agent_metrics = payload.get("agent_metrics") if isinstance(payload, Mapping) else None
-        if not isinstance(agent_metrics, Mapping):
-            continue
-        # One agent per run is the norm, so `runner.gym.<metric>` reads cleanly; qualify by agent only
-        # when a run really did produce several, where the unqualified names would collide.
-        prefix = f"runner.gym.{agent_name}." if multi_agent else "runner.gym."
-        scores.extend(_scores_from_agent_metrics(agent_metrics, prefix=prefix))
-    return scores
-
-
-def _scores_from_agent_metrics(agent_metrics: Mapping[str, Any], *, prefix: str) -> list[AggregateScore]:
-    families: dict[str, dict[str, float]] = {}
-    scalars: dict[str, float] = {}
-    for key, value in agent_metrics.items():
-        number = _as_float(value)
-        if number is None:
-            continue
-        stat, _, metric = key.partition("/")
-        if metric and stat in _GYM_STAT_FAMILY:
-            families.setdefault(metric, {})[stat] = number
-        else:
-            scalars[key] = number
-
-    scores: list[AggregateScore] = []
-    for metric, stats in sorted(families.items()):
-        # Redundancy is a property of the metric, not of how complete its family is. Checking after the
-        # fallback below would let a partial `reward` family survive as `mean/reward` scalars, since the
-        # scalar filter matches the bare name -- reintroducing the duplicate of `gym_reward.reward` that
-        # skipping reward exists to prevent.
-        if metric in _GYM_REDUNDANT_METRICS:
-            continue
-        if set(stats) < set(_GYM_STAT_FAMILY):
-            # Not a distribution we can vouch for; keep each key as the standalone number it may well be.
-            scalars.update({f"{stat}/{metric}": value for stat, value in stats.items()})
-            continue
-        scores.append(
-            AggregateRangeScore(
-                name=f"{prefix}{metric}",
-                # Gym reports the statistics but not the sample size behind them, and inventing one
-                # would misreport coverage. `None` says "unknown"; 0 would assert nothing was evaluated.
-                count=None,
-                nan_count=0,
-                mean=stats["mean"],
-                min=stats["min"],
-                max=stats["max"],
-                median=stats["median"],
-                # Gym computes this with pandas (ddof=1), so it is the sample standard deviation.
-                sample_std_dev=stats["std"],
-                sample_variance=stats["std"] ** 2,
-            )
-        )
-    scores.extend(
-        AggregateScalarScore(name=f"{prefix}{key}", count=None, nan_count=0, value=value)
-        for key, value in sorted(scalars.items())
-        if key not in _GYM_REDUNDANT_METRICS
-    )
-    return scores
-
-
-def _as_float(value: Any) -> float | None:
-    """``float`` for a numeric Gym metric value; None for anything else (bools included: not measurements)."""
-    if isinstance(value, bool) or not isinstance(value, (int, float)):
-        return None
-    return float(value) if math.isfinite(value) else None
-
-
-def _ensure_fresh_output(rollouts_path: Path) -> None:
-    """Enforce one Gym run per output dir (the AgentEvaluator convention).
-
-    Gym appends to the failures sidecar (``open("ab")``) and doesn't clear it between runs, so reusing
-    a populated directory would silently mix this run's failures with a prior run's. Rather than clear
-    (which would clobber an earlier run's results — infra failures are useful signal), refuse to run
-    into a directory that already holds Gym rollout output.
-    """
-    preexisting = [path for path in (rollouts_path, _failures_path_for(rollouts_path)) if path.exists()]
-    if preexisting:
-        names = ", ".join(path.name for path in preexisting)
-        raise FileExistsError(
-            f"{rollouts_path.parent} already holds Gym rollout output ({names}); give each run a fresh "
-            "output_dir (the AgentEvaluator convention). Gym appends to the failures sidecar, so reusing a "
-            "directory would mix runs and could obscure a prior run's results."
-        )
-
-
-def _coerce_reward(value: Any) -> float | None:
-    """Best-effort ``float`` for a reward; returns None (never raises) on a missing/malformed value."""
-    if value is None:
-        return None
-    try:
-        return float(value)
-    except (TypeError, ValueError):
-        logger.warning("Gym rollout carried a non-numeric reward %r; recording the trial as unscored.", value)
-        return None
-
-
-def _resolve_task_id(
-    record: Mapping[str, Any], index_to_task_id: Mapping[int, str], *, strict: bool = True
-) -> str | None:
-    """Map a rollout record to a task id via the ``_ng_task_index`` we stamped in the input dataset.
-
-    Returns None when the record carries no usable index (kill-shaped failures can lack one).
-
-    An index we never stamped is handled by ``strict``. In the **successes** file (``strict=True``)
-    it is fatal: every row Gym read came from :func:`_materialize_dataset`, so an unknown index means
-    Gym reindexed the dataset rather than honoring our stamp, and every reward attribution is
-    therefore suspect. In the **failures sidecar** (``strict=False``) it is merely unattributable and
-    is counted instead — that file is written during abnormal termination and read tolerantly by
-    design, so one odd record must not discard the successes already parsed. That relaxation cannot
-    inflate a result: failure records carry no reward, and a task left with no trial still trips
-    :func:`_require_full_coverage`.
-    """
-    index = record.get(NG_TASK_INDEX)
-    if not isinstance(index, int) or isinstance(index, bool):
-        return None
-    task_id = index_to_task_id.get(index)
-    if task_id is None:
-        if not strict:
-            logger.warning(
-                "Gym failure record carried %s=%s, which was not stamped on the materialized dataset; "
-                "counting it as unattributable rather than failing the run.",
-                NG_TASK_INDEX,
-                index,
-            )
-            return None
-        raise ValueError(
-            f"Gym emitted {NG_TASK_INDEX}={index}, which is not one of the {len(index_to_task_id)} index value(s) "
-            f"stamped on the materialized dataset. Gym appears to have reassigned {NG_TASK_INDEX} instead "
-            "of honoring the supplied one, so rollout-to-task attribution is unsafe."
-        )
-    return task_id
-
-
-def _rollout_trial_id(task_id: str, raw_rollout_index: Any, synth_seq: dict[str, int], *, missing_label: str) -> str:
-    """Per-attempt trial id: use the record's ``_ng_rollout_index`` when it's a real int, else a
-    per-task-unique ``{missing_label}{n}`` suffix so records lacking an index don't collide on one id."""
-    if isinstance(raw_rollout_index, int) and not isinstance(raw_rollout_index, bool):
-        return f"{task_id}:{raw_rollout_index}"
-    seq = synth_seq[task_id] = synth_seq.get(task_id, 0) + 1
-    return f"{task_id}:{missing_label}{seq}"
-
-
-def _trials_from_rollouts(
-    rollouts_path: Path,
-    tasks: Sequence[AgentEvalTask],
-    index_to_task_id: Mapping[int, str],
-    *,
-    reward_key: str = DEFAULT_REWARD_KEY,
-) -> list[AgentEvalTrial]:
-    """Fan Gym's rollout records out into one :class:`AgentEvalTrial` per attempt.
-
-    Reads *both* Gym output files: successes from ``rollouts.jsonl`` (COMPLETED, carrying the reward)
-    and failures from the ``*_failures.jsonl`` sidecar (FAILED — so a failed attempt is counted and
-    diagnosed rather than silently dropped). Every record is attributed to its task by
-    ``_ng_task_index`` → ``index_to_task_id``, the map we stamped in :func:`_materialize_dataset`.
-    """
-    known_task_ids = {task.id for task in tasks}
-    # Defence in depth: the index map and the task list must describe the same run. Gym is an external
-    # project on its own release cadence, and people make agent-optimization decisions from these
-    # numbers — so any disagreement about *which task a result belongs to* is a hard error, never a
-    # best-effort join. Cheap to check once; impossible to notice downstream if it silently drifts.
-    mapped_task_ids = set(index_to_task_id.values())
-    if mapped_task_ids != known_task_ids:
-        raise ValueError(
-            f"the {NG_TASK_INDEX} map does not match the task list it will be joined against "
-            f"({len(mapped_task_ids - known_task_ids)} mapped task(s) absent from the list, "
-            f"{len(known_task_ids - mapped_task_ids)} listed task(s) absent from the map); "
-            "attributing rollouts across a mismatch would report results against the wrong tasks"
-        )
-    trials: list[AgentEvalTrial] = []
-    # Shared per-task counter for records lacking a usable _ng_rollout_index (success + failure); a
-    # missing/null index must not collapse multiple attempts for one task onto the same trial id.
-    synth_seq: dict[str, int] = {}
-
-    success_evidence = CandidateEvidence(
-        descriptors={"rollouts": EvidenceDescriptor(kind="filesystem", format="file", ref=str(rollouts_path))}
-    )
-    unattributed_successes = 0
-    empty_output_count = 0
-    #: Tasks with at least one trial where the agent never ran. Tracked separately from trial
-    #: status, since the failures sidecar also produces FAILED trials for unrelated reasons.
-    empty_output_task_ids: set[str] = set()
-    for record in _read_jsonl(rollouts_path):
-        task_id = _resolve_task_id(record, index_to_task_id)
-        if task_id is None:
-            # No usable index (an unknown one raises). Skipping is right, but silence is not: with
-            # num_repeats > 1 the task keeps its other trials, so _require_full_coverage won't fire and
-            # the task would simply be scored on fewer attempts than were actually run.
-            unattributed_successes += 1
-            continue
-        raw_rollout_index = record.get(NG_ROLLOUT_INDEX)
-        reward = _coerce_reward(record.get(reward_key))
-        # An agent that never ran did not earn its reward — reporting 0.0 as a completed trial would
-        # let a broken environment read as a poorly-performing one.
-        never_ran = _agent_never_ran(record)
-        if never_ran:
-            empty_output_count += 1
-            empty_output_task_ids.add(task_id)
-            status = AgentEvalTrialStatus.FAILED
-        elif reward is not None:
-            status = AgentEvalTrialStatus.COMPLETED
-        else:
-            status = AgentEvalTrialStatus.PARTIAL
-        metadata: dict[str, Any] = {
-            "reward": None if never_ran else reward,
-            NG_TASK_INDEX: record.get(NG_TASK_INDEX),
-            NG_ROLLOUT_INDEX: raw_rollout_index,
-        }
-        if never_ran:
-            # Kept alongside the reward Gym reported, so the discarded value is still auditable.
-            metadata["gym_failure"] = (
-                "the agent produced no output and consumed no tokens, so the model was never called; "
-                f"the reported score of {reward!r} is not a measurement of this agent"
-            )
-        trials.append(
-            AgentEvalTrial(
-                id=_rollout_trial_id(task_id, raw_rollout_index, synth_seq, missing_label="noidx"),
-                task_id=task_id,
-                status=status,
-                output=AgentOutput(response=record.get("response"), metadata={"agent_ref": record.get("agent_ref")}),
-                evidence=success_evidence,
-                metadata=metadata,
-            )
-        )
-
-    failures_path = _failures_path_for(rollouts_path)
-    unattributed_failures = 0
-    if failures_path.exists():
-        failure_evidence = CandidateEvidence(
-            descriptors={
-                "rollout_failures": EvidenceDescriptor(kind="filesystem", format="file", ref=str(failures_path))
-            }
-        )
-        # tolerant: a truncated line in the abnormal-termination sidecar must not sink the good trials.
-        for record in _read_jsonl(failures_path, tolerant=True):
-            # strict=False to match this loop's tolerant read: an odd record is counted, not fatal.
-            task_id = _resolve_task_id(record, index_to_task_id, strict=False)
-            if task_id is None:
-                # kill-shaped failures (SIGTERM/OOM/actor-death) can lack a usable index — count, don't drop.
-                unattributed_failures += 1
-                continue
-            raw_rollout_index = record.get(NG_ROLLOUT_INDEX)
-            trials.append(
-                AgentEvalTrial(
-                    id=_rollout_trial_id(task_id, raw_rollout_index, synth_seq, missing_label="fail"),
-                    task_id=task_id,
-                    status=AgentEvalTrialStatus.FAILED,
-                    output=AgentOutput(
-                        response=record.get("response"), metadata={"agent_ref": record.get("agent_ref")}
-                    ),
-                    evidence=failure_evidence,
-                    metadata={
-                        "reward": None,
-                        # best-effort: Gym's failure-record schema isn't contractual, so probe common keys.
-                        "gym_failure": record.get("error") or record.get("exception") or record.get("failure_reason"),
-                        NG_TASK_INDEX: record.get(NG_TASK_INDEX),
-                        NG_ROLLOUT_INDEX: raw_rollout_index,
-                    },
-                )
-            )
-    if empty_output_count:
-        # Every trial empty is a configuration failure, not a result: nothing was measured, so there
-        # is no score to report and letting the run "succeed" would publish zeros as findings. A
-        # partial count is left as failed trials — a flaky environment is still worth the attempts
-        # that did run.
-        #
-        # Measured against *every* trial, including those the failures sidecar contributed. A run
-        # that also recorded real failures is not the uniform "nothing ran" picture this raise is
-        # for: those trials carry their own diagnosis (a verifier timeout, say, from an attempt that
-        # did reach the model), and raising would discard it for a less specific message. Likewise
-        # `unattributed_successes` never became trials and so are invisible to `len(trials)`, but
-        # each is a result the environment produced and one may have run perfectly well — claiming
-        # nothing was measured while such a record exists would be false.
-        if empty_output_count == len(trials) and not unattributed_successes:
-            affected_tasks = {trial.task_id for trial in trials}
-            raise RuntimeError(
-                f"No agent ran: all {empty_output_count} trial(s) across all {len(affected_tasks)} "
-                "task(s) produced no output and consumed no tokens, so the model was never called "
-                "and nothing was measured. The scores reported for them describe an agent that did "
-                "not run.\n"
-                "This is normally the environment failing to start its agent rather than a bad "
-                f"model: check the per-server startup output in {rollouts_path.parent / _ENV_LOG_NAME} "
-                "for a traceback from the environment's own agent server.\n"
-                f"Results as collected: {rollouts_path}"
-            )
-        logger.warning(
-            "%d of %d trial(s), across %d of %d task(s), produced no agent output and consumed no "
-            "tokens; those are marked FAILED rather than scored, since the model was never called "
-            "for them.",
-            empty_output_count,
-            len(trials),
-            len(empty_output_task_ids),
-            len({trial.task_id for trial in trials}),
-        )
-    if unattributed_successes:
-        logger.warning(
-            "%d Gym rollout record(s) in %s carried no usable %s and could not be attributed to a task; "
-            "affected tasks are scored on fewer attempts than were collected.",
-            unattributed_successes,
-            rollouts_path,
-            NG_TASK_INDEX,
-        )
-    if unattributed_failures:
-        logger.warning(
-            "%d Gym failure record(s) lacked a usable %s and could not be attributed to a task.",
-            unattributed_failures,
-            NG_TASK_INDEX,
-        )
-
-    # Reported here, enforced by _require_full_coverage in run_tasks (which knows the run's log paths).
-    missing = known_task_ids - {trial.task_id for trial in trials}
-    if missing:
-        logger.warning("No Gym rollout produced a trial for %d requested task(s): %s", len(missing), sorted(missing))
-    return trials
-
-
-def _signal_group(pgid: int, sig: int) -> None:
-    """Send ``sig`` to a process group, tolerating an already-dead group."""
-    try:
-        os.killpg(pgid, sig)
-    except ProcessLookupError:
-        pass
-
-
-async def _terminate(proc: asyncio.subprocess.Process, *, grace_s: float = 30.0) -> None:
-    """Tear down a backgrounded Gym subprocess *and its whole process group*.
-
-    ``gym env start`` fans out into a Ray cluster + uvicorn servers, and Gym stops them from a
-    ``KeyboardInterrupt`` handler (``finally: self.shutdown()``) — so we send **SIGINT** (not SIGTERM,
-    which would bypass that handler and orphan Ray's detached ``gcs_server``). The child is spawned with
-    ``start_new_session=True`` and leads its own group, so SIGINT to the group mimics Ctrl-C exactly;
-    SIGKILL to the group is the escalation if Gym's graceful shutdown overstays ``grace_s``.
-
-    Known limitation: Ray daemonizes ``gcs_server`` into its *own* session, outside this group, so the
-    escalation SIGKILL cannot reach it — clean Ray teardown depends on Gym's SIGINT ``shutdown()``
-    completing within ``grace_s``. POSIX-only (``killpg`` / ``getpgid`` / ``start_new_session`` / SIGINT).
-    """
-    if proc.returncode is not None:
-        return
-    try:
-        pgid = os.getpgid(proc.pid)
-    except ProcessLookupError:
-        return
-    _signal_group(pgid, signal.SIGINT)
-    try:
-        await asyncio.wait_for(proc.wait(), timeout=grace_s)
-    except asyncio.TimeoutError:
-        _signal_group(pgid, signal.SIGKILL)
-        await proc.wait()
-
-
-__all__ = [
-    "DEFAULT_REWARD_KEY",
-    "GymAgentTaskRunner",
-    "GymRewardMetric",
-    "GymRuntimeConfig",
-    "discover_gym_tasks",
-]