`. Harbor uploads a task's `tests/` directory to the sandbox by tarring it, and
+# tar faithfully preserves symlinks — so the sandbox receives `test.sh -> ../../../blobs/09b32…`,
+# pointing at a path that does not exist there. Bash reports a dangling symlink as
+# "No such file or directory", which makes it look like the upload failed when the entry is right
+# there in `ls`.
+#
+# That cost a long debugging session: `upload_dir` appeared to work, `chmod +x` as root succeeded,
+# and only `ls -la` revealed the arrow. Backends differ in whether they hit it — E2B's upload path
+# does not preserve symlinks, Modal's tar-based one does — so it presents as "Modal is broken".
+#
+# `local_dir=` makes huggingface_hub write real files instead of populating the symlink cache, which
+# fixes it for every backend at once and needs no Harbor change.
+_DATASET_ROOT = Path(
+ os.environ.get("OPENENV_DATASET_CACHE")
+ or (Path.home() / ".cache" / "openenv" / "harbor-datasets")
+)
+
+
+# A Harbor task suite is thousands of tiny files (a `task.toml`, a Dockerfile, a test script per
+# task), so wall clock is dominated by per-file round trips rather than bytes. Raising concurrency is
+# the lever that matters; `hf_transfer` optimises large-file throughput and does comparatively little
+# here, but costs nothing when it is installed.
+_DOWNLOAD_WORKERS = int(os.environ.get("OPENENV_DATASET_WORKERS", "32"))
+
+
+def _materialise_hf_dataset(spec: str) -> Path:
+ """Download an HF dataset as real files and return its local root.
+
+ Mounting beats downloading where it is available: a deployed Space can attach the dataset repo
+ as a read-only volume and pass the mount path as the dataset spec, which skips this entirely.
+ `openenv harbor push` does that automatically. This path is for local runs.
+ """
+ from huggingface_hub import snapshot_download
+
+ target = _DATASET_ROOT / spec.replace("/", "__")
+ target.mkdir(parents=True, exist_ok=True)
+ snapshot_download(
+ spec,
+ repo_type="dataset",
+ allow_patterns=["tasks/**"],
+ local_dir=str(target),
+ max_workers=_DOWNLOAD_WORKERS,
+ )
+ return target
+
+
+def has_symlinks(task_dir: Path) -> list[Path]:
+ """Any symlinks under `task_dir`. Non-empty means uploads to a tar-based backend will break."""
+ return [p for p in task_dir.rglob("*") if p.is_symlink()]
+
+
+def _registry_task_dirs(spec: str) -> list[Path]:
+ """A Harbor registry dataset, e.g. `terminal-bench@1.0`. Downloads on first use."""
+ from harbor.models.job.config import DatasetConfig
+ from openenv.core.utils import run_async_safely
+
+ name, _, version = spec.partition("@")
+ config = DatasetConfig(name=name, version=version or None)
+ # Discovery is reached from async callers too (`run_batch` is a coroutine), and `asyncio.run`
+ # cannot be called from a running loop.
+ task_configs = run_async_safely(config.get_task_configs(disable_verification=True))
+ return [Path(str(t.get_local_path())) for t in task_configs]
+
+
+def read_instruction(task_dir: Path, *, limit: int = 4000) -> str:
+ """The task's prompt, for previewing in discovery. Truncated: this is not the authoritative copy.
+
+ The sandbox gets the real instruction from Harbor at run time. Serving a huge prompt over the
+ Task API for every listed task would make `list_tasks` enormous for no benefit.
+ """
+ path = task_dir / "instruction.md"
+ if not path.is_file():
+ return ""
+ text = path.read_text(errors="replace").strip()
+ return text if len(text) <= limit else text[:limit] + "\n…"
+
+
+def prefetch(datasets: list[str]) -> dict[str, Any]:
+ """Resolve every dataset up front, downloading if needed.
+
+ Called before the server accepts traffic. Two reasons it is worth doing eagerly rather than on
+ first use: a 2000-task HF repo takes real time to fetch, and a caller who mistypes a dataset
+ name should learn at startup rather than when the first rollout 404s. Failures are collected
+ rather than raised, so one bad dataset does not stop the server serving the good ones.
+
+ Args:
+ datasets (`list[str]`):
+ Dataset specs — HF repo id, local path, or Harbor `name@version`.
+
+ Returns:
+ `dict` mapping each spec to `{"num_tasks": int}` or `{"error": str}`.
+ """
+ report: dict[str, Any] = {}
+ for spec in datasets:
+ try:
+ report[spec] = {"num_tasks": len(resolve_task_dirs(spec))}
+ except Exception as exc: # noqa: BLE001 - one broken dataset must not hide the others
+ report[spec] = {"error": f"{type(exc).__name__}: {str(exc)[:200]}"}
+ return report
+
+
+class HarborTaskProvider:
+ """Serves one or more Harbor datasets as OpenEnv splits.
+
+ A split IS a dataset spec: start the server with two datasets and you get two splits. That keeps
+ the mapping obvious in both directions — a split name is something you can paste back into
+ `--dataset` — rather than inventing a train/test split Harbor does not have.
+ """
+
+ def __init__(self, datasets: list[str] | None = None) -> None:
+ self._datasets = list(datasets or [])
+
+ # --- TaskProvider protocol ------------------------------------------
+ def list_splits(self) -> list[dict[str, Any]]:
+ splits = []
+ for spec in self._datasets:
+ try:
+ n = len(resolve_task_dirs(spec))
+ splits.append({"name": spec, "num_tasks": n})
+ except Exception as exc: # noqa: BLE001 - a broken dataset must not hide the good ones
+ splits.append({"name": spec, "num_tasks": 0, "error": str(exc)[:200]})
+ return splits
+
+ def num_tasks(self, split: str) -> int:
+ return len(resolve_task_dirs(self._check(split)))
+
+ def list_tasks(self, split: str) -> list[dict[str, Any]]:
+ spec = self._check(split)
+ return [
+ self._ref(spec, i, d).model_dump()
+ for i, d in enumerate(resolve_task_dirs(spec))
+ ]
+
+ def get_task(self, split: str, index: int) -> dict[str, Any]:
+ spec = self._check(split)
+ dirs = resolve_task_dirs(spec)
+ if not 0 <= index < len(dirs):
+ raise IndexError(
+ f"task index {index} out of range for {spec!r} ({len(dirs)} tasks)"
+ )
+ return self._ref(spec, index, dirs[index]).model_dump()
+
+ def get_task_range(
+ self, split: str, start: int | None = None, stop: int | None = None
+ ) -> list[dict[str, Any]]:
+ spec = self._check(split)
+ dirs = resolve_task_dirs(spec)
+ return [
+ self._ref(spec, i, d).model_dump()
+ for i, d in list(enumerate(dirs))[start:stop]
+ ]
+
+ # --- internals -------------------------------------------------------
+ def task_dir(self, split: str, index: int) -> Path:
+ """The on-disk task dir for an index. Used by the rollout path, not by discovery."""
+ dirs = resolve_task_dirs(self._check(split))
+ if not 0 <= index < len(dirs):
+ raise IndexError(f"task index {index} out of range ({len(dirs)} tasks)")
+ return dirs[index]
+
+ def _check(self, split: str) -> str:
+ if not self._datasets:
+ raise ValueError("this server was started with no datasets; pass --dataset")
+ if not split:
+ return self._datasets[0]
+ if split not in self._datasets:
+ raise ValueError(
+ f"unknown split {split!r}; served splits are {self._datasets}"
+ )
+ return split
+
+ @staticmethod
+ def _ref(spec: str, index: int, task_dir: Path) -> HarborTaskRef:
+ return HarborTaskRef(
+ index=index,
+ task_id=str(task_dir),
+ task_name=task_dir.name,
+ dataset=spec,
+ instruction=read_instruction(task_dir),
+ )
diff --git a/src/openenv/harbor/ui.py b/src/openenv/harbor/ui.py
new file mode 100644
index 0000000000..b86d64d4f7
--- /dev/null
+++ b/src/openenv/harbor/ui.py
@@ -0,0 +1,1511 @@
+"""Human-facing UI for a Harbor env server.
+
+Two columns: the LLM on the left, the task on the right. Validate, pick, run.
+
+Status text is deliberately terse. The long explanations belong in docs — what a person needs on
+screen is whether it will work, what got rewritten, and which sandboxes are usable.
+
+Validation is a gate, not a hint: an LLM endpoint without token-id capture answers every request
+normally and returns nothing trainable, so a rollout looks perfect and is worthless.
+
+Rich output (the rollout graph, per-turn tokens) is rendered as HTML rather than Gradio widgets,
+because a conversation tree with branches and discarded retries is a shape, and a dataframe cannot
+show a shape.
+"""
+
+from __future__ import annotations
+
+import html
+import json
+import re
+from typing import Any
+
+import gradio as gr
+
+_UNVALIDATED = "_Enter your LLM URL and press Validate._"
+
+_CSS = """
+.hb-wrap { max-width: 1400px; margin: 0 auto; }
+.hb-card { border: 1px solid var(--border-color-primary); border-radius: 10px; padding: 14px 16px; }
+.hb-dim { opacity: .6; }
+.hb-kv { display: flex; gap: 22px; flex-wrap: wrap; margin: 4px 0 2px; }
+.hb-kv b { font-variant-numeric: tabular-nums; }
+
+/* The two panels read as one undifferentiated wall of controls without a boundary; the border is
+ what makes "pick a model" and "pick a task" look like two separate decisions. */
+.hb-cell { border: 1px solid var(--border-color-primary); border-radius: 10px;
+ padding: 14px 16px; }
+.hb-panel { border: 1px solid var(--border-color-primary) !important;
+ border-radius: 10px !important; padding: 16px !important;
+ background: var(--block-background-fill); }
+.hb-cell { min-width: 0 !important; }
+.hb-wrap .hb-panel + .hb-panel { margin-top: 12px; }
+.hb-tx, .hb-card { overflow-wrap: anywhere; }
+@media (max-width: 700px) {
+ .hb-cell, .hb-panel { padding: 12px !important; }
+ .hb-hero { flex-wrap: wrap; }
+ .hb-kv { gap: 12px; }
+}
+@media (prefers-reduced-motion: reduce) {
+ .hb-pulse, .hb-step.now .hb-dot { animation: none; }
+}
+
+/* Live conversation. Roles are colour-coded down the left edge so the shape of the loop
+ (assistant calls a tool, tool answers, assistant calls again) is readable at a glance. */
+/* No max-height here. A fixed-height scroll box nests a second scroller inside the page:
+ the wheel gets captured while the pointer is over the conversation, and the page stops
+ growing so there is nothing left to scroll to. Let it run at natural height and let the
+ page do the scrolling. Length is bounded by the message cap, not by CSS. */
+.hb-tx { margin-top: 10px; }
+.hb-msg { border-left: 3px solid var(--border-color-primary); padding: 6px 0 6px 10px;
+ margin: 8px 0; font-size: 13px; line-height: 1.45; }
+.hb-msg pre { white-space: pre-wrap; word-break: break-word; margin: 4px 0 0;
+ font-size: 12px; opacity: .85; }
+.hb-role { display: inline-block; font-size: 11px; text-transform: uppercase;
+ letter-spacing: .04em; opacity: .65; margin-bottom: 2px; }
+.hb-assistant { border-left-color: #22c55e; }
+.hb-tool { border-left-color: #38bdf8; }
+.hb-user { border-left-color: #a78bfa; }
+.hb-system { border-left-color: #94a3b8; opacity: .75; }
+.hb-tc { margin-top: 4px; padding: 4px 8px; border-radius: 6px;
+ background: var(--background-fill-secondary); }
+.hb-tc { display: block; }
+.hb-tc b { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; font-size: 12px; }
+.hb-arrow { opacity: .5; margin-right: 6px; }
+.hb-tr { margin-top: 4px; padding: 4px 8px; border-radius: 6px; border-left: 2px solid #38bdf8;
+ background: var(--background-fill-secondary); }
+/* No inner scroller here either, for the same reason as the conversation above, and the previous
+ version of this rule was the bug: `overscroll-behavior: contain` does not stop a box from
+ swallowing the page scroll, it is what *prevents* the wheel from chaining to the page once the
+ box reaches its own end. Tool output is clipped to 500 characters server side, but 500
+ characters of shell output is 25 short lines, which overflowed the 220px cap and left the page
+ feeling frozen wherever the pointer happened to be. Length is bounded by the clip, not by CSS. */
+.hb-tr pre{ margin: 0; font-size: 11.5px; opacity: .8; }
+
+/* A run in flight should look like one. */
+.hb-live { display: flex; align-items: center; gap: 10px; margin-bottom: 6px; }
+.hb-pulse { width: 8px; height: 8px; border-radius: 50%; background: #22c55e;
+ animation: hb-blink 1.2s ease-in-out infinite; }
+@keyframes hb-blink { 0%, 100% { opacity: 1; } 50% { opacity: .25; } }
+.hb-drop-msg { opacity: .5; border-left-color: #ef4444; }
+
+/* Verdict. The outcome should be legible from across the room; the numbers behind it should not
+ compete with it for attention. */
+.hb-verdict { border-left-width: 4px; }
+.hb-head { font-size: 17px; font-weight: 650; margin-bottom: 8px; }
+.hb-good { border-left-color: #22c55e; }
+.hb-warn { border-left-color: #f59e0b; }
+.hb-bad { border-left-color: #ef4444; }
+.hb-err { white-space: pre-wrap; word-break: break-word; font-size: 12px; margin: 10px 0 0;
+ padding: 8px 10px; border-radius: 6px; background: var(--background-fill-secondary); }
+
+/* A qualifier on the result: true, load-bearing, and not an error. Bordered rather than coloured
+ like a finding, so "this rollout is eval-only" does not read as "this rollout failed". */
+.hb-note { font-size: 12.5px; line-height: 1.5; margin: 10px 0 0; padding: 8px 11px;
+ border-radius: 6px; border: 1px solid var(--border-color-primary);
+ background: var(--background-fill-secondary); }
+.hb-note code { font-size: 11.5px; }
+
+/* Hover explanations. `data-tip` rather than `title=` for the two long ones: the native tooltip
+ truncates, takes a second to appear, and cannot wrap a paragraph. Short hints use Gradio's own
+ `info=`, which renders under the label and needs no hover at all. */
+.hb-i { display: inline-flex; align-items: center; justify-content: center; cursor: help;
+ width: 15px; height: 15px; margin-left: 6px; border-radius: 50%; font-size: 10px;
+ font-weight: 700; font-style: normal; vertical-align: 1px;
+ border: 1px solid var(--border-color-primary); opacity: .75; position: relative; }
+.hb-i:hover { opacity: 1; }
+.hb-i::after { content: attr(data-tip); position: absolute; left: 50%; bottom: 130%;
+ transform: translateX(-50%); width: max-content; max-width: 320px; padding: 8px 10px;
+ border-radius: 6px; border: 1px solid var(--border-color-primary);
+ background: var(--background-fill-primary); color: var(--body-text-color);
+ font-size: 11.5px; font-weight: 400; line-height: 1.5; text-align: left;
+ white-space: pre-line; opacity: 0; visibility: hidden; transition: opacity .12s;
+ z-index: 40; box-shadow: 0 4px 14px rgba(0,0,0,.18); }
+.hb-i:hover::after { opacity: 1; visibility: visible; }
+/* The label row the icon sits on, so the icon lines up with a Gradio label rather than floating. */
+.hb-lbl { display: flex; align-items: center; font-size: 13px; font-weight: 600;
+ margin: 2px 0 -6px; }
+
+/* Findings carry severity: a FATAL means unusable, a WARN means read before training on it. */
+.hb-find { font-size: 12.5px; margin: 5px 0; line-height: 1.45; }
+.hb-tag { display: inline-block; min-width: 46px; margin-right: 8px; padding: 1px 6px;
+ border-radius: 4px; font-size: 10px; font-weight: 700; letter-spacing: .04em;
+ text-align: center; vertical-align: 1px; }
+.hb-fatal .hb-tag { background: #ef4444; color: #fff; }
+.hb-warn2 .hb-tag { background: #f59e0b; color: #1f2937; }
+.hb-info .hb-tag { background: var(--background-fill-secondary); opacity: .7; }
+.hb-info { opacity: .7; }
+
+/* Turn table: dense, aligned, and the numbers read as numbers. */
+.hb-tbl { width: 100%; border-collapse: collapse; margin-top: 8px; font-size: 13px; }
+.hb-tbl th{ text-align: left; font-weight: 600; font-size: 11px; text-transform: uppercase;
+ letter-spacing: .04em; opacity: .55; padding: 4px 10px 6px 0;
+ border-bottom: 1px solid var(--border-color-primary); }
+.hb-tbl td{ padding: 7px 10px 7px 0; border-bottom: 1px solid var(--border-color-primary);
+ vertical-align: top; }
+.hb-tbl code { font-size: 12px; padding: 1px 6px; border-radius: 4px;
+ background: var(--background-fill-secondary); }
+.hb-num { font-variant-numeric: tabular-nums; text-align: right; white-space: nowrap;
+ padding-right: 14px !important; }
+.hb-prev { margin-top: 3px; font-size: 12px; }
+.hb-drop-row { opacity: .45; }
+.hb-drop-tag { background: #ef4444; color: #fff; }
+.hb-conf { display: inline-block; width: 76px; height: 7px; border-radius: 4px;
+ background: var(--background-fill-secondary); overflow: hidden; vertical-align: middle; }
+.hb-conf span { display: block; height: 100%; }
+
+/* Each conversation folds away; the main one starts open. */
+.hb-convo { margin-top: 10px; border-top: 1px solid var(--border-color-primary); padding-top: 8px; }
+.hb-convo summary { cursor: pointer; padding: 4px 0; }
+
+/* Setup, before the agent has said anything. */
+.hb-steps { margin: 8px 0 0; }
+.hb-step { display: flex; align-items: center; gap: 9px; padding: 3px 0; font-size: 13px; }
+.hb-dot { width: 7px; height: 7px; border-radius: 50%; background: var(--border-color-primary); }
+.hb-step.done .hb-dot { background: #22c55e; }
+.hb-step.now .hb-dot { background: #f59e0b; animation: hb-blink 1.2s ease-in-out infinite; }
+.hb-step.todo { opacity: .45; }
+
+/* The outcome, at a glance. */
+.hb-hero { display: flex; align-items: center; justify-content: space-between; gap: 20px;
+ padding-bottom: 12px; margin-bottom: 4px;
+ border-bottom: 1px solid var(--border-color-primary); }
+.hb-badge { display: inline-flex; align-items: center; gap: 9px; font-size: 19px;
+ font-weight: 700; letter-spacing: -.01em; }
+.hb-mark { display: inline-flex; align-items: center; justify-content: center;
+ width: 30px; height: 30px; border-radius: 50%; font-size: 15px; color: #fff; }
+.hb-b-good .hb-mark { background: #22c55e; }
+.hb-b-warn .hb-mark { background: #f59e0b; }
+.hb-b-bad .hb-mark { background: #ef4444; }
+.hb-score { text-align: right; line-height: 1.05; }
+.hb-score-v { font-size: 42px; font-weight: 700; font-variant-numeric: tabular-nums;
+ letter-spacing: -.02em; }
+.hb-score-c { font-size: 11px; text-transform: uppercase; letter-spacing: .06em; opacity: .55; }
+.hb-kv-big span { font-size: 11px; text-transform: uppercase; letter-spacing: .04em;
+ opacity: .55; }
+.hb-kv-big b { display: block; font-size: 19px; margin-top: 3px; text-transform: none;
+ letter-spacing: normal; opacity: 1; }
+.hb-kv-big .hb-key b { color: var(--body-text-color); }
+.hb-kv-big .hb-key { opacity: .85; }
+
+footer { display: none !important; }
+"""
+
+
+def _labelled(label: str, tip: str) -> str:
+ """A field label with a hover-explained `i` beside it.
+
+ For the explanations too long to sit under a Gradio label as `info=` text — which is where every
+ one-liner belongs instead, since it needs no hover to be seen.
+ """
+ return (
+ f'{html.escape(label)}'
+ f'i
'
+ )
+
+
+_KEY_TIP = (
+ "Only needed for a hosted endpoint: OpenAI, Anthropic, HF Inference Providers.\n\n"
+ "It is sent to the inference endpoint by this server and nothing else. It is NOT the key the "
+ "agent receives — that one is a capture session id, minted per rollout, which is how one proxy "
+ "serves many rollouts and how an unregistered caller is rejected.\n\n"
+ "Leave empty for a local vLLM or SGLang."
+)
+
+_LEVEL_TIP = (
+ "There are two kinds of rollout, and the endpoint decides which you get.\n\n"
+ "TRAIN needs the engine to return token ids and per-token logprobs: vLLM started with "
+ "--return-tokens-as-token-ids --logprobs-mode processed_logprobs, or SGLang built from git "
+ "main. You get the reward, the trace, and the exact tokens and logprobs to train on.\n\n"
+ "EVAL is everything else, including a vLLM started without those flags. You get the reward and "
+ "the full trace; there are no token ids, so nothing is trainable. Logprobs alone do not help — "
+ "with no ids to pair them with there is nothing to align them to."
+)
+
+
+def _clip(text: Any, limit: int = 400) -> str:
+ """Escape and shorten a value for display, keeping the head where the meaning usually is."""
+ body = text if isinstance(text, str) else json.dumps(text, default=str)
+ body = body.strip()
+ return html.escape(body[:limit]) + ("…" if len(body) > limit else "")
+
+
+def _tool_calls(message: dict[str, Any]) -> list[dict[str, Any]]:
+ """Tool calls on a message, normalised across all four dialects.
+
+ Chat-completions puts them in `tool_calls`; Anthropic puts them in the content block list as
+ `tool_use`. Reading only the former shows claude-code as a stream of text with no visible
+ actions, which is exactly the case the live view exists to make visible.
+ """
+ out: list[dict[str, Any]] = []
+ for call in message.get("tool_calls") or []:
+ function = call.get("function") or {}
+ name = function.get("name") or call.get("name")
+ if name:
+ out.append(
+ {
+ "name": str(name),
+ "arguments": function.get("arguments", call.get("arguments", "")),
+ }
+ )
+ content = message.get("content")
+ if isinstance(content, list):
+ for block in content:
+ if (
+ isinstance(block, dict)
+ and block.get("type") == "tool_use"
+ and block.get("name")
+ ):
+ out.append(
+ {"name": str(block["name"]), "arguments": block.get("input", "")}
+ )
+ return out
+
+
+def _message_text(message: dict[str, Any]) -> str:
+ """Readable text of a message, ignoring tool-call and tool-result blocks."""
+ content = message.get("content")
+ if isinstance(content, str):
+ return content
+ if isinstance(content, list):
+ parts = []
+ for block in content:
+ if not isinstance(block, dict):
+ continue
+ if block.get("type") in ("tool_use", "tool_result"):
+ continue
+ if block.get("text"):
+ parts.append(str(block["text"]))
+ return " ".join(parts)
+ return ""
+
+
+def _tool_results(message: dict[str, Any]) -> list[str]:
+ """What came back from a tool, in either the chat-completions or the Anthropic shape."""
+ if message.get("role") == "tool":
+ return [
+ _message_text(message) or json.dumps(message.get("content"), default=str)
+ ]
+ content = message.get("content")
+ if not isinstance(content, list):
+ return []
+ out = []
+ for block in content:
+ if isinstance(block, dict) and block.get("type") == "tool_result":
+ body = block.get("content")
+ if isinstance(body, list):
+ body = " ".join(b.get("text", "") for b in body if isinstance(b, dict))
+ out.append(str(body if body is not None else ""))
+ return out
+
+
+def _render_calls(calls: list[dict[str, Any]]) -> str:
+ return "".join(
+ f'▸'
+ f"
{html.escape(str(c.get('name', 'tool')))}"
+ f"
{_clip(c.get('arguments', ''), 600)} "
+ for c in calls
+ )
+
+
+def _render_message(message: dict[str, Any], *, label: str = "") -> str:
+ """One row of the conversation: who spoke, what they said, what they invoked or returned."""
+ role = str(message.get("role", "?"))
+ calls = _tool_calls(message)
+ results = _tool_results(message)
+ text = _message_text(message)
+
+ # A user message carrying only tool results is the tool speaking, not the user; labelling it
+ # "user" makes the agent look like it is being prompted between every action.
+ shown_role = "tool" if results and role != "assistant" else role
+ # For a `role: tool` message the content IS the result, so rendering both duplicates it.
+ if shown_role == "tool":
+ text = ""
+ body = _clip(text, 700 if shown_role in ("user", "system") else 450) if text else ""
+ blocks = "".join(
+ f'' for r in results
+ )
+ if not body and not blocks and not calls:
+ return ""
+ return (
+ f''
+ f'
{html.escape(label or shown_role)}'
+ + (f"
{body}
" if body else "")
+ + _render_calls(calls)
+ + blocks
+ + "
… {len(rows) - len(shown)} earlier message(s)
'
+ if len(rows) > len(shown)
+ else ""
+ )
+ # Count across the conversation, not just the response messages: Anthropic carries tool use in
+ # the assistant content blocks the harness replays back, so a response-only tally reads 0.
+ calls_so_far = sum(
+ len(_tool_calls(m)) for m in (latest.request_messages or [])
+ ) + len(_tool_calls(latest.response_message or {}))
+ return (
+ f''
+ f'Live conversation'
+ f'turn {latest.index} · {calls_so_far} tool call(s) so far · '
+ f"{latest.n_tools} tool(s) offered
{elided}{''.join(shown)}
'
+ f"{html.escape(label)}
"
+ )
+ return f'{"".join(rows)}
'
+
+
+def _live_html(
+ harness: str,
+ sandbox: str,
+ phase: str,
+ elapsed: float,
+ stats: dict[str, Any] | None,
+ stage: int = -1,
+) -> str:
+ """The running header: what is running, how far in, and what it has produced so far."""
+ bits = [
+ f''
+ f'
'
+ f"Running {html.escape(harness)} on "
+ f"{html.escape(sandbox)}"
+ f'{html.escape(phase)} · {elapsed:.0f}s
'
+ ]
+ # Before the first call there are no numbers worth showing, so show progress instead. A row of
+ # zeros for a minute reads as "stuck" when the sandbox is simply still booting.
+ if stage >= 0:
+ bits.append(_steps_html(stage))
+ if stats:
+ bits.append(
+ '
'
+ + "".join(f"{k}
{v}" for k, v in stats.items())
+ + "
"
+ )
+ bits.append("
'
+ f'{level}{html.escape(item[:400])}
'
+ )
+ return "".join(out)
+
+
+def _result_html(r: dict[str, Any]) -> str:
+ """The verdict, the numbers behind it, and anything that qualifies it.
+
+ The outcome is the one thing every reader wants first, so the reward is set at display size and
+ the supporting counts are deliberately quieter. Getting that hierarchy wrong is how a failed
+ rollout reads as a successful one at a glance.
+ """
+ reward = r.get("reward")
+ if not r.get("ok"):
+ tone, mark, label = "bad", "✕", "Failed"
+ value, caption = "—", str(r.get("exception_type") or "error")
+ elif reward is None:
+ # Not a zero. The verifier never ran, so this says nothing about the model.
+ tone, mark, label = "warn", "!", "Not graded"
+ value, caption = "—", "the verifier never ran"
+ elif reward > 0:
+ tone, mark, label = "good", "✓", "Solved"
+ value, caption = f"{reward:.2f}", "reward"
+ else:
+ tone, mark, label = "warn", "○", "Not solved"
+ value, caption = f"{reward:.2f}", "reward"
+
+ turns = r.get("turns") or []
+ generated = sum(len(t.get("completion_token_ids") or []) for t in turns)
+ dropped = sum(
+ len(t.get("completion_token_ids") or []) for t in turns if t.get("discarded")
+ )
+ tools = sum(len(t.get("tool_calls") or []) for t in turns)
+ atif = r.get("atif", "none")
+
+ # `key` marks the figures that decide whether this rollout is usable, as opposed to describing it.
+ # The initial prompt: task instruction plus the harness's system prompt and tool manifest.
+ # Constant across turns, so it is a property of the rollout rather than a per-row column.
+ context = len((turns[0].get("prompt_token_ids") or [])) if turns else 0
+
+ is_eval = r.get("rollout_type", "train") == "eval"
+ kv = [
+ # "trainable tokens: 0" on an eval rollout reads as a capture failure. It is not one, so the
+ # slot says what kind of rollout this is instead of reporting a zero that means nothing here.
+ ("rollout", f"EVAL · {r.get('capture_level', '?')}", True)
+ if is_eval
+ else ("trainable tokens", f"{r.get('n_trainable_tokens', 0):,}", True),
+ ("context", f"{context:,}", False),
+ ("trace check", atif, atif != "match"),
+ ("model calls", r.get("n_turns", 0), False),
+ ("tool calls", tools, False),
+ ("conversations", r.get("n_roots", 0), False),
+ (
+ "generated",
+ f"{generated:,}" + (f" · {dropped:,} discarded" if dropped else ""),
+ False,
+ ),
+ ("wall", f"{r.get('wall_s', 0):.0f}s", False),
+ ]
+
+ out = [
+ f'',
+ '
',
+ f'
{mark}'
+ f"{html.escape(label)}
",
+ f'
{html.escape(value)}
'
+ f'
{html.escape(caption)}
',
+ "
",
+ '
'
+ + "".join(
+ f'{k}
{v}'
+ for k, v, key in kv
+ )
+ + "
",
+ ]
+
+ if is_eval:
+ out.append(
+ '
This is an eval rollout. The endpoint returned '
+ f"{'logprobs but no token ids' if r.get('capture_level') == 'logprobs' else 'no token ids and no logprobs'}, "
+ "so you get the reward and the full trace below, but nothing trainable — there is no "
+ "contract.json and no per-token logprobs. Point the server at vLLM "
+ "(--return-tokens-as-token-ids --logprobs-mode processed_logprobs) or "
+ "SGLang built from main for trainable rollouts.
"
+ )
+ for fix in r.get("param_fixes") or []:
+ out.append(
+ f'
upstream compatibility: {html.escape(fix)} — '
+ "the request differs from what the harness asked for.
"
+ )
+
+ rewards = r.get("rewards") or {}
+ if len(rewards) > 1:
+ chosen = r.get("reward_key", "")
+ parts = [
+ f"
{html.escape(k)} {v:.3f}" + (" ←" if k == chosen else "")
+ for k, v in sorted(rewards.items())
+ ]
+ out.append(f'
{" ".join(parts)}
')
+
+ for step in r.get("step_results") or []:
+ vals = ", ".join(f"{k}={v:.2f}" for k, v in (step.get("rewards") or {}).items())
+ out.append(
+ f'
step {html.escape(step.get("name", ""))} {vals}
'
+ )
+
+ if r.get("error"):
+ out.append(f'
{html.escape(str(r["error"])[:1200])}')
+ if r.get("agent_log_tail"):
+ out.append(
+ '
agent log
'
+ f"{html.escape(str(r['agent_log_tail'])[:4000])}"
+ )
+
+ out.append(_findings_html(r.get("findings") or []))
+ out.append(
+ '
Capture quality and reward are '
+ "independent: a perfectly captured rollout can still score 0 because the model was "
+ "wrong, and reward — means the verifier never ran at all.
'
+ f"{html.escape(badge)} "
+ f'{convo.get("n_turns", 0)} model call(s), '
+ f"{len(rows)} message(s)
{''.join(rows)} "
+ )
+ if not blocks:
+ return ""
+ return (
+ f'Conversation '
+ f'everything the model saw and produced'
+ f"{''.join(blocks)}
"
+ )
+
+
+def _confidence(mean_logp: float) -> str:
+ """A bar for mean logprob. Closer to 0 is more confident; -1.0 is the practical floor here."""
+ pct = max(0.0, min(1.0, 1.0 + mean_logp)) # -0 -> 1.0, -1 -> 0.0
+ hue = 8 + int(112 * pct) # red through amber to green
+ return (
+ f''
+ f''
+ )
+
+
+def _turns_html(r: dict[str, Any]) -> str:
+ """Turn by turn: what it did, how much it wrote, how sure it was.
+
+ Replaces a table whose most prominent column was "tools", meaning the number of tools *offered*
+ to the model. That number is a property of the harness, identical on every row, and told nobody
+ anything. What varies per turn, and is worth reading, is the action taken, the tokens spent on
+ it, and the model's confidence while producing them.
+ """
+ turns = r.get("turns") or []
+ if not turns:
+ return 'No model calls were captured.
'
+
+ used: dict[str, int] = {}
+ for t in turns:
+ for call in t.get("tool_calls") or []:
+ name = str(call.get("name", "?"))
+ used[name] = used.get(name, 0) + 1
+
+ rows = []
+ for t in turns:
+ lp = t.get("per_token_logps") or []
+ mean = sum(lp) / len(lp) if lp else 0.0
+ gen = len(t.get("completion_token_ids") or [])
+ calls = t.get("tool_calls") or []
+ if calls:
+ action = " ".join(
+ f"{html.escape(str(c.get('name', 'tool')))}" for c in calls
+ )
+ elif t.get("finish_reason") == "stop":
+ action = 'final answer'
+ else:
+ action = 'text only'
+ note = (
+ ' discarded'
+ if t.get("discarded")
+ else ""
+ )
+ preview = _clip(t.get("text") or "", 160)
+ rows.append(
+ f''
+ f'| {t.get("turn")} | '
+ f"{action}{note}"
+ + (f' {preview} ' if preview else "")
+ + f' | {gen:,} | '
+ f"{_confidence(mean) if lp else ''} | "
+ f'{html.escape(str(t.get("finish_reason") or ""))} |
'
+ )
+
+ histogram = ""
+ if used:
+ top = sorted(used.items(), key=lambda kv: -kv[1])
+ histogram = (
+ 'tools used: '
+ + " ".join(f"{html.escape(k)}×{v}" for k, v in top)
+ + "
"
+ )
+
+ return (
+ 'Turn by turn'
+ '
| # | action | tokens | '
+ "confidence | stopped because |
|---|
"
+ + "".join(rows)
+ + "
"
+ + histogram
+ + '
Confidence is the mean logprob of the '
+ "sampled tokens: full bar means the model was near-certain, short means it was "
+ "guessing. Discarded turns were generated and billed but lead nowhere, so they are "
+ "excluded from training paths.
"
+ )
+
+
+def _write_contract(r: dict[str, Any]) -> str | None:
+ """Write `contract.json`: exactly what a trainer consumes, nothing else.
+
+ Per turn, `(prompt_token_ids, completion_token_ids, per_token_logps)` plus the reward. The
+ logprobs are the load-bearing part and the reason this is a separate file: they are the
+ behaviour policy's, recorded at sampling time, and cannot be recovered afterwards by re-running
+ the prompt. Discarded turns are kept but flagged, because they were generated and billed and a
+ trainer must be able to see them in order to exclude them deliberately.
+
+ Returns `None` for an eval rollout. Writing a file whose every `prompt_token_ids` is `[]` would
+ hand someone a download named `contract.json` containing no contract, and a file on disk is far
+ more convincing than an empty list in a JSON blob.
+ """
+ import tempfile
+ from pathlib import Path as _Path
+
+ turns = r.get("turns") or []
+ if not turns or r.get("rollout_type", "train") == "eval":
+ return None
+ from .contract import export_training_contract
+ from .models import HarborRolloutResult
+
+ contract = export_training_contract(HarborRolloutResult.model_validate(r))
+ name = re.sub(r"[^A-Za-z0-9_.-]", "_", str(r.get("task_name") or "rollout"))
+ target = (
+ _Path(tempfile.mkdtemp(prefix="harbor-contract-")) / f"{name}.contract.json"
+ )
+ target.write_text(json.dumps(contract, indent=2))
+ return str(target)
+
+
+def _summary_json(r: dict[str, Any]) -> str:
+ """The result with the token arrays summarised, which is the part anyone actually reads.
+
+ The full document stays available below; printing 8000 integers first buries the fields that
+ carry meaning.
+ """
+ compact = {k: v for k, v in r.items() if k not in ("turns", "conversations")}
+ compact["turns"] = [
+ {
+ "turn": t.get("turn"),
+ "action": [c.get("name") for c in (t.get("tool_calls") or [])] or "text",
+ "prompt_token_ids": f"<{len(t.get('prompt_token_ids') or [])} ids>",
+ "completion_token_ids": f"<{len(t.get('completion_token_ids') or [])} ids>",
+ "per_token_logps": f"<{len(t.get('per_token_logps') or [])} floats>",
+ "finish_reason": t.get("finish_reason"),
+ "discarded": t.get("discarded"),
+ "text": (t.get("text") or "")[:200],
+ }
+ for t in (r.get("turns") or [])[:200]
+ ]
+ compact["conversations"] = [
+ {
+ "role": c.get("role"),
+ "n_turns": c.get("n_turns"),
+ "messages": f"<{len(c.get('messages') or [])} messages>",
+ }
+ for c in (r.get("conversations") or [])
+ ]
+ return json.dumps(compact, indent=2)[:200_000]
+
+
+def _read(path: Any, limit: int = 20000) -> str:
+ try:
+ text = path.read_text(errors="replace")
+ except Exception: # noqa: BLE001
+ return ""
+ return text if len(text) <= limit else text[:limit] + "\n…truncated…"
+
+
+def harbor_gradio_builder(
+ *,
+ datasets: list[str] | None = None,
+ title: str | None = None,
+) -> gr.Blocks:
+ """Build the Harbor UI.
+
+ Args:
+ datasets (`list[str]`, *optional*):
+ Dataset specs served by this server; each becomes a selectable split.
+
+ Returns:
+ `gr.Blocks`: The interface.
+ """
+ from .tasks import HarborTaskProvider, resolve_task_dirs
+
+ datasets = list(datasets or [])
+
+ def on_validate(
+ url: str,
+ model: str,
+ api_key: str,
+ provider: str = "openai",
+ purpose: str = "eval",
+ include_experimental: bool = False,
+ ):
+ from openenv.core.harness.capture.validate_llm import list_models, validate_llm
+
+ from .capabilities import capabilities
+ from .seams import agent_facing_model, get as get_seam
+ from .serving import HarborService
+
+ url = (url or "").strip().rstrip("/")
+ api_key = (api_key or "").strip() or None
+ if not url:
+ return (
+ _UNVALIDATED,
+ gr.update(),
+ gr.update(),
+ {},
+ gr.update(interactive=False),
+ )
+
+ if not model:
+ served = list_models(url, api_key=api_key)
+ if len(served) != 1:
+ hint = (
+ f"`{', '.join(served[:12])}`"
+ if served
+ else "nothing reachable — check the URL, and the API key if it needs one"
+ )
+ return (
+ f"**Pick a model** — this endpoint serves {hint}.",
+ gr.update(),
+ gr.update(),
+ {},
+ gr.update(interactive=False),
+ )
+ model = served[0]
+
+ report = validate_llm(url, model, api_key=api_key, provider=provider)
+ if not report.reachable or (purpose == "train" and not report.trainable):
+ why = "; ".join(report.findings) or (
+ "Exact engine tokens are required for training"
+ if report.reachable
+ else "unreachable"
+ )
+ return (
+ f"**Not usable** — {why}\n\n"
+ "Needs vLLM with `--return-tokens-as-token-ids --logprobs-mode "
+ "processed_logprobs`, SGLang built from git main, or any reachable OpenAI-spec "
+ "endpoint (with an API key) for eval rollouts.",
+ gr.update(),
+ gr.update(),
+ {},
+ gr.update(interactive=False),
+ )
+
+ caps = capabilities(
+ datasets=datasets,
+ llm={
+ "url": url,
+ "model": model,
+ "ok": report.ok,
+ "capture_level": report.capture_level,
+ "reachable": True,
+ "authenticated": bool(api_key),
+ },
+ )
+ sandboxes = caps.available_sandboxes
+ from .qualification import harness_maturity_rows
+
+ try:
+ evidence = (
+ json.loads(Path(report_path).read_text()) if report_path else None
+ )
+ tiers = {
+ name: tier
+ for name, tier, _ in harness_maturity_rows(
+ [h.name for h in caps.harnesses], evidence
+ )
+ }
+ except (OSError, ValueError, TypeError):
+ tiers = {h.name: "experimental" for h in caps.harnesses}
+ evidence = None
+ profile_provider = "vllm" if purpose == "train" else provider
+ harness_profiles = {}
+ unavailable_profiles = set()
+ for cell in (evidence or {}).get("cells", []):
+ if cell.get("provider") != profile_provider:
+ continue
+ config = cell.get("configuration") or {}
+ profile = config.get("acp_profile") or config.get("nemo_profile")
+ if profile:
+ name = cell["harness"]
+ harness_profiles[name] = profile
+ try:
+ get_seam(name, profile=profile)
+ except (ValueError, KeyError):
+ unavailable_profiles.add(name)
+ choices = [
+ (
+ f"{h.name} ({h.dialect}; {tiers[h.name]}"
+ + (
+ f"; profile: {harness_profiles[h.name]}"
+ if h.name in harness_profiles
+ else ""
+ )
+ + ")",
+ h.name,
+ )
+ for h in sorted(caps.harnesses, key=lambda h: h.name)
+ if h.name not in unavailable_profiles
+ and (
+ tiers[h.name] == "stable"
+ or (include_experimental and tiers[h.name] == "experimental")
+ )
+ ]
+ values = [v for _, v in choices]
+
+ leaf = agent_facing_model(model)
+ if purpose == "train":
+ lines = [
+ f"**Endpoint ready — TRAINING CAPTURE** · `{model}` · token ids + logprobs ✓"
+ ]
+ else:
+ detail = (
+ "token capture available; training export disabled for this eval"
+ if report.capture_level == "tokens"
+ else "logprobs, no token ids"
+ if report.capture_level == "logprobs"
+ else "no token ids, no logprobs"
+ )
+ lines = [
+ f"**Endpoint ready — EVAL** · `{model}` · {detail}",
+ "Rollouts carry the reward and the full trace, but nothing trainable.",
+ ]
+ if leaf != model:
+ lines.append(f"Sent to agents as `{leaf}`, rewritten back on the way out.")
+ if not values:
+ lines.append(
+ "No agents match the support filter. Load qualification evidence or explicitly include experimental adapters."
+ )
+ for fix in report.param_fixes:
+ lines.append(f"upstream compat: {fix}")
+ # The one thing a user cannot discover by reading the endpoint's own docs: whether a model
+ # will actually sustain an agent loop here. Shown at Validate rather than after a rollout,
+ # because a rollout costs a sandbox and several minutes to learn the same thing.
+ for finding in report.findings:
+ if "behaviour_changed" in finding or "tool_call" in finding:
+ detail = finding.split(": ", 2)[-1]
+ lines.append(f"⚠️ {detail}")
+
+ # Run uses the endpoint typed above. The engine is a per-rollout argument, so a browser can
+ # point this server at any reachable OpenAI-spec endpoint without restarting it — which is
+ # the whole point of validating a URL here. Say which one will be used, because a server may
+ # also have been booted with a default and the two can differ.
+ service = HarborService.current()
+ if (
+ service is not None
+ and service.llm_url
+ and service.llm_url.rstrip("/") != url
+ ):
+ lines.append(
+ f"Rollouts will use **this** endpoint, not the server's default "
+ f"(`{service.llm_url}`)."
+ )
+ lines.append(
+ f"Sandboxes: {', '.join(f'`{s}`' for s in sandboxes) or '**none usable**'}"
+ )
+ blocked = [s.name for s in caps.sandboxes if not s.available]
+ if blocked:
+ lines.append(
+ f"unavailable: {', '.join(blocked)}"
+ )
+
+ return (
+ " \n".join(lines),
+ gr.update(
+ choices=choices,
+ value="opencode"
+ if "opencode" in values
+ else (values[0] if values else None),
+ ),
+ gr.update(choices=sandboxes, value=sandboxes[0] if sandboxes else None),
+ # `ok` gates the Run button and now means "reachable", not "trainable": an eval endpoint
+ # is a perfectly good thing to press Run against.
+ {
+ "url": url,
+ "model": model,
+ "ok": True,
+ "capture_level": report.capture_level,
+ "trainable": report.trainable,
+ # Carried so Run can reach a token-gated endpoint. Without it, validating a hosted
+ # provider succeeded and pressing Run then failed to authenticate against the same
+ # URL. `gr.State` is held server-side and this is never rendered back into the page,
+ # which is the same rule the API key box itself follows.
+ "api_key": api_key or "",
+ "provider": provider,
+ "purpose": purpose,
+ "allowed_harnesses": values,
+ "harness_profiles": harness_profiles,
+ },
+ gr.update(
+ interactive=bool(sandboxes and values),
+ value="Run training capture"
+ if purpose == "train"
+ else "Run eval rollout",
+ ),
+ )
+
+ def on_dataset(spec: str):
+ if not spec:
+ return gr.update(), ""
+ try:
+ n = len(resolve_task_dirs(spec))
+ except Exception as exc: # noqa: BLE001
+ return gr.update(value=0), f"Cannot load `{spec}` — {exc}"
+ return gr.update(value=0), f"**{n}** tasks · 0–{n - 1}"
+
+ def on_task(spec: str, index: int):
+ if not spec:
+ return "", "", "", "", ""
+ try:
+ task_dir = HarborTaskProvider([spec]).task_dir(spec, int(index))
+ except Exception as exc: # noqa: BLE001
+ return f"_{exc}_", "", "", "", ""
+ env_dir, tests_dir = task_dir / "environment", task_dir / "tests"
+ return (
+ f"`{task_dir.name}`",
+ _read(task_dir / "instruction.md"),
+ _read(env_dir / "Dockerfile"),
+ _read(task_dir / "task.toml"),
+ _read(tests_dir / "test.sh"),
+ )
+
+ def on_run(engine: dict, spec: str, index: int, harness: str, sandbox: str):
+ """Stream progress while the rollout runs, then the result and its graph."""
+ import asyncio
+ import queue
+ import threading
+ import time
+ from pathlib import Path
+
+ from .rollout import run_rollout as _run
+ from .serving import HarborService
+
+ if not engine.get("ok"):
+ yield _UNVALIDATED, "", "", "{}", None, gr.update(interactive=True)
+ return
+ if harness not in engine.get("allowed_harnesses", []):
+ yield (
+ "Selected harness is outside the validated support filter. Validate again.",
+ "",
+ "",
+ "{}",
+ None,
+ gr.update(interactive=False),
+ )
+ return
+ service = HarborService.current()
+ if service is None:
+ yield (
+ "Server not initialised — no capture proxy running.",
+ "",
+ "",
+ "{}",
+ None,
+ gr.update(interactive=True),
+ )
+ return
+ try:
+ task_dir = HarborTaskProvider([spec]).task_dir(spec, int(index))
+ except Exception as exc: # noqa: BLE001
+ yield (
+ f"Bad task — {html.escape(str(exc))}",
+ "",
+ "",
+ "{}",
+ None,
+ gr.update(interactive=True),
+ )
+ return
+
+ done: queue.Queue = queue.Queue(maxsize=1)
+ live_sessions: queue.Queue[str] = queue.Queue(maxsize=1)
+
+ async def _run_with_engine():
+ """Resolve the engine the user validated, then run against it.
+
+ The engine is per rollout, so the URL in the box is the one used. Resolving it through the
+ capture server's pool means the tier comes from a real probe of that endpoint rather than
+ from whatever the server happened to boot with — and the probe is cached, so pressing Run
+ repeatedly costs nothing after the first time.
+ """
+ from openenv.core.harness.capture.sessions import Upstream
+
+ pool = service.capture.app.state.upstreams
+ typed_url = str((engine or {}).get("url") or "").strip()
+ if typed_url:
+ upstream = Upstream(
+ llm_url=typed_url,
+ model=str((engine or {}).get("model") or ""),
+ api_key=str((engine or {}).get("api_key") or "") or None,
+ provider=str((engine or {}).get("provider") or "openai"),
+ )
+ client, level = await pool.resolve(upstream)
+ served = client.served_model or upstream.model
+ else:
+ # Nothing validated in the box: fall back to the server's default, which is what a
+ # server booted with --llm-url provides. With neither, the rollout reports the
+ # missing engine rather than silently producing an untrainable result.
+ upstream, (client, level) = None, pool.default
+ level = getattr(service, "capture_level", "text")
+ served = service.model
+ return await _run(
+ task_dir=task_dir,
+ harness=harness,
+ harness_profile=engine.get("harness_profiles", {}).get(harness),
+ sandbox=sandbox,
+ registry=service.capture.registry,
+ intercept_url=service.public_url,
+ model=served,
+ trials_dir=Path("/tmp/openenv-harbor-trials"),
+ dataset=spec,
+ capture_level=level,
+ purpose=str((engine or {}).get("purpose") or "eval"),
+ upstream=upstream,
+ inference=client,
+ on_session_created=live_sessions.put_nowait,
+ )
+
+ def worker() -> None:
+ try:
+ res = asyncio.run(_run_with_engine())
+ done.put(("ok", res.model_dump()))
+ except Exception as exc: # noqa: BLE001 - show it, never take the server down
+ done.put(("err", f"{type(exc).__name__}: {exc}"))
+
+ thread = threading.Thread(target=worker, daemon=True)
+ started = time.monotonic()
+ thread.start()
+ session_id = None
+
+ while thread.is_alive():
+ if session_id is None:
+ try:
+ session_id = live_sessions.get_nowait()
+ except queue.Empty:
+ pass
+ stats, phase, stage = None, "starting up", 0
+ if session_id:
+ session = service.capture.registry.get(session_id)
+ if session is not None:
+ st = session.graph.stats()
+ # n_trainable_tokens only exists after export; mid-run we can count only what
+ # has been sampled, before masking and discards.
+ sampled = sum(
+ len(n.sampled_ids or []) for n in session.graph.nodes()
+ )
+ turns = st.get("n_turns", 0)
+ stats = {
+ "calls": turns,
+ "roots": st.get("n_roots", 0),
+ "sampled tokens": sampled,
+ "discarded": st.get("n_discarded", 0),
+ }
+ if turns:
+ stage = -1 # past setup; the numbers mean something now
+ phase = "agent working"
+ # Only meaningful once a call has landed. Before that `idle_seconds` counts
+ # from session creation, which renders as a stall during a normal boot.
+ stats["since last call"] = f"{session.idle_seconds:.0f}s"
+ else:
+ stage = 0
+ phase = "preparing the run or waiting for its first response"
+ # The transcript rides in the graph slot: it is empty until the run finishes anyway,
+ # and the two answer the same question at different times.
+ transcript = ""
+ if session_id:
+ live = service.capture.registry.get(session_id)
+ if live is not None:
+ transcript = _transcript_html(live)
+ yield (
+ _live_html(
+ harness, sandbox, phase, time.monotonic() - started, stats, stage
+ ),
+ transcript,
+ "",
+ "{}",
+ None,
+ gr.update(interactive=False),
+ )
+ time.sleep(2.0)
+
+ kind, payload = done.get()
+ if kind == "err":
+ yield (
+ f'Run failed
'
+ f'
{html.escape(payload)}Training export rejected: " + html.escape(str(exc)) + "
"
+ )
+ yield (
+ _result_html(payload) + contract_error,
+ _conversation_html(payload),
+ _turns_html(payload),
+ _summary_json(payload),
+ contract,
+ gr.update(interactive=True),
+ )
+
+ with gr.Blocks(title=title or "Harbor") as app:
+ gr.HTML(f"")
+ state = gr.State({})
+
+ with gr.Column(elem_classes="hb-wrap"):
+ gr.Markdown(
+ "## Harbor task playground\nChoose a model, validate the connection, then run an agent "
+ "on a task. Follow its tool calls and results below. "
+ "Evaluation works with supported hosted providers; training capture requires "
+ "verified engine token IDs and log probabilities."
+ )
+
+ with gr.Row(equal_height=False):
+ # left — the model
+ with gr.Column(scale=1, elem_classes="hb-cell"):
+ with gr.Column(elem_classes="hb-panel"):
+ gr.Markdown("### 1 · Connect a model")
+ # Deliberately empty. Prefilling meant the box already held whatever URL the
+ # server was started with, so Validate confirmed a value nobody chose and a
+ # stale endpoint could be used without anyone noticing it was stale.
+ provider_in = gr.Dropdown(
+ label="Upstream provider",
+ choices=[
+ ("OpenAI-compatible", "openai"),
+ ("Anthropic native", "anthropic"),
+ ("Hugging Face Inference Providers", "hf"),
+ ("vLLM", "vllm"),
+ ],
+ value="openai",
+ info="Select the upstream API. Exact training tokens are verified separately.",
+ )
+ purpose_in = gr.Dropdown(
+ label="Use",
+ choices=[
+ ("Evaluation", "eval"),
+ ("Training capture", "train"),
+ ],
+ value="eval",
+ )
+ url_in = gr.Textbox(
+ label="LLM URL",
+ placeholder="https://your-endpoint/v1",
+ info="vLLM, SGLang, OpenAI, Anthropic, HF Inference Providers. "
+ "Accepts a bare root or one ending in /v1.",
+ )
+ gr.HTML(_labelled("API key (optional)", _KEY_TIP))
+ key_in = gr.Textbox(
+ label="",
+ type="password",
+ placeholder="only for a hosted provider",
+ show_label=False,
+ )
+ model_in = gr.Textbox(
+ label="Model (optional)",
+ placeholder="read from the endpoint",
+ info="Required when the endpoint serves more than one model.",
+ )
+ validate_btn = gr.Button(
+ "Validate connection", variant="secondary"
+ )
+ gr.HTML(_labelled("Capture level", _LEVEL_TIP))
+ engine_md = gr.Markdown(_UNVALIDATED)
+
+ # right — task preview and agent. Implementation files stay folded away.
+ with gr.Column(scale=1, elem_classes="hb-cell"):
+ gr.Markdown("### 2 · Choose a task")
+ with gr.Row():
+ ds_in = gr.Dropdown(
+ label="Dataset",
+ choices=datasets,
+ value=datasets[0] if datasets else None,
+ scale=3,
+ )
+ idx_in = gr.Number(
+ label="Index", value=0, precision=0, minimum=0, scale=1
+ )
+ count_md = gr.Markdown()
+ task_md = gr.Markdown()
+ instruction_box = gr.Code(
+ label="Task instruction", language="markdown", lines=10
+ )
+ with gr.Accordion("Task files and grader", open=False):
+ with gr.Accordion("Dockerfile", open=False):
+ dockerfile_box = gr.Code(
+ label="", language="dockerfile", lines=12
+ )
+ with gr.Accordion("task.toml", open=False):
+ toml_box = gr.Code(label="", language="python", lines=12)
+ with gr.Accordion("Grader", open=False):
+ tests_box = gr.Code(label="", language="shell", lines=12)
+
+ with gr.Column(elem_classes="hb-panel"):
+ gr.Markdown("### 3 · Choose an agent")
+ experimental_in = gr.Checkbox(
+ label="Include experimental adapters",
+ value=False,
+ info="Stable adapters are shown by default. Unstable adapters remain excluded.",
+ )
+ harness_in = gr.Dropdown(
+ label="Agent",
+ choices=[],
+ info="The coding agent to run. Its dialect is shown in brackets; the "
+ "capture proxy connects it to your selected provider.",
+ )
+ sandbox_in = gr.Dropdown(
+ label="Sandbox",
+ choices=[],
+ info="Where the agent executes. Harbor's backends, not OpenEnv's "
+ "container providers — only those with working credentials are listed.",
+ )
+
+ # Full width, under both columns: the action belongs to the pair, not to either one.
+ run_btn = gr.Button(
+ "Run rollout", variant="primary", interactive=False, scale=1
+ )
+
+ with gr.Column(elem_classes="hb-panel"):
+ gr.Markdown("### Run status")
+ result_html = gr.HTML(
+ 'Validate your model and choose a task to begin.
'
+ )
+ with gr.Column(elem_classes="hb-panel"):
+ gr.Markdown(
+ "### Live trace\nAgent messages, tool calls and tool results appear as "
+ "model calls complete. A request in progress may take a moment."
+ )
+ convo_html = gr.HTML()
+ with gr.Accordion("Token details and training export", open=False):
+ analysis_html = gr.HTML()
+ contract_file = gr.File(
+ label="Training contract — captured tokens, log probabilities and reward",
+ interactive=False,
+ visible=True,
+ )
+ with gr.Accordion("Harness/provider qualification evidence", open=False):
+ import os
+ from pathlib import Path
+
+ from .qualification import (
+ harness_maturity_rows,
+ qualification_details,
+ qualification_rows,
+ )
+ from .seams import SEAMS
+
+ report_path = os.environ.get("OPENENV_HARBOR_QUALIFICATION_REPORT", "")
+
+ def read_qualification_evidence():
+ try:
+ evidence = (
+ json.loads(Path(report_path).read_text())
+ if report_path
+ else None
+ )
+ return (
+ qualification_rows(list(SEAMS), evidence),
+ qualification_details(evidence),
+ harness_maturity_rows(list(SEAMS), evidence),
+ "Loaded recorded evidence."
+ if evidence
+ else "No qualification report configured.",
+ )
+ except (OSError, ValueError, TypeError) as exc:
+ return (
+ qualification_rows(list(SEAMS)),
+ [],
+ harness_maturity_rows(list(SEAMS)),
+ "Invalid qualification report: " + str(exc),
+ )
+
+ evidence_rows, evidence_details, maturity_rows, evidence_status = (
+ read_qualification_evidence()
+ )
+ gr.Markdown(
+ "Recorded results apply to the listed model, harness version, and captures. "
+ "They do not certify the endpoint currently selected above. "
+ "Capture/reader passes exclude optimizer validation; optimizer details state "
+ "whether the test used diagnostic replay and whether it covered weight sync."
+ )
+ evidence_status_md = gr.Markdown(evidence_status)
+ gr.Markdown(
+ "Stable means all four recorded profiles passed, including optimizer replay. "
+ "It is limited to this test coverage, not a production-scale guarantee. "
+ "Experimental adapters have partial or pending support; unstable adapters "
+ "have no passing profile in the recorded matrix."
+ )
+ maturity_table = gr.Dataframe(
+ headers=["Harness", "Maturity", "Qualification scope"],
+ value=maturity_rows,
+ interactive=False,
+ )
+ evidence_table = gr.Dataframe(
+ headers=[
+ "Harness",
+ "OpenAI eval",
+ "Anthropic eval",
+ "HF eval",
+ "vLLM training",
+ ],
+ value=evidence_rows,
+ interactive=False,
+ )
+ evidence_detail_table = gr.Dataframe(
+ headers=[
+ "Harness",
+ "Provider",
+ "Status",
+ "Model",
+ "Harness version",
+ "Tasks",
+ "Workflow profile",
+ "Optimizer scope",
+ "Optimizer model revision",
+ "Capture evidence",
+ "Reason",
+ ],
+ value=evidence_details,
+ interactive=False,
+ )
+ refresh_evidence = gr.Button("Refresh recorded evidence")
+ refresh_evidence.click(
+ read_qualification_evidence,
+ [],
+ [
+ evidence_table,
+ evidence_detail_table,
+ maturity_table,
+ evidence_status_md,
+ ],
+ )
+ with gr.Accordion("Result JSON", open=False):
+ raw_json = gr.Code(language="json", lines=22)
+
+ validate_btn.click(
+ on_validate,
+ [url_in, model_in, key_in, provider_in, purpose_in, experimental_in],
+ [engine_md, harness_in, sandbox_in, state, run_btn],
+ )
+ for setting in (
+ url_in,
+ model_in,
+ key_in,
+ provider_in,
+ purpose_in,
+ experimental_in,
+ ):
+ setting.change(
+ lambda: ({}, gr.update(interactive=False), _UNVALIDATED),
+ outputs=[state, run_btn, engine_md],
+ )
+ ds_in.change(on_dataset, [ds_in], [idx_in, count_md])
+ ds_in.change(
+ on_task,
+ [ds_in, idx_in],
+ [task_md, instruction_box, dockerfile_box, toml_box, tests_box],
+ )
+ idx_in.change(
+ on_task,
+ [ds_in, idx_in],
+ [task_md, instruction_box, dockerfile_box, toml_box, tests_box],
+ )
+ run_btn.click(
+ on_run,
+ [state, ds_in, idx_in, harness_in, sandbox_in],
+ [result_html, convo_html, analysis_html, raw_json, contract_file, run_btn],
+ )
+
+ if datasets:
+ app.load(on_dataset, [ds_in], [idx_in, count_md])
+ app.load(
+ on_task,
+ [ds_in, idx_in],
+ [task_md, instruction_box, dockerfile_box, toml_box, tests_box],
+ )
+ return app
diff --git a/tests/envs/test_capture_inprocess_trace.py b/tests/envs/test_capture_inprocess_trace.py
new file mode 100644
index 0000000000..6834a3aaba
--- /dev/null
+++ b/tests/envs/test_capture_inprocess_trace.py
@@ -0,0 +1,177 @@
+# Copyright 2026 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Reading a rollout back out of a live `CaptureServer`, in process.
+
+This is the path `CaptureServer` exists for, and why it runs as a thread rather than a subprocess:
+the caller mints a session on the registry the proxy is writing into, then reads the graph straight
+back out of it. Going through HTTP for that would add a serialisation round trip and a failure mode
+for no benefit -- and it is also how a caller ends up never deleting the session, because over HTTP
+there is no obvious place to.
+
+The property under test is the one the whole training contract rests on: turn k+1's prompt IS turn
+k's prompt plus its completion, so turns link by exact token prefix. When that breaks, one
+conversation silently fragments into several short ones and every fragment still trains.
+
+A stub engine stands in for vLLM so this needs no GPU.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import pytest
+from fastapi.testclient import TestClient
+from openenv.core.harness.capture import to_trace_entries
+from openenv.core.harness.capture.export import export_session
+from openenv.core.harness.capture.runner import CaptureServer
+from openenv.core.harness.capture.sessions import Upstream
+
+
+LLM_URL = "http://engine.invalid/v1"
+MODEL = "test-model"
+
+
+class _TokenEngine:
+ """A vLLM served with `--return-tokens-as-token-ids --logprobs-mode processed_logprobs`."""
+
+ served_model = MODEL
+ param_fixes: dict[str, Any] = {}
+ capture_level = "tokens"
+
+ def __init__(self) -> None:
+ self.turn = 0
+ # The prompt grows by the previous turn's completion. Faking that relationship is the only
+ # way the graph's prefix linking can be exercised at all.
+ self._prompt = [1, 2, 3]
+
+ async def completion(self, request: dict[str, Any]) -> dict[str, Any]:
+ self.turn += 1
+ prompt = list(self._prompt)
+ completion = [100 + self.turn, 200 + self.turn]
+ self._prompt = prompt + completion
+ return {
+ "id": f"c{self.turn}",
+ "object": "chat.completion",
+ "model": MODEL,
+ # `token_ids` on the choice, `prompt_token_ids` on the response: the shape vLLM returns
+ # under `--return-tokens-as-token-ids`.
+ "prompt_token_ids": prompt,
+ "choices": [
+ {
+ "index": 0,
+ "message": {"role": "assistant", "content": f"turn {self.turn}"},
+ "finish_reason": "stop",
+ "token_ids": completion,
+ "logprobs": {"content": [{"logprob": -0.5} for _ in completion]},
+ }
+ ],
+ "usage": {
+ "prompt_tokens": len(prompt),
+ "completion_tokens": 2,
+ "total_tokens": 0,
+ },
+ }
+
+
+@pytest.fixture
+def server():
+ """A `CaptureServer` that is never `start()`ed -- the registry is what this exercises.
+
+ Binding a port would make the test flaky on a busy machine and would test uvicorn rather than the
+ contract.
+ """
+ srv = CaptureServer(llm_url=LLM_URL, model=MODEL)
+ engine = _TokenEngine()
+ # Seed the ENGINE POOL, not only the default: a session that names its own upstream resolves
+ # through the pool, which is what lets one server drive a train-tier engine and an eval-tier one
+ # at the same time. Without this the proxy would probe `engine.invalid` for real.
+ srv.app.state.upstreams._by_engine[
+ Upstream(llm_url=LLM_URL, model=MODEL).cache_key
+ ] = (
+ engine,
+ "tokens",
+ )
+ srv.app.state.upstreams._default = (engine, "tokens")
+ return srv
+
+
+def _mint(server, **kwargs):
+ return server.registry.create(
+ upstream=Upstream(llm_url=LLM_URL, model=MODEL),
+ capture_level="tokens",
+ **kwargs,
+ )
+
+
+def _chat(client: TestClient, session_id: str) -> None:
+ client.post(
+ "/v1/chat/completions",
+ headers={"Authorization": f"Bearer {session_id}"},
+ json={"model": MODEL, "messages": [{"role": "user", "content": "hi"}]},
+ )
+
+
+def _entries(server, session):
+ document = export_session(
+ session, include_messages=True, capture_level=session.capture_level
+ )
+ return to_trace_entries(session.graph, document)
+
+
+def test_entries_carry_the_engines_own_prompt_tokens(server):
+ session = _mint(server)
+ with TestClient(server.app) as client:
+ for _ in range(3):
+ _chat(client, session.session_id)
+
+ entries = _entries(server, session)
+ assert len(entries) == 3
+ for entry in entries:
+ assert entry["prompt_token_ids"], (
+ "an entry came back with no engine tokenisation"
+ )
+ assert entry["completion_token_ids"]
+ assert len(entry["per_token_logps"]) == len(entry["completion_token_ids"])
+ # The mask spans prompt + completion, and only the completion is trained.
+ assert len(entry["loss_mask"]) == len(entry["prompt_token_ids"]) + len(
+ entry["completion_token_ids"]
+ )
+ assert set(entry["loss_mask"][: len(entry["prompt_token_ids"])]) == {0}
+
+ # THE CONTRACT: turn k+1's prompt is everything before it, token for token.
+ first, second = entries[0], entries[1]
+ assert (
+ second["prompt_token_ids"]
+ == first["prompt_token_ids"] + first["completion_token_ids"]
+ )
+
+
+def test_deleting_a_session_releases_it(server):
+ session = _mint(server)
+ sid = session.session_id
+ assert server.registry.get(sid) is not None
+ assert server.registry.delete(sid)
+ # Sessions held past their rollout collide with the next run's claim and surface as a burst of
+ # CAPACITY_REACHED on a server that looks idle, so this is not bookkeeping.
+ assert server.registry.get(sid) is None
+
+
+def test_a_budget_bounds_what_is_captured(server):
+ session = _mint(server, max_model_calls=2)
+ with TestClient(server.app) as client:
+ for _ in range(5):
+ _chat(client, session.session_id)
+ # Five requests, two captured: the proxy answered the rest itself, before ingest.
+ assert len(_entries(server, session)) == 2
diff --git a/tests/envs/test_capture_logprobs_mode_sentinel.py b/tests/envs/test_capture_logprobs_mode_sentinel.py
new file mode 100644
index 0000000000..d30f7396dc
--- /dev/null
+++ b/tests/envs/test_capture_logprobs_mode_sentinel.py
@@ -0,0 +1,86 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""`probe_logprobs_mode` must say "unknown", not "raw", when the distribution is saturated.
+
+The probe decides raw-vs-processed from how the top-two logprob GAP scales with temperature: `T1/T2`
+when processed, `1.0` when raw. It measures at completion position 1 because that is the only
+position independent of sampling.
+
+But a reasoning model's chat template FORCES its opening token -- `` for Qwen3, at p~1.0 with
+every alternative at -inf. Engines report -inf as a sentinel (vLLM: -9999), and +-inf is unchanged by
+division, so the gap is identical at both temperatures and the ratio is 1.0. The probe then reports
+"raw", `validate_llm` demotes `capture_level` from `tokens` to `logprobs`, and every rollout comes
+back 409 -- which reads as "this model cannot train" when the truth is "this probe cannot measure
+here".
+
+Measured on a live Qwen3-8B served WITH `--logprobs-mode processed_logprobs`:
+
+ position 1 (forced ``) gap 9999.0 @T=1.0 -> 9999.0 @T=2.0 ratio 1.000
+ position 1, prefilled past `` gap 3.7500 @T=1.0 -> 1.8750 @T=2.0 ratio 0.500
+
+So "raw" was wrong about a correctly-configured engine. "unknown" is what the function documents for
+this case: not a failure, only an absence of evidence -- and unlike "raw" it does not demote the tier.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+validate_llm = pytest.importorskip("openenv.core.harness.capture.validate_llm")
+
+
+def _payload(top: list[float]) -> dict:
+ """A chat completion whose first position carries `top` as its `top_logprobs`."""
+ return {
+ "choices": [
+ {"logprobs": {"content": [{"top_logprobs": [{"logprob": v} for v in top]}]}}
+ ]
+ }
+
+
+def _probe_with(monkeypatch, per_temperature: list[list[float]]) -> str:
+ """Run the probe against canned responses, one per temperature it asks about."""
+ answers = list(per_temperature)
+
+ def fake_post(url, body, timeout, api_key=None, auth_header="Authorization"):
+ return _payload(answers.pop(0))
+
+ monkeypatch.setattr(validate_llm, "_post", fake_post)
+ return validate_llm.probe_logprobs_mode("http://engine", "some/model")
+
+
+def test_saturated_gap_is_unknown_not_raw(monkeypatch):
+ # The real shape: top token at 0.0, runners-up at the -inf sentinel, unchanged by temperature.
+ mode = _probe_with(monkeypatch, [[0.0, -9999.0, -9999.0], [0.0, -9999.0, -9999.0]])
+ assert mode == "unknown", (
+ "a sentinel gap is an absence of evidence; calling it 'raw' demotes a correctly-configured "
+ "engine to the eval tier and every rollout then 409s"
+ )
+
+
+def test_genuinely_raw_is_still_detected(monkeypatch):
+ # The guard must not blind the check it lives in: a real, unchanging gap is still raw.
+ mode = _probe_with(monkeypatch, [[-1.0, -7.75], [-1.0, -7.75]])
+ assert mode == "raw"
+
+
+def test_processed_is_still_detected(monkeypatch):
+ # Gap halves as temperature doubles -> processed. Matches the measured 3.75 -> 1.875.
+ mode = _probe_with(monkeypatch, [[-1.0, -4.75], [-1.0, -2.875]])
+ assert mode == "processed"
+
+
+def test_flat_distribution_remains_unknown(monkeypatch):
+ # The pre-existing guard at the other extreme, kept honest alongside the new one.
+ mode = _probe_with(monkeypatch, [[-1.0, -1.2], [-1.0, -1.1]])
+ assert mode == "unknown"
+
+
+def test_sentinel_threshold_admits_real_tail_values(monkeypatch):
+ # A deep but REAL tail value must still be measured, or the guard would swallow valid data.
+ mode = _probe_with(monkeypatch, [[-1.0, -41.0], [-1.0, -21.0]])
+ assert mode == "processed"
diff --git a/tests/envs/test_capture_model_call_budget.py b/tests/envs/test_capture_model_call_budget.py
new file mode 100644
index 0000000000..43ce1826f3
--- /dev/null
+++ b/tests/envs/test_capture_model_call_budget.py
@@ -0,0 +1,297 @@
+# Copyright 2026 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""The per-session model-call budget.
+
+Written because the thing it replaces was imaginary. `agent.build.steps` is a real key in opencode's
+schema and is simply not honoured -- measured against a fake engine that always asks for one more
+tool call, `steps=3`, `maxSteps=3` and no setting at all each produced 61 model calls. So the tests
+that matter here are the two that distinguish a real cap from a decorative one: that the (n+1)th call
+is never forwarded, and that it never enters the capture graph.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import pytest
+from fastapi.testclient import TestClient
+from openenv.core.harness.capture.server import create_app
+from openenv.core.harness.capture.upstream import UpstreamHTTPError
+
+
+class _CountingEngine:
+ """Stands in for vLLM. Records what it was asked to do, so 'not forwarded' is observable."""
+
+ def __init__(self) -> None:
+ self.calls = 0
+ self.served_model = "test-model"
+ self.param_fixes: dict[str, Any] = {}
+ self.capture_level = "text"
+
+ async def completion(self, request: dict[str, Any]) -> dict[str, Any]:
+ self.calls += 1
+ return {
+ "id": f"c{self.calls}",
+ "object": "chat.completion",
+ "model": self.served_model,
+ "choices": [
+ {
+ "index": 0,
+ "message": {"role": "assistant", "content": f"turn {self.calls}"},
+ "finish_reason": "stop",
+ }
+ ],
+ "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
+ }
+
+
+@pytest.fixture
+def app_and_engine():
+ app = create_app(
+ llm_url="http://engine.invalid/v1", model="test-model", capture_level="text"
+ )
+ engine = _CountingEngine()
+ app.state.inference = engine
+ # The pool captured the real client at create_app time, so replacing `app.state.inference` alone
+ # leaves every request going to `engine.invalid`. Sessions here name no upstream, so they take the
+ # pool's default and this is the hook that matters.
+ app.state.upstreams._default = (engine, "text")
+ return app, engine
+
+
+def _chat(client: TestClient, session_id: str) -> Any:
+ return client.post(
+ "/v1/chat/completions",
+ headers={"Authorization": f"Bearer {session_id}"},
+ json={"model": "test-model", "messages": [{"role": "user", "content": "hi"}]},
+ )
+
+
+@pytest.mark.parametrize("stream", [False, True])
+def test_context_limit_stops_without_fabricating_a_captured_turn(
+ app_and_engine, stream
+):
+ app, engine = app_and_engine
+ session = app.state.registry.create(max_model_calls=17)
+ with TestClient(app) as client:
+ _chat(client, session.session_id)
+ before = session.graph.stats()["n_turns"]
+
+ async def too_long(request):
+ raise UpstreamHTTPError(
+ 400,
+ {
+ "error": {
+ "message": (
+ "This model's maximum context length is 131072 tokens. "
+ "However, you requested 4096 output tokens and your prompt contains "
+ "at least 126977 input tokens."
+ )
+ }
+ },
+ )
+
+ engine.completion = too_long
+ response = client.post(
+ "/v1/chat/completions",
+ headers={"Authorization": f"Bearer {session.session_id}"},
+ json={
+ "model": "test-model",
+ "messages": [{"role": "user", "content": "long"}],
+ "stream": stream,
+ },
+ )
+ assert response.status_code == 200
+ assert session.budget_stop_count == 1
+ assert session.upstream_errors == 0
+ assert session.graph.stats()["n_turns"] == before
+ assert any("context_budget_exhausted" in finding for finding in session.findings)
+ from openenv.core.harness.capture.export import export_session
+
+ document = export_session(session, capture_level="text")
+ assert not any(
+ "degenerate_rollout" in finding for finding in document["validation"]
+ )
+ if stream:
+ assert response.headers["content-type"].startswith("text/event-stream")
+ assert '"finish_reason":"stop"' in response.text.replace(" ", "")
+ else:
+ assert response.json()["choices"][0]["finish_reason"] == "stop"
+
+
+@pytest.mark.parametrize(
+ "upstream_status, expected_status", [(400, 400), (413, 413), (422, 422), (500, 502)]
+)
+def test_other_upstream_errors_are_not_converted_to_budget_stops(
+ app_and_engine, upstream_status, expected_status
+):
+ app, engine = app_and_engine
+ session = app.state.registry.create()
+
+ async def invalid(request):
+ raise UpstreamHTTPError(
+ upstream_status, {"error": {"message": "invalid tools"}}
+ )
+
+ engine.completion = invalid
+ with TestClient(app) as client:
+ assert _chat(client, session.session_id).status_code == expected_status
+ assert session.budget_stop_count == 0
+ assert session.upstream_errors == 1
+ assert session.graph.stats()["n_turns"] == 0
+
+
+def test_single_turn_without_recorded_budget_stop_still_fails(app_and_engine):
+ from openenv.core.harness.capture.export import export_session
+
+ app, _ = app_and_engine
+ session = app.state.registry.create()
+ with TestClient(app) as client:
+ _chat(client, session.session_id)
+ document = export_session(session, capture_level="text")
+ assert any(
+ "[FATAL] degenerate_rollout" in finding for finding in document["validation"]
+ )
+
+
+def test_budget_stop_does_not_make_an_empty_capture_valid(app_and_engine):
+ from openenv.core.harness.capture.export import export_session
+
+ app, _ = app_and_engine
+ session = app.state.registry.create()
+ session.budget_stop_count = 1
+ document = export_session(session, capture_level="text")
+ assert any("[FATAL] no_turns" in finding for finding in document["validation"])
+
+
+def test_budget_stops_forwarding_at_the_cap(app_and_engine):
+ app, engine = app_and_engine
+ session = app.state.registry.create(max_model_calls=3)
+ with TestClient(app) as client:
+ for _ in range(5):
+ assert _chat(client, session.session_id).status_code == 200
+ # Five requests, three forwarded. Without the cap the engine would see all five.
+ assert engine.calls == 3
+ assert session.model_calls == 3
+
+
+def test_the_capped_turn_is_terminal_and_never_captured(app_and_engine):
+ app, engine = app_and_engine
+ session = app.state.registry.create(max_model_calls=1)
+ with TestClient(app) as client:
+ _chat(client, session.session_id)
+ over = _chat(client, session.session_id).json()
+
+ # Terminal: this is what actually ends the agent's loop. opencode exits 0 on it.
+ assert over["choices"][0]["finish_reason"] == "stop"
+ assert not over["choices"][0]["message"].get("tool_calls")
+ # Non-empty: an empty assistant message reads as a failed generation and is retried. See
+ # `test_the_stop_message_is_not_empty`.
+ assert over["choices"][0]["message"]["content"].strip()
+ # And it is not in the graph. A synthetic turn in the training data is the failure this guards.
+ assert session.graph.stats()["n_turns"] == 1
+
+ # The harness may put the terminal response in ATIF. The independent cross-check needs
+ # explicit evidence that this zero-token step came from the proxy, rather than the model.
+ from openenv.core.harness.capture.export import export_session
+ from openenv.harbor.atif import reconcile
+
+ document = export_session(session, capture_level="text")
+ trace = {
+ "steps": [
+ {
+ "source": "agent",
+ "message": "turn 1",
+ "metrics": {"completion_tokens": 1},
+ },
+ {
+ "source": "agent",
+ "message": over["choices"][0]["message"]["content"],
+ "metrics": over["usage"],
+ },
+ ]
+ }
+ assert document["budget_stop_count"] == 1
+ assert not any(
+ "degenerate_rollout" in finding for finding in document["validation"]
+ )
+ report = reconcile(document, trace)
+ assert "proxy_budget_stops" in {f.code for f in report.findings}
+ assert "atif_calls_missing" not in {f.code for f in report.findings}
+
+
+def test_the_stop_is_streamed_when_the_caller_streams(app_and_engine):
+ """A streaming caller must get SSE back, not a JSON body.
+
+ This is the failure that made the cap useless in practice. opencode streams; answering it with a
+ plain JSON body did not end its loop, so it retried, the proxy answered the stop again, and the
+ rollout spun until its timeout -- "budget enforced" in the log, forever.
+ """
+ app, engine = app_and_engine
+ session = app.state.registry.create(max_model_calls=1)
+ with TestClient(app) as client:
+ _chat(client, session.session_id) # spends the budget
+ over = client.post(
+ "/v1/chat/completions",
+ headers={"Authorization": f"Bearer {session.session_id}"},
+ json={
+ "model": "test-model",
+ "messages": [{"role": "user", "content": "hi"}],
+ "stream": True,
+ },
+ )
+
+ assert over.status_code == 200
+ assert over.headers["content-type"].startswith("text/event-stream")
+ body = over.text
+ assert "data: " in body and "[DONE]" in body
+ # The terminal signal has to be in the stream, or the loop never learns it should stop.
+ assert '"finish_reason": "stop"' in body or '"finish_reason":"stop"' in body
+ # And still nothing synthetic in the graph.
+ assert session.graph.stats()["n_turns"] == 1
+
+
+def test_the_stop_message_is_not_empty(app_and_engine):
+ """An empty assistant message reads as a failed generation and gets retried."""
+ app, engine = app_and_engine
+ session = app.state.registry.create(max_model_calls=1)
+ with TestClient(app) as client:
+ _chat(client, session.session_id)
+ over = _chat(client, session.session_id).json()
+ assert over["choices"][0]["message"]["content"].strip()
+
+
+def test_zero_means_unlimited(app_and_engine):
+ app, engine = app_and_engine
+ session = app.state.registry.create()
+ assert session.max_model_calls == 0
+ with TestClient(app) as client:
+ for _ in range(6):
+ _chat(client, session.session_id)
+ assert engine.calls == 6
+ assert not session.over_budget
+
+
+def test_budget_is_per_session_not_per_server(app_and_engine):
+ """One deployment serves a capped training run and an uncapped eval run at the same time."""
+ app, engine = app_and_engine
+ capped = app.state.registry.create(max_model_calls=2)
+ uncapped = app.state.registry.create()
+ with TestClient(app) as client:
+ for _ in range(4):
+ _chat(client, capped.session_id)
+ _chat(client, uncapped.session_id)
+ assert capped.model_calls == 2
+ assert uncapped.model_calls == 4
diff --git a/tests/envs/test_capture_session_output_budget.py b/tests/envs/test_capture_session_output_budget.py
new file mode 100644
index 0000000000..a12ada6e54
--- /dev/null
+++ b/tests/envs/test_capture_session_output_budget.py
@@ -0,0 +1,72 @@
+"""One shared proxy applies independent rollout caps without relaxing its own limit."""
+
+from concurrent.futures import ThreadPoolExecutor
+
+import pytest
+from fastapi.testclient import TestClient
+from openenv.core.harness.capture.server import create_app
+
+
+class Engine:
+ served_model = "test-model"
+ capture_level = "text"
+ param_fixes = {}
+
+ async def completion(self, request):
+ return {
+ "id": "cap",
+ "model": self.served_model,
+ "choices": [
+ {
+ "message": {
+ "role": "assistant",
+ "content": str(request["max_tokens"]),
+ },
+ "finish_reason": "stop",
+ }
+ ],
+ }
+
+
+def test_parallel_rollouts_keep_distinct_caps_and_cannot_raise_server_limit():
+ app = create_app(
+ llm_url="http://unused.invalid",
+ model="test-model",
+ capture_level="text",
+ max_output_tokens=16384,
+ )
+ app.state.upstreams._default = (Engine(), "text")
+ caps = [4096, 16384, 32768]
+ sessions = [app.state.registry.create(max_output_tokens=cap) for cap in caps]
+ with TestClient(app) as client:
+
+ def call(index):
+ response = client.post(
+ "/v1/chat/completions",
+ headers={"Authorization": "Bearer " + sessions[index].session_id},
+ json={
+ "messages": [{"role": "user", "content": "hi"}],
+ "max_tokens": 32768,
+ },
+ )
+ assert response.status_code == 200
+ return int(response.json()["choices"][0]["message"]["content"])
+
+ with ThreadPoolExecutor(max_workers=3) as pool:
+ assert list(pool.map(call, [0, 1, 2] * 3)) == [4096, 16384, 16384] * 3
+
+
+@pytest.mark.parametrize("cap", [0, -1, True, 1.5, "4096"])
+def test_invalid_session_budget_never_reaches_inference(cap):
+ app = create_app(
+ llm_url="http://unused.invalid", model="test-model", capture_level="text"
+ )
+ session = app.state.registry.create(max_output_tokens=cap)
+ with TestClient(app) as client:
+ response = client.post(
+ "/v1/chat/completions",
+ headers={"Authorization": "Bearer " + session.session_id},
+ json={"messages": [{"role": "user", "content": "hi"}]},
+ )
+ assert response.status_code == 400
+ assert session.model_calls == 0
diff --git a/tests/envs/test_capture_stream_keepalive.py b/tests/envs/test_capture_stream_keepalive.py
new file mode 100644
index 0000000000..6e6352ef8f
--- /dev/null
+++ b/tests/envs/test_capture_stream_keepalive.py
@@ -0,0 +1,169 @@
+"""Delayed streaming must stay connected without manufacturing captured tokens."""
+
+import asyncio
+import json
+import socket
+import threading
+
+import httpx
+import pytest
+from openenv.core.harness.capture import sse
+from openenv.core.harness.capture.detection import APIType
+from openenv.core.harness.capture.export import export_session
+from openenv.core.harness.capture.runner import CaptureServer
+from starlette.responses import JSONResponse
+
+
+class DelayedEngine:
+ served_model = "test-model"
+ capture_level = "tokens"
+ param_fixes = {}
+ api_key = None
+
+ def __init__(self):
+ self.release = threading.Event()
+ self.cancelled = threading.Event()
+ self.calls = 0
+
+ async def completion(self, request):
+ self.calls += 1
+ assert request["stream"] is False
+ try:
+ while not self.release.is_set():
+ await asyncio.sleep(0.005)
+ except asyncio.CancelledError:
+ self.cancelled.set()
+ raise
+ return {
+ "id": "delayed",
+ "object": "chat.completion",
+ "model": self.served_model,
+ "prompt_token_ids": [1, 2],
+ "choices": [
+ {
+ "index": 0,
+ "message": {"role": "assistant", "content": "hello"},
+ "finish_reason": "stop",
+ "token_ids": [3, 4],
+ "logprobs": {
+ "content": [
+ {"token": "3", "logprob": -0.25},
+ {"token": "4", "logprob": -0.5},
+ ]
+ },
+ }
+ ],
+ "usage": {"prompt_tokens": 2, "completion_tokens": 2, "total_tokens": 4},
+ }
+
+
+@pytest.fixture
+def delayed_server(monkeypatch):
+ monkeypatch.setattr(sse, "KEEPALIVE_INTERVAL_S", 0.02)
+ with socket.socket() as sock:
+ sock.bind(("127.0.0.1", 0))
+ port = sock.getsockname()[1]
+ server = CaptureServer(
+ llm_url="http://127.0.0.1:9/v1",
+ model="test-model",
+ port=port,
+ capture_level="tokens",
+ )
+ engine = DelayedEngine()
+ server.app.state.inference = engine
+ server.app.state.upstreams._default = (engine, "tokens")
+ session = server.app.state.registry.create(max_model_calls=2)
+ server.start()
+ try:
+ yield server, engine, session
+ finally:
+ engine.release.set()
+ server.stop()
+
+
+def test_heartbeat_arrives_before_generation_then_exact_capture(delayed_server):
+ server, engine, session = delayed_server
+ with httpx.stream(
+ "POST",
+ f"http://127.0.0.1:{server.port}/v1/chat/completions",
+ headers={"Authorization": "Bearer " + session.session_id},
+ json={
+ "model": "test-model",
+ "messages": [{"role": "user", "content": "hi"}],
+ "stream": True,
+ },
+ timeout=3,
+ ) as response:
+ assert response.status_code == 200
+ lines = response.iter_lines()
+ assert next(lines) == ": openenv keepalive"
+ assert session.graph.stats()["n_turns"] == 0
+ assert engine.calls == session.model_calls == 1
+ engine.release.set()
+ text = "\n".join(lines)
+ assert "hello" in text and "data: [DONE]" in text
+ document = export_session(session, capture_level="tokens")
+ assert len(document["turns"]) == 1
+ row = document["sequences"][0]
+ assert row["input_ids"] == [1, 2, 3, 4]
+ assert row["logprobs"] == [0, 0, -0.25, -0.5]
+ assert row["loss_mask"] == [0, 0, 1, 1]
+
+
+def test_disconnect_cancels_pending_capture(delayed_server):
+ server, engine, session = delayed_server
+ with httpx.stream(
+ "POST",
+ f"http://127.0.0.1:{server.port}/v1/chat/completions",
+ headers={"Authorization": "Bearer " + session.session_id},
+ json={
+ "model": "test-model",
+ "messages": [{"role": "user", "content": "hi"}],
+ "stream": True,
+ },
+ timeout=3,
+ ) as response:
+ assert next(response.iter_lines()) == ": openenv keepalive"
+ assert engine.cancelled.wait(2)
+ assert session.graph.stats()["n_turns"] == 0
+
+
+@pytest.mark.parametrize("dialect", list(APIType))
+def test_late_error_is_an_error_event_without_completion(monkeypatch, dialect):
+ monkeypatch.setattr(sse, "KEEPALIVE_INTERVAL_S", 0.001)
+
+ async def check():
+ async def failed():
+ await asyncio.sleep(0.01)
+ return JSONResponse(
+ {"error": {"message": "engine failed"}}, status_code=502
+ )
+
+ response = await sse.keepalive_response(failed(), dialect)
+ body = "".join([part async for part in response.body_iterator])
+ assert ": openenv keepalive" in body
+ events = [
+ json.loads(line[6:])
+ for line in body.splitlines()
+ if line.startswith("data: ")
+ ]
+ assert len(events) == 1
+ assert events[0].get("type") == "error" or "error" in events[0]
+ assert (
+ "engine failed" in body and "[DONE]" not in body and "assistant" not in body
+ )
+
+ asyncio.run(check())
+
+
+def test_fast_error_keeps_http_status():
+ async def check():
+ async def failed():
+ return JSONResponse(
+ {"error": {"message": "invalid request"}}, status_code=400
+ )
+
+ response = await sse.keepalive_response(failed(), APIType.OPENAI_CHAT)
+ assert response.status_code == 400
+
+ asyncio.run(check())
diff --git a/tests/envs/test_harbor_acp_profile.py b/tests/envs/test_harbor_acp_profile.py
new file mode 100644
index 0000000000..191d01bb9c
--- /dev/null
+++ b/tests/envs/test_harbor_acp_profile.py
@@ -0,0 +1,59 @@
+"""The ACP profile uses Harbor's existing registry schema and isolates credentials."""
+
+import json
+
+import pytest
+
+pytest.importorskip("harbor.agents.installed.acp")
+
+from harbor.agents.installed.acp import AcpRegistryEntry
+from openenv.harbor.seams import acp_opencode_config, get
+
+
+def test_acp_opencode_profile_is_valid_and_routes_primary_and_auxiliary_calls():
+ first = acp_opencode_config("https://capture.example", "session-one", "Qwen3.5-4B")
+ entry = AcpRegistryEntry.model_validate(first["registry_entry"])
+ assert entry.distribution.npx.package == "opencode-ai@1.18.30"
+ assert entry.distribution.npx.args == ["acp"]
+ config = json.loads(entry.distribution.npx.env["OPENCODE_CONFIG_CONTENT"])
+ assert config["model"] == config["small_model"] == "intercepted/Qwen3.5-4B"
+ assert (
+ config["provider"]["intercepted"]["options"]["baseURL"]
+ == "https://capture.example/v1"
+ )
+ assert config["provider"]["intercepted"]["options"]["apiKey"] == "session-one"
+ second = acp_opencode_config("https://other.example", "session-two", "other-model")
+ assert "session-one" not in json.dumps(second)
+ assert "session-two" not in json.dumps(first)
+ assert get("acp").kwargs is None # Generic ACP must not silently select an agent.
+
+
+def test_profile_selection_is_per_rollout_and_preserves_generic_adapter(tmp_path):
+ from openenv.harbor.rollout import build_trial_config
+
+ generic = get("acp")
+ config = build_trial_config(
+ task_dir=tmp_path,
+ harness="acp",
+ sandbox="e2b",
+ intercept_url="https://capture.example",
+ session_id="session-profile",
+ model="Qwen3.5-4B",
+ trial_name="trial",
+ trials_dir=tmp_path,
+ harness_profile="opencode-1.18.30",
+ )
+ assert config.agent.model_name == "intercepted/Qwen3.5-4B"
+ entry = AcpRegistryEntry.model_validate(config.agent.kwargs["registry_entry"])
+ assert entry.distribution.npx.package == "opencode-ai@1.18.30"
+ assert get("acp") is generic
+ assert generic.kwargs is None
+
+
+def test_unknown_profile_is_not_silently_ignored():
+ import pytest
+
+ with pytest.raises(ValueError, match="unsupported harness profile"):
+ get("acp", profile="unverified-agent")
+ with pytest.raises(ValueError, match="unsupported harness profile"):
+ get("codex", profile="opencode-1.18.30")
diff --git a/tests/envs/test_harbor_async_contexts.py b/tests/envs/test_harbor_async_contexts.py
new file mode 100644
index 0000000000..baf9e51c3a
--- /dev/null
+++ b/tests/envs/test_harbor_async_contexts.py
@@ -0,0 +1,121 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""Entry points that are reached from inside a running event loop.
+
+`asyncio.run` raises `RuntimeError: asyncio.run() cannot be called from a running event loop`, so
+any code path that a server can reach has to use `run_async_safely`. This has now bitten three
+separate places (the client's MCP calls, the `run_rollout` tool handler, and registry dataset
+resolution), and each time it worked in a script and failed under the server, which is the worst
+place to find out.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import inspect
+
+import pytest
+
+pytest.importorskip("openenv.harbor.tasks")
+
+
+def sources_reachable_from_a_server() -> dict[str, str]:
+ """Source of the functions a request can reach, where a loop is already running."""
+ from openenv.harbor import environment, tasks
+ from openenv.harbor.client import HarborEnv
+
+ return {
+ "environment._run_rollout": inspect.getsource(
+ environment.HarborEnvironment._run_rollout
+ ),
+ "tasks._registry_task_dirs": inspect.getsource(tasks._registry_task_dirs),
+ "client._call": inspect.getsource(HarborEnv._call),
+ }
+
+
+@pytest.mark.parametrize("name", sorted(sources_reachable_from_a_server()))
+def test_no_bare_asyncio_run_on_a_server_reachable_path(name):
+ """`asyncio.run` here is a runtime error under ASGI, not a style preference."""
+ source = sources_reachable_from_a_server()[name]
+ assert "asyncio.run(" not in source, (
+ f"{name} calls asyncio.run, which raises when a loop is already running. "
+ "Use openenv.core.utils.run_async_safely."
+ )
+
+
+def test_run_async_safely_works_with_a_loop_already_running():
+ """The property the helper exists for, exercised the way a server would hit it."""
+ from openenv.core.utils import run_async_safely
+
+ async def inner() -> str:
+ await asyncio.sleep(0)
+ return "ok"
+
+ async def outer() -> str:
+ # A loop is running right now, which is exactly when asyncio.run would raise.
+ return run_async_safely(inner())
+
+ assert asyncio.run(outer()) == "ok"
+
+
+def test_bare_asyncio_run_would_have_failed_here():
+ """Pins the failure mode, so the test above cannot be mistaken for a tautology."""
+
+ async def inner() -> str:
+ return "ok"
+
+ async def outer() -> str:
+ return asyncio.run(inner())
+
+ with pytest.raises(
+ RuntimeError, match="cannot be called from a running event loop"
+ ):
+ asyncio.run(outer())
+
+
+# --- port forwarding --------------------------------------------------------
+def test_cloudflare_quick_forward_uses_the_tunnel_subcommand():
+ """`cloudflared forward` is an alias for `cloudflared access`, a different feature entirely.
+
+ Invoked that way the process prints `Incorrect Usage. flag provided but not defined: -url` and
+ exits without ever emitting a *.trycloudflare.com URL, so `--expose cloudflare` failed at
+ startup every time anyone selected it. The quick tunnel is `cloudflared tunnel --url`.
+ """
+ forwarding = pytest.importorskip("openenv.core.harness.capture.forwarding")
+
+ recorded: list[list[str]] = []
+
+ class _Proc:
+ stdout = None
+
+ def poll(self):
+ return None
+
+ def terminate(self):
+ pass
+
+ forwarder = forwarding.CloudflareForwarder()
+ forwarder.preflight = lambda *_a, **_k: None
+
+ def fake_popen(cmd, **_kwargs):
+ recorded.append(cmd)
+ raise RuntimeError("stop here; the command line is what matters")
+
+ import subprocess
+
+ original = subprocess.Popen
+ subprocess.Popen = fake_popen
+ try:
+ with pytest.raises(Exception):
+ forwarder.start(8100)
+ finally:
+ subprocess.Popen = original
+
+ assert recorded, "start() never built a command"
+ cmd = recorded[0]
+ assert "forward" not in cmd, "`forward` is cloudflared access, not a tunnel"
+ assert cmd[1] == "tunnel" and "--url" in cmd
diff --git a/tests/envs/test_harbor_aux_masking.py b/tests/envs/test_harbor_aux_masking.py
new file mode 100644
index 0000000000..ec54064ac8
--- /dev/null
+++ b/tests/envs/test_harbor_aux_masking.py
@@ -0,0 +1,115 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""Masking an auxiliary call out of a sequence that also contains real agent turns.
+
+Auxiliary detection is per node while demotion used to be per sequence, so a sequence mixing an aux
+call with genuine agent turns stayed `agent` in full and shipped the aux call as a training turn
+credited with the task's reward. Masking the aux node's sampled span is the fix — and getting the span
+arithmetic wrong makes the function silently do nothing, which is what happened first: an offset was
+advanced as if each turn were only prompt-plus-sampled, so from the second turn on it zeroed
+already-masked context and left the real completion tokens at 1.
+
+The middle turn is the load-bearing case. Masking the FIRST turn works under either arithmetic.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+graph_mod = pytest.importorskip("openenv.core.harness.capture.graph")
+export_mod = pytest.importorskip("openenv.core.harness.capture.export")
+rollout_mod = pytest.importorskip("openenv.harbor.rollout")
+
+
+def chain(lengths, *, context=1):
+ """A linear agent chain with `context` interstitial tokens between turns."""
+ graph = graph_mod.RolloutGraph()
+ prompt = [1, 2, 3]
+ for index, n_sampled in enumerate(lengths):
+ sampled = list(range(100 + index * 50, 100 + index * 50 + n_sampled))
+ graph.add_turn(
+ graph_mod.TurnNode(
+ node_id=f"n{index}",
+ prompt_ids=list(prompt),
+ sampled_ids=sampled,
+ sampled_logprobs=[-0.1] * n_sampled,
+ n_tools=1,
+ )
+ )
+ prompt = prompt + sampled + [900 + index] * context
+ return graph
+
+
+def document(graph):
+ class Session:
+ session_id = "s"
+ metadata: dict = {}
+ findings: list = []
+
+ session = Session()
+ session.graph = graph
+ return export_mod.export_session(session)
+
+
+def span(doc, node_id):
+ node = next(t for t in doc["turns"] if t["node_id"] == node_id)
+ return node["n_prompt"], node["n_prompt"] + node["n_sampled"]
+
+
+@pytest.mark.parametrize("aux_index", [0, 1, 2])
+def test_the_aux_span_is_zeroed_wherever_it_sits(aux_index):
+ """Parametrised across positions because only the non-first cases catch the offset bug."""
+ graph = chain([4, 5, 6])
+ doc = document(graph)
+ sequence = doc["sequences"][0]
+ before = sum(sequence["loss_mask"])
+ aux = f"n{aux_index}"
+ start, end = span(doc, aux)
+
+ rollout_mod._mask_out_nodes(doc, sequence, {aux})
+
+ assert all(m == 0 for m in sequence["loss_mask"][start:end]), (
+ f"the aux node's sampled span {start}:{end} must be fully masked"
+ )
+ expected_removed = [4, 5, 6][aux_index]
+ assert sum(sequence["loss_mask"]) == before - expected_removed, (
+ "exactly the aux node's tokens should stop being targets"
+ )
+ assert sequence["n_trainable"] == sum(sequence["loss_mask"])
+
+
+def test_the_other_turns_keep_every_target():
+ graph = chain([4, 5, 6])
+ doc = document(graph)
+ sequence = doc["sequences"][0]
+ rollout_mod._mask_out_nodes(doc, sequence, {"n1"})
+ for node_id, length in (("n0", 4), ("n2", 6)):
+ start, end = span(doc, node_id)
+ assert sum(sequence["loss_mask"][start:end]) == length, (
+ f"{node_id} lost targets it should have kept"
+ )
+
+
+def test_masking_every_node_leaves_nothing_trainable():
+ graph = chain([3, 3])
+ doc = document(graph)
+ sequence = doc["sequences"][0]
+ rollout_mod._mask_out_nodes(doc, sequence, {"n0", "n1"})
+ assert sum(sequence["loss_mask"]) == 0
+ assert sequence["n_trainable"] == 0
+ assert sequence["trainable"] is False
+
+
+def test_wider_interstitial_context_does_not_shift_the_span():
+ """The offset bug scaled with the amount of context between turns, so vary it."""
+ graph = chain([4, 5], context=7)
+ doc = document(graph)
+ sequence = doc["sequences"][0]
+ start, end = span(doc, "n1")
+ rollout_mod._mask_out_nodes(doc, sequence, {"n1"})
+ assert all(m == 0 for m in sequence["loss_mask"][start:end])
+ assert sum(sequence["loss_mask"]) == 4
diff --git a/tests/envs/test_harbor_aux_token_count.py b/tests/envs/test_harbor_aux_token_count.py
new file mode 100644
index 0000000000..3fa8a25e51
--- /dev/null
+++ b/tests/envs/test_harbor_aux_token_count.py
@@ -0,0 +1,86 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""The token-count estimator has to read every dialect the aux routes accept.
+
+`approximate_token_count` answers the harnesses' own token-counting endpoints (Anthropic's
+`/v1/messages/count_tokens`, Google's `:countTokens`). It is a character estimate on purpose — the
+alternative is re-rendering the chat template locally, which is the exact drift this layer exists to
+avoid — but it has to at least *find* the text. It reads the body of whichever dialect called it, so a
+dialect it does not know collapses to the `max(1, ...)` floor and answers 1 for a 50k-character
+conversation. An agent uses that figure to decide when to compact, so a constant 1 means it never
+compacts and blows its real context window mid-rollout.
+
+Google was that dialect: the estimator knew `messages` and `system`, and Gemini sends `contents` with
+`parts`, plus `systemInstruction`.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+server = pytest.importorskip("openenv.core.harness.capture.server")
+count = server.approximate_token_count
+
+TEXT = "x" * 400 # ~100 tokens at the estimator's 4-chars-per-token rate
+EXPECTED = 100
+
+
+def test_an_empty_body_is_one_token_not_zero():
+ """The floor exists so a caller never divides by zero; only an empty body should hit it."""
+ assert count({}) == 1
+
+
+def test_openai_messages():
+ assert count({"messages": [{"role": "user", "content": TEXT}]}) == EXPECTED
+
+
+def test_openai_content_parts():
+ body = {"messages": [{"role": "user", "content": [{"type": "text", "text": TEXT}]}]}
+ assert count(body) == EXPECTED
+
+
+def test_anthropic_system_block():
+ assert (
+ count({"messages": [], "system": [{"type": "text", "text": TEXT}]}) == EXPECTED
+ )
+
+
+def test_google_contents_are_counted():
+ """The regression: this used to be 1 regardless of how much text `contents` held."""
+ body = {"contents": [{"role": "user", "parts": [{"text": TEXT}]}]}
+ assert count(body) == EXPECTED, (
+ "Google's `contents` were invisible to the estimator"
+ )
+
+
+def test_google_system_instruction_is_counted():
+ body = {
+ "contents": [{"role": "user", "parts": [{"text": TEXT}]}],
+ "systemInstruction": {"parts": [{"text": TEXT}]},
+ }
+ assert count(body) == 2 * EXPECTED
+
+
+def test_google_snake_case_system_instruction():
+ """The REST API is camelCase, the Python SDK emits snake_case; the proxy sees both."""
+ body = {"contents": [], "system_instruction": {"parts": [{"text": TEXT}]}}
+ assert count(body) == EXPECTED
+
+
+def test_a_long_google_conversation_scales():
+ """A multi-turn body should grow with its length — pinning that it is not a per-request constant."""
+ turns = [{"role": "user", "parts": [{"text": TEXT}]} for _ in range(10)]
+ assert count({"contents": turns}) == 10 * EXPECTED
+
+
+def test_malformed_parts_do_not_raise():
+ """Bodies arrive from a sandboxed agent, so nothing here may throw on an unexpected shape."""
+ body = {
+ "contents": [{"parts": ["bare string", {"text": None}, 7]}, "not a dict", None],
+ "systemInstruction": "a plain string",
+ }
+ assert count(body) >= 1
diff --git a/tests/envs/test_harbor_capture_graph.py b/tests/envs/test_harbor_capture_graph.py
new file mode 100644
index 0000000000..4a4f8c38e6
--- /dev/null
+++ b/tests/envs/test_harbor_capture_graph.py
@@ -0,0 +1,358 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""The rollout graph: how captured calls become training sequences.
+
+This is the load-bearing piece of the capture layer. Turns are linked by exact token prefix and
+nothing else, so every structural claim a trainer relies on (this is one conversation, this branch
+was abandoned, these tokens are the model's own output) is a consequence of the linking rule. A bug
+here does not crash: it silently produces training data that misattributes tokens.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+graph_mod = pytest.importorskip("openenv.core.harness.capture.graph")
+
+RolloutGraph = graph_mod.RolloutGraph
+TurnNode = graph_mod.TurnNode
+common_prefix_len = graph_mod.common_prefix_len
+
+
+def node(node_id: str, prompt: list[int], sampled: list[int], **kwargs) -> TurnNode:
+ return TurnNode(
+ node_id=node_id,
+ prompt_ids=prompt,
+ sampled_ids=sampled,
+ sampled_logprobs=[-0.1] * len(sampled),
+ **kwargs,
+ )
+
+
+def chain(
+ graph: RolloutGraph, *lengths: int, base: int = 0, context: int = 1
+) -> list[TurnNode]:
+ """Add a linear conversation.
+
+ `context` is how many tokens the harness inserts between turns: a tool result plus the chat
+ template's scaffolding. Real rollouts always have some, and turn boundaries are derived from
+ those mask-0 runs, so a chain built without them is not representative.
+ """
+ added, prompt = [], [base]
+ for i, length in enumerate(lengths):
+ if i:
+ prompt = prompt + [base + 900 + i] * context
+ sampled = list(range(base + 100 + i * 10, base + 100 + i * 10 + length))
+ current = graph.add_turn(node(f"n{base}_{i}", list(prompt), sampled))
+ added.append(current)
+ prompt = current.end_ids
+ return added
+
+
+# --- the linking rule -------------------------------------------------------
+def test_common_prefix_len():
+ assert common_prefix_len([1, 2, 3], [1, 2, 9]) == 2
+ assert common_prefix_len([1, 2], [1, 2, 3]) == 2
+ assert common_prefix_len([], [1]) == 0
+ assert common_prefix_len([1], [2]) == 0
+
+
+def test_a_turn_whose_prompt_extends_another_becomes_its_child():
+ g = RolloutGraph()
+ first, second = chain(g, 3, 4)
+ assert second.parent_id == first.node_id
+ assert g.children(first.node_id) == [second]
+
+
+def test_an_unrelated_prompt_starts_a_new_root():
+ """A different system prompt breaks the prefix, which is what makes it a separate conversation."""
+ g = RolloutGraph()
+ chain(g, 3)
+ chain(g, 3, base=5000)
+ assert len(g.roots()) == 2
+
+
+def test_linking_ignores_arrival_order():
+ """Order of arrival must not decide structure; only the token prefix may."""
+ g = RolloutGraph()
+ parent = g.add_turn(node("p", [1, 2], [3, 4]))
+ other = g.add_turn(node("other", [9, 9], [8]))
+ child = g.add_turn(node("c", [1, 2, 3, 4], [5]))
+ assert child.parent_id == parent.node_id
+ assert other.parent_id is None
+
+
+# --- forks and discards -----------------------------------------------------
+def test_two_children_of_one_node_are_a_fork():
+ g = RolloutGraph()
+ parent = g.add_turn(node("p", [1], [2, 3]))
+ g.add_turn(node("a", [1, 2, 3], [4]))
+ g.add_turn(node("b", [1, 2, 3], [5]))
+ forks = g.forks()
+ assert len(forks) == 1
+ assert forks[0][0] == parent.node_id
+ assert len(forks[0][1]) == 2
+
+
+def test_an_abandoned_branch_is_discarded_and_the_continued_one_is_not():
+ """A retry the agent walked away from must not be trained with the task's reward."""
+ g = RolloutGraph()
+ g.add_turn(node("p", [1], [2, 3]))
+ g.add_turn(node("dead", [1, 2, 3], [4])) # never extended
+ g.add_turn(node("live", [1, 2, 3], [5]))
+ g.add_turn(node("live2", [1, 2, 3, 5], [6])) # extends `live`
+
+ discarded = {n.node_id for n in g.discarded_nodes()}
+ assert "dead" in discarded
+ assert "live" not in discarded and "live2" not in discarded
+
+
+# --- sequences, the actual training rows -------------------------------------
+def test_sequence_masks_prompt_and_marks_only_sampled_tokens():
+ g = RolloutGraph()
+ first, second = chain(g, 2, 3) # noqa: F841
+ seq = g.sequence_for(second.node_id)
+
+ assert len(seq.input_ids) == len(seq.loss_mask) == len(seq.logprobs)
+ # Exactly the sampled tokens are trainable: 2 from the first turn, 3 from the second.
+ assert sum(seq.loss_mask) == 5
+ assert seq.n_trainable == 5
+ trainable = [i for i, m in zip(seq.input_ids, seq.loss_mask) if m]
+ assert trainable == first.sampled_ids + second.sampled_ids
+
+
+def test_turn_lengths_match_what_each_turn_sampled():
+ g = RolloutGraph()
+ turns = chain(g, 2, 3, 4)
+ seq = g.sequence_for(turns[-1].node_id)
+ assert seq.turn_lengths() == [2, 3, 4]
+
+
+def test_turn_lengths_merge_when_a_turn_adds_no_context():
+ """A property of `turn_lengths`, which no longer decides turn boundaries anywhere.
+
+ Boundaries are runs of mask-1 tokens, so two turns with nothing between them read as one, and a
+ turn with unusable logprobs contributes no run at all. `turns_from_document` used to zip
+ `node_ids` against this, which dropped turns and misattributed the survivors; it now uses each
+ node's own recorded prompt and sampled counts, so this limitation is confined to the helper.
+
+ Kept because `turn_lengths` remains the join key when reconciling against an external trace,
+ where a merged run would show up as a per-call count mismatch.
+ """
+ g = RolloutGraph()
+ turns = chain(g, 2, 3, context=0)
+ seq = g.sequence_for(turns[-1].node_id)
+ assert seq.turn_lengths() == [5]
+ assert len(seq.node_ids) == 2
+
+
+def test_context_tokens_are_conditioned_on_but_never_trained():
+ """Tool results are real tokens the model saw and did not produce: mask 0, not absent."""
+ g = RolloutGraph()
+ parent = g.add_turn(node("p", [1, 2], [3]))
+ # The harness inserted a tool result (99) between the turns.
+ child = g.add_turn(node("c", [1, 2, 3, 99], [4]))
+
+ assert child.context_ids(parent) == [99]
+ seq = g.sequence_for(child.node_id)
+ assert 99 in seq.input_ids
+ assert seq.loss_mask[seq.input_ids.index(99)] == 0
+
+
+def test_one_sequence_per_leaf():
+ g = RolloutGraph()
+ g.add_turn(node("p", [1], [2]))
+ g.add_turn(node("a", [1, 2], [3]))
+ g.add_turn(node("b", [1, 2], [4]))
+ assert len(g.sequences()) == len(g.leaves()) == 2
+
+
+def test_stats_report_the_shape():
+ g = RolloutGraph()
+ chain(g, 2, 2)
+ chain(g, 2, base=5000)
+ stats = g.stats()
+ assert stats["n_turns"] == 3
+ assert stats["n_roots"] == 2
+ assert stats["n_leaves"] == 2
+
+
+def test_empty_graph_is_not_an_error():
+ g = RolloutGraph()
+ assert g.nodes() == [] and g.roots() == [] and g.sequences() == []
+ assert g.stats()["n_turns"] == 0
+
+
+# --- linking without token ids (an eval endpoint) ---------------------------
+#
+# A hosted provider returns no token ids at all, so `end_ids` is empty for every node and the token
+# rule can never find a parent: `len(end) <= best_len` holds for all candidates. The graph would
+# report a 20-turn conversation as 20 separate roots — not wrong exactly, but it reads as if the
+# agent restarted every turn, and every root-count heuristic downstream misfires.
+def eval_node(node_id: str, messages: list[dict], reply: str) -> TurnNode:
+ return TurnNode(
+ node_id=node_id,
+ prompt_ids=[],
+ sampled_ids=[],
+ sampled_logprobs=None,
+ request_messages=messages,
+ response_message={"role": "assistant", "content": reply},
+ )
+
+
+def test_message_prefix_links_an_eval_conversation_into_one_root():
+ graph = RolloutGraph()
+ first = [{"role": "system", "content": "sys"}, {"role": "user", "content": "go"}]
+ graph.add_turn(eval_node("a", first, "step 1"))
+ second = [
+ *first,
+ {"role": "assistant", "content": "step 1"},
+ {"role": "user", "content": "tool result"},
+ ]
+ graph.add_turn(eval_node("b", second, "step 2"))
+ third = [
+ *second,
+ {"role": "assistant", "content": "step 2"},
+ {"role": "user", "content": "tool result 2"},
+ ]
+ graph.add_turn(eval_node("c", third, "done"))
+
+ assert graph.stats()["n_roots"] == 1
+ assert graph.stats()["n_turns"] == 3
+ assert graph.get("b").parent_id == "a"
+ assert graph.get("c").parent_id == "b"
+
+
+def test_an_unrelated_eval_conversation_is_still_its_own_root():
+ """A subagent starts from a fresh system prompt and must not be grafted onto the main chain."""
+ graph = RolloutGraph()
+ graph.add_turn(eval_node("a", [{"role": "system", "content": "main"}], "working"))
+ graph.add_turn(
+ eval_node("b", [{"role": "system", "content": "subagent"}], "also working")
+ )
+ assert graph.stats()["n_roots"] == 2
+
+
+def test_provider_noise_on_the_assistant_message_does_not_break_linking():
+ """The message a provider returns and the one a harness echoes back are equal in meaning and
+ unequal as dicts: `refusal`, `annotations` and `audio: null` get added, `content` moves between
+ `null` and `""`. Comparing raw dicts would find no parent for any turn."""
+ graph = RolloutGraph()
+ first = [{"role": "user", "content": "go"}]
+ parent = TurnNode(
+ node_id="a",
+ prompt_ids=[],
+ sampled_ids=[],
+ request_messages=first,
+ response_message={
+ "role": "assistant",
+ "content": "step 1",
+ "refusal": None,
+ "annotations": [],
+ "audio": None,
+ },
+ )
+ graph.add_turn(parent)
+ graph.add_turn(
+ eval_node(
+ "b",
+ [
+ *first,
+ {"role": "assistant", "content": "step 1"},
+ {"role": "user", "content": "next"},
+ ],
+ "step 2",
+ )
+ )
+ assert graph.get("b").parent_id == "a"
+
+
+def test_token_linking_still_wins_when_ids_are_present():
+ """Messages are a weaker key — what the harness said it sent, not what the engine tokenised — so
+ they must never be consulted while ids are available."""
+ graph = RolloutGraph()
+ turns = chain(graph, 3, 4)
+ assert graph.get(turns[1].node_id).parent_id == turns[0].node_id
+ assert graph.stats()["n_roots"] == 1
+
+
+def test_tool_call_arguments_are_compared_as_json_not_as_bytes():
+ """The bug that made message linking inert on real data.
+
+ An eight-turn opencode rollout against the HF router came back as eight separate roots. The
+ arguments were identical; the *strings* were not, differing only in the space after the colon,
+ because the harness re-serialises what the provider sent:
+
+ {"command": "ls"} provider
+ {"command":"ls"} echoed back
+ """
+ graph = RolloutGraph()
+ first = [{"role": "user", "content": "go"}]
+ call = {"id": "call_1", "type": "function", "function": {"name": "bash"}}
+ graph.add_turn(
+ TurnNode(
+ node_id="a",
+ prompt_ids=[],
+ sampled_ids=[],
+ request_messages=first,
+ response_message={
+ "role": "assistant",
+ "content": None,
+ "tool_calls": [
+ {
+ **call,
+ "function": {
+ **call["function"],
+ "arguments": '{"command": "ls"}',
+ },
+ }
+ ],
+ },
+ )
+ )
+ graph.add_turn(
+ eval_node(
+ "b",
+ [
+ *first,
+ {
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [
+ {
+ **call,
+ "function": {
+ **call["function"],
+ "arguments": '{"command":"ls"}',
+ },
+ }
+ ],
+ },
+ {"role": "tool", "content": "a.txt"},
+ ],
+ "done",
+ )
+ )
+ assert graph.get("b").parent_id == "a"
+ assert graph.stats()["n_roots"] == 1
+
+
+def test_reordered_argument_keys_are_still_the_same_call():
+ """Any harness that round-trips arguments through a dict can reorder the keys."""
+ from openenv.core.harness.capture.graph import _canonical_arguments
+
+ assert _canonical_arguments('{"b": 2, "a": 1}') == _canonical_arguments(
+ '{"a":1,"b":2}'
+ )
+
+
+def test_malformed_arguments_are_not_forced_to_match():
+ """Two different malformed strings are two different calls, not one."""
+ from openenv.core.harness.capture.graph import _canonical_arguments
+
+ assert _canonical_arguments("{not json") != _canonical_arguments("{also not json")
+ assert _canonical_arguments("{not json") == _canonical_arguments(" {not json ")
diff --git a/tests/envs/test_harbor_capture_level.py b/tests/envs/test_harbor_capture_level.py
new file mode 100644
index 0000000000..c7192f3f98
--- /dev/null
+++ b/tests/envs/test_harbor_capture_level.py
@@ -0,0 +1,831 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""Which capture level an endpoint gets, and what follows from it.
+
+Two rollout types, and the endpoint decides which one you get: `train` when it returns token ids and
+aligned logprobs, `eval` otherwise. The tests below cover the decision (a probe that negotiates its
+way down through a provider's 400s) and the consequence that matters most — that an eval rollout
+cannot be mistaken for, or converted into, a training one.
+"""
+
+from __future__ import annotations
+
+import urllib.error
+
+import pytest
+
+validate_llm_mod = pytest.importorskip("openenv.core.harness.capture.validate_llm")
+export_mod = pytest.importorskip("openenv.core.harness.capture.export")
+contract_mod = pytest.importorskip("openenv.core.harness.capture.contract")
+graph_mod = pytest.importorskip("openenv.core.harness.capture.graph")
+
+validate_llm = validate_llm_mod.validate_llm
+
+
+def http_400(payload: dict) -> urllib.error.HTTPError:
+ import io
+ import json
+
+ return urllib.error.HTTPError(
+ "http://engine/v1/chat/completions",
+ 400,
+ "Bad Request",
+ {},
+ io.BytesIO(json.dumps(payload).encode()),
+ )
+
+
+def unsupported(param: str, code: str = "unsupported_parameter", message="") -> dict:
+ return {
+ "error": {
+ "message": message
+ or f"Unsupported parameter: '{param}' is not supported with this model.",
+ "param": param,
+ "code": code,
+ }
+ }
+
+
+def reply(*, prompt_ids=None, token_ids=None, logprobs=None) -> dict:
+ choice: dict = {"message": {"content": "ok"}, "finish_reason": "stop"}
+ if token_ids is not None:
+ choice["token_ids"] = token_ids
+ if logprobs is not None:
+ choice["logprobs"] = {"content": [{"logprob": lp} for lp in logprobs]}
+ payload: dict = {"choices": [choice]}
+ if prompt_ids is not None:
+ payload["prompt_token_ids"] = prompt_ids
+ return payload
+
+
+def probe(monkeypatch, script):
+ """Run `validate_llm` against a scripted endpoint. Returns the report and the bodies sent."""
+ sent: list[dict] = []
+ remaining = list(script)
+
+ def fake_post(url, body, timeout, api_key=None, auth_header="Authorization"):
+ sent.append(dict(body))
+ outcome = remaining.pop(0)
+ if isinstance(outcome, Exception):
+ raise outcome
+ return outcome
+
+ monkeypatch.setattr(validate_llm_mod, "_post", fake_post)
+ monkeypatch.setattr(validate_llm_mod, "list_models", lambda *a, **k: ["m"])
+ # The logprobs-mode probe issues its own two calls and is exercised on its own below; leaving it
+ # on here would make every scripted case account for requests it is not about.
+ return (
+ validate_llm(
+ "http://engine", "m", check_logprobs_mode=False, check_tools=False
+ ),
+ sent,
+ )
+
+
+# --- the probe decides the level --------------------------------------------
+def test_a_full_capture_engine_is_trainable(monkeypatch):
+ report, sent = probe(
+ monkeypatch,
+ [reply(prompt_ids=[1, 2], token_ids=[7, 8], logprobs=[-0.1, -0.2])],
+ )
+ assert (report.capture_level, report.rollout_type) == ("tokens", "train")
+ assert report.trainable and report.ok and report.reachable
+ assert len(sent) == 1, "grading a working engine must cost exactly one completion"
+ assert report.param_fixes == []
+
+
+def test_token_ids_returned_as_null_land_on_the_logprobs_level(monkeypatch):
+ """The HF router's shape: it accepts `return_token_ids` and answers `token_ids: null`. The level
+ must come from what came back, not from the request having been accepted."""
+ report, _ = probe(monkeypatch, [reply(token_ids=None, logprobs=[-0.1])])
+ assert (report.capture_level, report.rollout_type) == ("logprobs", "eval")
+ assert not report.trainable and report.reachable
+
+
+def test_a_provider_rejecting_return_token_ids_is_retried_without_it(monkeypatch):
+ report, sent = probe(
+ monkeypatch,
+ [
+ http_400(
+ {
+ "error": {
+ "message": "Unrecognized request argument supplied: "
+ "return_token_ids"
+ }
+ }
+ ),
+ reply(logprobs=[-0.1]),
+ ],
+ )
+ assert report.capture_level == "logprobs"
+ assert "return_token_ids" in sent[0] and "return_token_ids" not in sent[1]
+ assert report.param_fixes == ["dropped return_token_ids"]
+
+
+def test_a_provider_rejecting_logprobs_too_lands_on_text(monkeypatch):
+ """Every current OpenAI model: `return_token_ids` unknown, `logprobs` unsupported."""
+ report, sent = probe(
+ monkeypatch,
+ [
+ http_400(
+ unsupported(
+ "return_token_ids",
+ "unknown_parameter",
+ "Unknown parameter: 'return_token_ids'.",
+ )
+ ),
+ http_400(unsupported("logprobs")),
+ reply(),
+ ],
+ )
+ assert (report.capture_level, report.rollout_type) == ("text", "eval")
+ assert "logprobs" not in sent[-1]
+ assert "top_logprobs" not in sent[-1]
+ assert report.reachable
+
+
+def test_max_tokens_is_renamed_during_the_probe(monkeypatch):
+ report, sent = probe(
+ monkeypatch,
+ [
+ http_400(
+ unsupported(
+ "max_tokens",
+ message="Unsupported parameter: 'max_tokens' is not supported with "
+ "this model. Use 'max_completion_tokens' instead.",
+ )
+ ),
+ reply(prompt_ids=[1], token_ids=[2], logprobs=[-0.1]),
+ ],
+ )
+ assert sent[1]["max_completion_tokens"] == sent[0]["max_tokens"]
+ assert report.param_fixes == ["renamed max_tokens -> max_completion_tokens"]
+
+
+def test_an_unreachable_endpoint_is_neither_trainable_nor_eval(monkeypatch):
+ report, _ = probe(monkeypatch, [OSError("connection refused")])
+ assert not report.reachable
+ assert report.capture_level == ""
+ assert "connection refused" in report.summary()
+
+
+def test_a_model_the_endpoint_does_not_serve_fails_before_any_completion(monkeypatch):
+ monkeypatch.setattr(
+ validate_llm_mod, "list_models", lambda *a, **k: ["other-model"]
+ )
+ monkeypatch.setattr(
+ validate_llm_mod,
+ "_post",
+ lambda *a, **k: pytest.fail("must not spend a completion on a bad model name"),
+ )
+ report = validate_llm("http://engine", "m")
+ assert not report.ok and not report.reachable
+
+
+def test_an_endpoint_that_publishes_no_model_list_is_still_probed(monkeypatch):
+ """Some hosted gateways gate `/v1/models` differently from inference, or omit it. Refusing there
+ would reject an endpoint that serves completions perfectly well."""
+ monkeypatch.setattr(validate_llm_mod, "list_models", lambda *a, **k: [])
+ monkeypatch.setattr(
+ validate_llm_mod,
+ "_post",
+ lambda *a, **k: reply(prompt_ids=[1], token_ids=[2], logprobs=[-0.1]),
+ )
+ report = validate_llm("http://engine", "m")
+ assert report.trainable
+
+
+def test_require_llm_accepts_an_eval_endpoint_but_can_be_told_not_to(monkeypatch):
+ monkeypatch.setattr(validate_llm_mod, "list_models", lambda *a, **k: ["m"])
+ monkeypatch.setattr(validate_llm_mod, "_post", lambda *a, **k: reply())
+
+ report = validate_llm_mod.require_llm("http://engine", "m")
+ assert report.capture_level == "text"
+
+ with pytest.raises(RuntimeError, match="needs trainable rollouts"):
+ validate_llm_mod.require_llm("http://engine", "m", require_tokens=True)
+
+
+# --- what an eval level means downstream ------------------------------------
+class FakeSession:
+ def __init__(self, graph):
+ self.session_id = "s1"
+ self.metadata: dict = {}
+ self.findings: list[str] = []
+ self.graph = graph
+
+
+def eval_graph():
+ graph = graph_mod.RolloutGraph()
+ messages = [{"role": "user", "content": "go"}]
+ graph.add_turn(
+ graph_mod.TurnNode(
+ node_id="a",
+ prompt_ids=[],
+ sampled_ids=[],
+ request_messages=messages,
+ response_message={"role": "assistant", "content": "step 1"},
+ n_tools=1,
+ )
+ )
+ graph.add_turn(
+ graph_mod.TurnNode(
+ node_id="b",
+ prompt_ids=[],
+ sampled_ids=[],
+ request_messages=[
+ *messages,
+ {"role": "assistant", "content": "step 1"},
+ {"role": "user", "content": "result"},
+ ],
+ response_message={"role": "assistant", "content": "done"},
+ n_tools=1,
+ )
+ )
+ return graph
+
+
+def test_an_eval_export_keeps_the_whole_trace_and_claims_nothing_trainable():
+ graph = eval_graph()
+ document = export_mod.export_session(
+ FakeSession(graph), include_messages=True, capture_level="logprobs"
+ )
+ assert document["rollout_type"] == "eval"
+ assert document["capture_level"] == "logprobs"
+ assert document["trainable"] is False
+
+ # The trace is the payload of an eval rollout, and all of it has to survive.
+ assert len(document["turns"]) == 2
+ assert document["turns"][0]["request_messages"]
+
+ # Structure survives too. `conversations_from_document` and `turns_from_document` both walk
+ # `sequences`, so emptying it on an eval rollout would delete the very trace this exists for —
+ # which is exactly the bug a live rollout caught: reward 1.0, six turns, zero conversations.
+ assert len(document["sequences"]) == 1
+ row = document["sequences"][0]
+ assert row["node_ids"] == ["a", "b"]
+ assert row["n_turns"] == 2
+ # What is withheld is the training claim, not the structure.
+ assert row["trainable"] is False
+ assert row["input_ids"] == []
+ assert document["stats"]["n_trainable_tokens"] == 0
+
+
+def test_an_eval_rollout_still_rebuilds_its_conversations():
+ """The regression a live run found: an eval result with a reward, six turns and no transcript."""
+ models = pytest.importorskip("openenv.harbor.models")
+ document = export_mod.export_session(
+ FakeSession(eval_graph()), include_messages=True, capture_level="logprobs"
+ )
+ conversations = models.conversations_from_document(document)
+ assert len(conversations) == 1
+ # The deepest node replays the whole thread, plus its own reply.
+ assert [m["role"] for m in conversations[0].messages] == [
+ "user",
+ "assistant",
+ "user",
+ "assistant",
+ ]
+ turns = models.turns_from_document(document)
+ assert len(turns) == 2
+ assert [t.text for t in turns] == ["step 1", "done"]
+ assert all(t.prompt_token_ids == [] for t in turns)
+
+
+def test_a_training_contract_cannot_be_built_from_an_eval_rollout():
+ graph = eval_graph()
+ document = export_mod.export_session(FakeSession(graph), capture_level="text")
+ for build in (contract_mod.to_turn_records, contract_mod.to_trace_entries):
+ with pytest.raises(ValueError, match="EVAL rollout"):
+ build(graph, document)
+
+
+def test_a_train_export_is_unchanged():
+ """The regression that matters: nothing on the trainable path may move."""
+ graph = graph_mod.RolloutGraph()
+ graph.add_turn(
+ graph_mod.TurnNode(
+ node_id="a",
+ prompt_ids=[1, 2, 3],
+ sampled_ids=[4, 5],
+ sampled_logprobs=[-0.1, -0.2],
+ n_tools=1,
+ )
+ )
+ document = export_mod.export_session(FakeSession(graph))
+ assert document["rollout_type"] == "train"
+ assert document["capture_level"] == "tokens"
+ assert len(document["sequences"]) == 1
+ assert document["sequences"][0]["input_ids"] == [1, 2, 3, 4, 5]
+ assert contract_mod.to_turn_records(graph, document) == [
+ ([1, 2, 3], [4, 5], [-0.1, -0.2])
+ ]
+
+
+# --- the proxy reports its level, and never its credential ------------------
+def app_client(**kwargs):
+ from fastapi.testclient import TestClient
+
+ server = pytest.importorskip("openenv.core.harness.capture.server")
+ return TestClient(
+ server.create_app(llm_url="http://127.0.0.1:9/v1", model="m", **kwargs)
+ )
+
+
+def test_health_states_the_rollout_type():
+ with app_client(capture_level="logprobs") as client:
+ body = client.get("/health").json()
+ assert body["capture_level"] == "logprobs"
+ assert body["rollout_type"] == "eval"
+
+
+def test_health_confirms_auth_without_revealing_the_key():
+ """On a Space this endpoint is public, and the key it holds buys paid inference."""
+ with app_client(api_key="sk-secret-value") as client:
+ body = client.get("/health").json()
+ assert body["upstream_auth"] is True
+ assert "sk-secret-value" not in str(body)
+
+
+def test_health_says_train_by_default():
+ with app_client() as client:
+ body = client.get("/health").json()
+ assert (body["capture_level"], body["rollout_type"]) == ("tokens", "train")
+ assert body["upstream_auth"] is False
+
+
+def test_an_eval_rollout_is_recorded_without_fatal_findings(monkeypatch):
+ """`check_turn`'s FATALs (`no_prompt_ids`, `no_logprobs`) are the expected condition here.
+ Letting them fire would mark every eval turn unusable and teach everyone to ignore findings.
+
+ Two calls, not one: `degenerate_rollout` is FATAL for a single-call agentic rollout and stays
+ that way on this path, because an agent that made one call and stopped did not attempt the task
+ whether or not its tokens were captured.
+ """
+ replies = [reply(), reply()]
+
+ async def fake_completion(_self, _request):
+ return replies.pop(0)
+
+ server = pytest.importorskip("openenv.core.harness.capture.server")
+ upstream = pytest.importorskip("openenv.core.harness.capture.upstream")
+ monkeypatch.setattr(upstream.InferenceClient, "completion", fake_completion)
+
+ app = server.create_app(
+ llm_url="http://127.0.0.1:9/v1", model="m", capture_level="text"
+ )
+ from fastapi.testclient import TestClient
+
+ first = [{"role": "user", "content": "go"}]
+ second = [
+ *first,
+ {"role": "assistant", "content": "ok"},
+ {"role": "user", "content": "tool result"},
+ ]
+ with TestClient(app) as client:
+ session = client.post("/sessions").json()["session_id"]
+ for messages in (first, second):
+ response = client.post(
+ "/v1/chat/completions",
+ json={"model": "m", "messages": messages},
+ headers={"Authorization": f"Bearer {session}"},
+ )
+ assert response.status_code == 200
+ status = client.get(f"/sessions/{session}").json()
+ rollout = client.get(f"/sessions/{session}/rollout").json()
+
+ assert status["n_turns"] == 2
+ # One root, via the message-prefix fallback: with no token ids the token rule would make each
+ # call its own root and the rollout would read as two unrelated conversations. Note that
+ # `reply()` omits `role` on the assistant message while the harness names it on the way back —
+ # linking has to survive that asymmetry, since it is invisible until the graph collapses.
+ assert status["n_roots"] == 1
+ assert rollout["rollout_type"] == "eval"
+ assert not [f for f in rollout["validation"] if f.startswith("[FATAL")], (
+ "an eval rollout must not be reported as a capture failure"
+ )
+
+
+# --- the train path must not move -------------------------------------------
+#
+# The acceptance rows for a real vLLM and a real SGLang need a GPU. This is the gate that runs
+# anywhere, including CI: a stub upstream returning exactly the shape vLLM returns with
+# `--return-tokens-as-token-ids --logprobs-mode processed_logprobs`, driven through the whole proxy,
+# with every field a trainer reads pinned. A change to the eval path that leaks into the token path
+# fails here rather than in a loss curve days later.
+def test_the_trainable_path_end_to_end_is_unchanged(monkeypatch):
+ server = pytest.importorskip("openenv.core.harness.capture.server")
+ upstream = pytest.importorskip("openenv.core.harness.capture.upstream")
+ from fastapi.testclient import TestClient
+
+ # Turn k's prompt is turn k-1's prompt + completion + one interstitial context token, which is
+ # what a real engine returns once the harness has appended a tool result.
+ scripted = [
+ {"prompt": [1, 2, 3], "sampled": [10, 11], "logprobs": [-0.5, -0.25]},
+ {
+ "prompt": [1, 2, 3, 10, 11, 90],
+ "sampled": [12, 13, 14],
+ "logprobs": [-0.1, -0.2, -0.3],
+ },
+ {
+ "prompt": [1, 2, 3, 10, 11, 90, 12, 13, 14, 91],
+ "sampled": [15],
+ "logprobs": [-0.05],
+ },
+ ]
+ remaining = list(scripted)
+
+ async def fake_completion(_self, _request):
+ step = remaining.pop(0)
+ return upstream.normalize_response(
+ {
+ "prompt_token_ids": step["prompt"],
+ "choices": [
+ {
+ "message": {"role": "assistant", "content": "step"},
+ "finish_reason": "stop",
+ "token_ids": step["sampled"],
+ "logprobs": {
+ "content": [{"logprob": lp} for lp in step["logprobs"]]
+ },
+ }
+ ],
+ }
+ )
+
+ monkeypatch.setattr(upstream.InferenceClient, "completion", fake_completion)
+ app = server.create_app(llm_url="http://127.0.0.1:9/v1", model="m")
+
+ with TestClient(app) as client:
+ session = client.post("/sessions").json()["session_id"]
+ for _ in scripted:
+ assert (
+ client.post(
+ "/v1/chat/completions",
+ json={
+ "model": "m",
+ "messages": [{"role": "user", "content": "go"}],
+ "tools": [{"type": "function", "function": {"name": "bash"}}],
+ },
+ headers={"Authorization": f"Bearer {session}"},
+ ).status_code
+ == 200
+ )
+ document = client.get(f"/sessions/{session}/rollout").json()
+ graph = app.state.registry.get(session).graph
+
+ assert document["rollout_type"] == "train"
+ assert document["capture_level"] == "tokens"
+ assert document["trainable"] is True
+ assert document["stats"]["n_turns"] == 3
+ assert document["stats"]["n_roots"] == 1
+
+ (row,) = document["sequences"]
+ assert row["role"] == "agent"
+ assert row["trainable"] is True
+ # The flattened sequence: every prompt token the model conditioned on, in order, with each
+ # sampled span marked 1 and the interstitial context tokens marked 0.
+ assert row["input_ids"] == [1, 2, 3, 10, 11, 90, 12, 13, 14, 91, 15]
+ assert row["loss_mask"] == [0, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1]
+ assert row["logprobs"] == [
+ 0.0,
+ 0.0,
+ 0.0,
+ -0.5,
+ -0.25,
+ 0.0,
+ -0.1,
+ -0.2,
+ -0.3,
+ 0.0,
+ -0.05,
+ ]
+ assert row["prompt_len"] == 3
+ assert row["turn_lengths"] == [2, 3, 1]
+ assert row["n_trainable"] == 6
+ assert document["stats"]["n_trainable_tokens"] == 6
+
+ # And the contract a trainer actually consumes, per turn.
+ assert contract_mod.to_turn_records(graph, document) == [
+ ([1, 2, 3], [10, 11], [-0.5, -0.25]),
+ ([1, 2, 3, 10, 11, 90], [12, 13, 14], [-0.1, -0.2, -0.3]),
+ ([1, 2, 3, 10, 11, 90, 12, 13, 14, 91], [15], [-0.05]),
+ ]
+
+
+def test_the_probe_does_not_restate_the_tier_as_fatal_findings(monkeypatch):
+ """An eval endpoint's findings must not read like a broken one.
+
+ `no_prompt_token_ids` is reported FATAL by `check_upstream_response`, and it is the *definition*
+ of an eval endpoint. Printing three fatal-looking lines under a heading that already says
+ EVAL ONLY is how a findings list stops being read at all.
+ """
+ report, _ = probe(monkeypatch, [reply()])
+ assert report.capture_level == "text"
+ assert report.findings == []
+
+
+def test_a_genuinely_broken_response_still_reports(monkeypatch):
+ """`no_choices` is not "merely eval-only" — the endpoint answered with nothing at all."""
+ report, _ = probe(monkeypatch, [{"id": "x"}])
+ assert any("no_choices" in f for f in report.findings)
+
+
+# --- raw vs processed logprobs ----------------------------------------------
+#
+# The one hole no other check here can see. `token_ids` arrives from the REQUEST parameter, not from a
+# serving flag, so an engine started with neither flag returns aligned, negative, correctly-counted
+# logprobs that are pre-temperature — and grades as fully trainable. vLLM's `logprobs_mode` defaults
+# to `raw_logprobs`. The test follows from the definition: raw values cannot move with temperature.
+def temperature_scripted(monkeypatch, by_temperature, *, fail=False):
+ """Serve first-position `top_logprobs` per temperature. Returns the calls made."""
+ calls: list[float] = []
+
+ def fake_post(url, body, timeout, api_key=None, auth_header="Authorization"):
+ if fail:
+ raise OSError("endpoint refused")
+ calls.append(body["temperature"])
+ tops = by_temperature[body["temperature"]]
+ return {
+ "choices": [
+ {
+ "message": {"content": "x"},
+ "finish_reason": "stop",
+ "logprobs": {
+ "content": [
+ {
+ "token": "a",
+ "logprob": -0.1,
+ "top_logprobs": [
+ {"token": t, "logprob": lp}
+ for t, lp in tops.items()
+ ],
+ }
+ ]
+ },
+ }
+ ]
+ }
+
+ monkeypatch.setattr(validate_llm_mod, "_post", fake_post)
+ return calls
+
+
+def test_an_unchanged_gap_is_raw(monkeypatch):
+ """Live measurement: gap 6.7500 at both temperatures on a default-flags vLLM."""
+ calls = temperature_scripted(
+ monkeypatch,
+ {1.0: {"a": -0.0037, "b": -6.7537}, 2.0: {"a": -0.0037, "b": -6.7537}},
+ )
+ assert validate_llm_mod.probe_logprobs_mode("http://engine", "m") == "raw"
+ assert calls == [1.0, 2.0]
+
+
+def test_a_halved_gap_is_processed(monkeypatch):
+ """Live measurement: gap 6.7500 -> 3.3750, i.e. exactly T1/T2, on a processed_logprobs vLLM."""
+ temperature_scripted(
+ monkeypatch,
+ {1.0: {"a": -0.0037, "b": -6.7537}, 2.0: {"a": -1.4380, "b": -4.8130}},
+ )
+ assert validate_llm_mod.probe_logprobs_mode("http://engine", "m") == "processed"
+
+
+def test_a_constant_offset_between_calls_does_not_change_the_verdict(monkeypatch):
+ """Why the gap, not the value: a data-parallel engine answers consecutive calls from different
+ replicas, and comparing values directly misread one such engine (DP=4) as processed. A constant
+ shift cancels in a difference, so the same gap survives it."""
+ temperature_scripted(
+ monkeypatch,
+ {1.0: {"a": -0.0037, "b": -6.7537}, 2.0: {"a": -0.9037, "b": -7.6537}},
+ )
+ assert validate_llm_mod.probe_logprobs_mode("http://engine", "m") == "raw"
+
+
+def test_a_distribution_too_flat_to_divide_is_unknown(monkeypatch):
+ """Guessing from noise is how a check becomes something people override on principle."""
+ temperature_scripted(
+ monkeypatch, {1.0: {"a": -0.5, "b": -0.6}, 2.0: {"a": -0.5, "b": -0.9}}
+ )
+ assert validate_llm_mod.probe_logprobs_mode("http://engine", "m") == "unknown"
+
+
+def test_a_single_top_logprob_is_unknown(monkeypatch):
+ """A gap needs two values."""
+ temperature_scripted(monkeypatch, {1.0: {"a": -0.5}, 2.0: {"a": -0.5}})
+ assert validate_llm_mod.probe_logprobs_mode("http://engine", "m") == "unknown"
+
+
+def test_an_unreachable_endpoint_is_unknown(monkeypatch):
+ """Absence of evidence, not evidence of a problem."""
+ temperature_scripted(monkeypatch, {}, fail=True)
+ assert validate_llm_mod.probe_logprobs_mode("http://engine", "m") == "unknown"
+
+
+def test_a_provider_that_refused_temperature_cannot_be_asked(monkeypatch):
+ """gpt-5.6 drops `temperature`, so a question about temperature has no meaning — and must cost
+ no calls at all."""
+ compat = pytest.importorskip("openenv.core.harness.capture.compat")
+ calls = temperature_scripted(monkeypatch, {})
+ assert (
+ validate_llm_mod.probe_logprobs_mode(
+ "http://engine", "m", fixes=[compat.ParamFix(param="temperature")]
+ )
+ == "unknown"
+ )
+ assert calls == []
+
+
+def test_raw_logprobs_demote_a_trainable_endpoint_to_eval(monkeypatch):
+ """The endpoint answers fine and is a good eval backend; what it cannot do is train."""
+ monkeypatch.delenv("OPENENV_ALLOW_RAW_LOGPROBS", raising=False)
+ monkeypatch.setattr(validate_llm_mod, "list_models", lambda *a, **k: ["m"])
+ monkeypatch.setattr(
+ validate_llm_mod,
+ "probe_logprobs_mode",
+ lambda *a, **k: "raw",
+ )
+ monkeypatch.setattr(
+ validate_llm_mod,
+ "_post",
+ lambda *a, **k: reply(prompt_ids=[1], token_ids=[2], logprobs=[-0.1]),
+ )
+ report = validate_llm("http://engine", "m")
+ assert report.logprobs_mode == "raw"
+ assert report.capture_level == "logprobs", "must not stay trainable"
+ assert report.trainable is False
+ assert report.reachable is True, "still perfectly usable for eval"
+ assert any("raw_logprobs" in f and "[FATAL]" in f for f in report.findings)
+ # The measurement supersedes the inference that pointed at it; printing both says one thing twice.
+ assert not any("token_strings" in f for f in report.findings)
+
+
+def test_the_override_keeps_it_trainable_but_says_so(monkeypatch):
+ """A refusal that cannot be overridden becomes a reason to stop trusting the tool."""
+ monkeypatch.setenv("OPENENV_ALLOW_RAW_LOGPROBS", "1")
+ monkeypatch.setattr(validate_llm_mod, "list_models", lambda *a, **k: ["m"])
+ monkeypatch.setattr(validate_llm_mod, "probe_logprobs_mode", lambda *a, **k: "raw")
+ monkeypatch.setattr(
+ validate_llm_mod,
+ "_post",
+ lambda *a, **k: reply(prompt_ids=[1], token_ids=[2], logprobs=[-0.1]),
+ )
+ report = validate_llm("http://engine", "m")
+ assert report.capture_level == "tokens"
+ assert report.trainable is True
+ assert any("raw_logprobs_forced" in f for f in report.findings)
+
+
+def test_the_mode_is_not_probed_below_the_tokens_tier(monkeypatch):
+ """Nothing below `tokens` is trainable, so the answer would inform no decision."""
+ monkeypatch.setattr(validate_llm_mod, "list_models", lambda *a, **k: ["m"])
+ monkeypatch.setattr(
+ validate_llm_mod,
+ "probe_logprobs_mode",
+ lambda *a, **k: pytest.fail("must not spend two calls on an eval endpoint"),
+ )
+ monkeypatch.setattr(validate_llm_mod, "_post", lambda *a, **k: reply())
+ assert validate_llm("http://engine", "m").logprobs_mode == ""
+
+
+# --- can a coding agent work here at all? -----------------------------------
+#
+# The capture probe sends no tools; every validated harness sends one on every call. So this is the
+# only signal about agent viability, and it caught a real failure that `harbor info` previously
+# reported as a perfectly healthy endpoint.
+def tool_reply(*, tool_call=True, finish="stop"):
+ message = {"role": "assistant", "content": None if tool_call else "I'd run ls."}
+ if tool_call:
+ message["tool_calls"] = [
+ {
+ "id": "c1",
+ "type": "function",
+ "function": {"name": "bash", "arguments": "{}"},
+ }
+ ]
+ return {"choices": [{"message": message, "finish_reason": finish}]}
+
+
+def test_a_tool_call_means_agents_can_work(monkeypatch):
+ monkeypatch.setattr(validate_llm_mod, "_post", lambda *a, **k: tool_reply())
+ assert validate_llm_mod.probe_tool_support("http://engine", "m")[0] == "ok"
+
+
+def test_prose_instead_of_a_tool_call_is_reported(monkeypatch):
+ monkeypatch.setattr(
+ validate_llm_mod, "_post", lambda *a, **k: tool_reply(tool_call=False)
+ )
+ assert (
+ validate_llm_mod.probe_tool_support("http://engine", "m")[0] == "no-tool-call"
+ )
+
+
+def test_truncation_is_inconclusive_not_a_failure(monkeypatch):
+ """The false positive this avoids: Qwen3.6-35B-A3B spent 224 tokens reasoning and hit the cap, so
+ a 64-token probe called it tool-incapable while it in fact worked with all 16 harnesses."""
+ monkeypatch.setattr(
+ validate_llm_mod,
+ "_post",
+ lambda *a, **k: tool_reply(tool_call=False, finish="length"),
+ )
+ assert validate_llm_mod.probe_tool_support("http://engine", "m")[0] == "unknown"
+
+
+def test_an_endpoint_that_refuses_tools_outright_is_flagged(monkeypatch):
+ """`tools` is protected from being dropped, so this cannot be papered over."""
+
+ def refuse(*a, **k):
+ raise http_400(
+ {"error": {"message": "tools are not supported", "param": "tools"}}
+ )
+
+ monkeypatch.setattr(validate_llm_mod, "_post", refuse)
+ assert validate_llm_mod.probe_tool_support("http://engine", "m")[0] == "rejected"
+
+
+def test_reasoning_forced_off_warns_at_validate_time(monkeypatch):
+ """The gpt-5.6 case: tools are accepted only with reasoning disabled, after which agentic loops
+ make one model call and stop. Discovered only when the probe carries a tool manifest."""
+ monkeypatch.setattr(validate_llm_mod, "list_models", lambda *a, **k: ["m"])
+ monkeypatch.setattr(
+ validate_llm_mod, "probe_logprobs_mode", lambda *a, **k: "processed"
+ )
+ compat = pytest.importorskip("openenv.core.harness.capture.compat")
+ monkeypatch.setattr(
+ validate_llm_mod,
+ "probe_tool_support",
+ lambda *a, **k: (
+ "ok",
+ [compat.ParamFix(param="reasoning_effort", value="none")],
+ ),
+ )
+ monkeypatch.setattr(
+ validate_llm_mod,
+ "_post",
+ lambda *a, **k: reply(prompt_ids=[1], token_ids=[2], logprobs=[-0.1]),
+ )
+ report = validate_llm("http://engine", "m")
+ assert any(
+ "reasoning_effort" in f and "behaviour_changed" in f for f in report.findings
+ )
+ assert any("single model call" in f for f in report.findings)
+
+
+# --- roles must not depend on token counts that eval endpoints never have ----
+def test_a_toolless_harness_is_still_the_agent_on_an_eval_endpoint():
+ """terminus-2 parses tool calls out of raw text, so it sends no manifest. Role assignment used
+ `n_trainable` as the tiebreak when nothing had tools, and on an eval endpoint that is 0 for every
+ sequence — so every path was labelled auxiliary, `result.turns` came back empty and the
+ conversations were mistagged, on a rollout that had captured perfectly well."""
+ models = pytest.importorskip("openenv.harbor.models")
+ graph = graph_mod.RolloutGraph()
+ first = [{"role": "user", "content": "go"}]
+ graph.add_turn(
+ graph_mod.TurnNode(
+ node_id="a",
+ prompt_ids=[],
+ sampled_ids=[],
+ n_tools=0,
+ request_messages=first,
+ response_message={"role": "assistant", "content": "step 1"},
+ )
+ )
+ graph.add_turn(
+ graph_mod.TurnNode(
+ node_id="b",
+ prompt_ids=[],
+ sampled_ids=[],
+ n_tools=0,
+ request_messages=[
+ *first,
+ {"role": "assistant", "content": "step 1"},
+ {"role": "user", "content": "result"},
+ ],
+ response_message={"role": "assistant", "content": "done"},
+ )
+ )
+ document = export_mod.export_session(
+ FakeSession(graph), include_messages=True, capture_level="logprobs"
+ )
+ assert [r["role"] for r in document["sequences"]] == ["agent"]
+ assert len(models.turns_from_document(document)) == 2
+ assert len(models.conversations_from_document(document)) == 1
+
+
+def test_the_train_path_still_uses_trainable_tokens_as_the_tiebreak():
+ """Where token counts DO mean something, a toolless sequence with nothing trainable is auxiliary."""
+ graph = graph_mod.RolloutGraph()
+ graph.add_turn(
+ graph_mod.TurnNode(
+ node_id="a",
+ prompt_ids=[1, 2],
+ sampled_ids=[3],
+ sampled_logprobs=None, # rejected on ingest -> masked out -> nothing trainable
+ n_tools=0,
+ )
+ )
+ document = export_mod.export_session(FakeSession(graph), capture_level="tokens")
+ assert [r["role"] for r in document["sequences"]] == ["auxiliary"]
diff --git a/tests/envs/test_harbor_capture_normalise.py b/tests/envs/test_harbor_capture_normalise.py
new file mode 100644
index 0000000000..6dd2995dc2
--- /dev/null
+++ b/tests/envs/test_harbor_capture_normalise.py
@@ -0,0 +1,159 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""Request shapes that vLLM rejects outright, normalised before they reach it.
+
+Each case here is a real harness sending something an OpenAI-spec engine 400s on. A 400 does not
+degrade a rollout, it truncates it: the agent loses the call, and the captured trajectory ends early
+while still looking structurally valid. These are cheap to assert and expensive to rediscover.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+server = pytest.importorskip("openenv.core.harness.capture.server")
+
+normalise_for_capture = server.normalise_for_capture
+
+
+def test_stream_is_forced_off():
+ """Capture needs one whole response; reassembling ids from SSE deltas corrupts silently."""
+ request = {"messages": [], "stream": True}
+ normalise_for_capture(request)
+ assert request["stream"] is False
+
+
+def test_stream_options_is_dropped():
+ """vLLM 400s on `stream_options` once `stream` is False. opencode sends it on every call."""
+ request = {
+ "messages": [],
+ "stream": True,
+ "stream_options": {"include_usage": True},
+ }
+ normalise_for_capture(request)
+ assert "stream_options" not in request
+
+
+def test_empty_tools_is_dropped():
+ """kimi-cli sends `tools: []`, which vLLM rejects: 'must not be an empty array'."""
+ request = {"messages": [], "tools": []}
+ normalise_for_capture(request)
+ assert "tools" not in request
+
+
+def test_empty_functions_is_dropped():
+ """The legacy spelling fails the same way."""
+ request = {"messages": [], "functions": []}
+ normalise_for_capture(request)
+ assert "functions" not in request
+
+
+def test_tool_choice_is_dropped_with_the_tools_it_referenced():
+ """`tool_choice` without `tools` is invalid, and means nothing once the list is gone."""
+ request = {"messages": [], "tools": [], "tool_choice": "auto"}
+ normalise_for_capture(request)
+ assert "tools" not in request
+ assert "tool_choice" not in request
+
+
+def test_populated_tools_are_left_alone():
+ """The guard must be narrow: dropping real tools would change what the model can do."""
+ tools = [{"type": "function", "function": {"name": "bash"}}]
+ request = {"messages": [], "tools": tools, "tool_choice": "auto"}
+ normalise_for_capture(request)
+ assert request["tools"] == tools
+ assert request["tool_choice"] == "auto"
+
+
+def test_absent_tool_keys_are_not_invented():
+ """A request with no tool keys must stay that way rather than gain empty ones."""
+ request = {"messages": []}
+ normalise_for_capture(request)
+ assert "tools" not in request
+ assert "functions" not in request
+ assert "tool_choice" not in request
+
+
+# --- engine shape differences -----------------------------------------------
+upstream = pytest.importorskip("openenv.core.harness.capture.upstream")
+normalize_response = upstream.normalize_response
+
+
+def test_sglang_per_choice_prompt_ids_are_hoisted():
+ """SGLang returns the prompt ids on the choice; vLLM returns them at the top level.
+
+ Every reader downstream (`check_upstream_response`, the capture server's `_ingest`, the UI) looks
+ only at the top level, so an un-hoisted SGLang response reads as "no prompt ids" and fails
+ validation even though the rollout path handles it perfectly well.
+ """
+ response = {
+ "choices": [
+ {
+ "prompt_token_ids": [1, 2, 3],
+ "token_ids": [4, 5],
+ "message": {"content": "hi"},
+ }
+ ]
+ }
+
+ out = normalize_response(response)
+
+ assert out["prompt_token_ids"] == [1, 2, 3]
+
+
+def test_an_engines_own_top_level_prompt_ids_win():
+ """vLLM's value must not be overwritten by a choice that also carries one."""
+ response = {
+ "prompt_token_ids": [9, 9, 9],
+ "choices": [{"prompt_token_ids": [1, 2, 3], "message": {"content": "hi"}}],
+ }
+
+ assert normalize_response(response)["prompt_token_ids"] == [9, 9, 9]
+
+
+def test_nothing_is_invented_when_no_choice_carries_prompt_ids():
+ response = {"choices": [{"message": {"content": "hi"}}]}
+ assert "prompt_token_ids" not in normalize_response(response)
+
+
+def test_hoisting_survives_a_response_with_no_choices():
+ assert normalize_response({}) == {}
+
+
+def test_parallel_tool_calls_goes_with_an_empty_tools_array():
+ """Found by the compatibility matrix: codex against OpenAI failed EVERY call with
+
+ Invalid value for 'parallel_tool_calls': 'parallel_tool_calls' is only allowed when
+ 'tools' are specified.
+
+ It sends `tools: []` plus `parallel_tool_calls`; stripping only the empty list left the orphan.
+ vLLM ignores the orphan, which is why this survived until a hosted provider was tried.
+ """
+ server = pytest.importorskip("openenv.core.harness.capture.server")
+ body = {
+ "model": "m",
+ "tools": [],
+ "parallel_tool_calls": True,
+ "tool_choice": "auto",
+ }
+ server.normalise_for_capture(body)
+ assert "tools" not in body
+ assert "parallel_tool_calls" not in body
+ assert "tool_choice" not in body
+
+
+def test_parallel_tool_calls_survives_when_tools_are_real():
+ """It is only invalid without tools; a genuine manifest must keep its companions."""
+ server = pytest.importorskip("openenv.core.harness.capture.server")
+ body = {
+ "model": "m",
+ "tools": [{"type": "function", "function": {"name": "bash"}}],
+ "parallel_tool_calls": True,
+ }
+ server.normalise_for_capture(body)
+ assert body["parallel_tool_calls"] is True
+ assert len(body["tools"]) == 1
diff --git a/tests/envs/test_harbor_capture_request_validation.py b/tests/envs/test_harbor_capture_request_validation.py
new file mode 100644
index 0000000000..e6e5455599
--- /dev/null
+++ b/tests/envs/test_harbor_capture_request_validation.py
@@ -0,0 +1,128 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""Request-validation behavior at the Capture Proxy boundary."""
+
+from __future__ import annotations
+
+import pytest
+
+pytest.importorskip("fastapi")
+from fastapi.testclient import TestClient # noqa: E402
+
+server = pytest.importorskip("openenv.core.harness.capture.server")
+
+
+@pytest.fixture
+def client() -> TestClient:
+ with TestClient(server.create_app(), raise_server_exceptions=False) as client:
+ yield client
+
+
+def _headers(client: TestClient) -> dict[str, str]:
+ session = client.post("/sessions", json={}).json()
+ return {"Authorization": f"Bearer {session['session_id']}"}
+
+
+@pytest.mark.parametrize(
+ ("path", "body", "message"),
+ [
+ (
+ "/v1/messages",
+ {"messages": "not-an-array"},
+ "Anthropic messages must be an array",
+ ),
+ (
+ "/v1/messages",
+ {"messages": ["not-an-object"]},
+ "Anthropic messages must be objects",
+ ),
+ ("/v1/messages", {"tools": "not-an-array"}, "Anthropic tools must be an array"),
+ (
+ "/v1/messages",
+ {"tools": ["not-an-object"]},
+ "Anthropic tools must be objects",
+ ),
+ (
+ "/v1/responses",
+ {"input": {"not": "supported"}},
+ "Responses input must be a string or an array of objects",
+ ),
+ (
+ "/v1/responses",
+ {"input": ["not-an-object"]},
+ "Responses input items must be objects",
+ ),
+ (
+ "/v1/responses",
+ {"tools": "not-an-array"},
+ "Responses tools must be an array",
+ ),
+ (
+ "/v1/responses",
+ {"tools": ["not-an-object"]},
+ "Responses tools must be objects",
+ ),
+ ],
+)
+def test_invalid_dialect_payload_returns_a_client_error(
+ client: TestClient, path: str, body: dict[str, object], message: str
+) -> None:
+ response = client.post(path, json=body, headers=_headers(client))
+
+ assert response.status_code == 400
+ assert response.json() == {
+ "error": {"message": message, "type": "invalid_request_error"}
+ }
+
+
+def test_non_object_request_body_returns_a_client_error(client: TestClient) -> None:
+ response = client.post("/v1/chat/completions", json=[], headers=_headers(client))
+
+ assert response.status_code == 400
+ assert response.json() == {
+ "error": {
+ "message": "body must be a JSON object",
+ "type": "invalid_request_error",
+ }
+ }
+
+
+@pytest.mark.parametrize("metadata", [[], ["unexpected"], "unexpected"])
+def test_session_registration_rejects_non_object_metadata(
+ client: TestClient, metadata: object
+) -> None:
+ response = client.post("/sessions", json={"metadata": metadata})
+
+ assert response.status_code == 400
+ assert response.json() == {
+ "error": {
+ "message": "metadata must be a JSON object",
+ "type": "invalid_request_error",
+ }
+ }
+
+
+@pytest.mark.parametrize("key", ["session_id", "upstream", "capture_level"])
+def test_session_registration_rejects_reserved_metadata_keys(
+ client: TestClient, key: str
+) -> None:
+ response = client.post("/sessions", json={"metadata": {key: "conflict"}})
+
+ assert response.status_code == 400
+ assert response.json() == {
+ "error": {
+ "message": f"metadata cannot include reserved key: {key}",
+ "type": "invalid_request_error",
+ }
+ }
+
+
+def test_session_registration_accepts_non_reserved_metadata(client: TestClient) -> None:
+ response = client.post("/sessions", json={"metadata": {"task_id": "task-123"}})
+
+ assert response.status_code == 200
+ assert response.json()["session_id"]
diff --git a/tests/envs/test_harbor_capture_server.py b/tests/envs/test_harbor_capture_server.py
new file mode 100644
index 0000000000..1697fef198
--- /dev/null
+++ b/tests/envs/test_harbor_capture_server.py
@@ -0,0 +1,125 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""Port-ownership guarantees for the capture proxy.
+
+A capture server that reports healthy while a *different* process owns its port is the worst
+failure this layer has: sessions are minted in one registry and validated against another, so the
+agent is rejected with 401, every rollout reports zero model calls, and the UI shows a live view
+that can never advance. Nothing in that chain names the port, so these tests pin the invariant that
+`start()` refuses rather than proceeds.
+
+No credentials and no engine are needed: `start()` binds a socket and never contacts `llm_url`.
+"""
+
+from __future__ import annotations
+
+import socket
+
+import pytest
+
+harbor_runner = pytest.importorskip("openenv.harbor.runner")
+
+CaptureServer = harbor_runner.CaptureServer
+
+# Never contacted. Discard port, so a stray request would fail loudly rather than reach a real host.
+UNUSED_ENGINE = "http://127.0.0.1:9/v1"
+
+
+def _free_port() -> int:
+ with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+ sock.bind(("127.0.0.1", 0))
+ return int(sock.getsockname()[1])
+
+
+@pytest.fixture
+def capture():
+ """Yield a factory that tears every server it built back down."""
+ built = []
+
+ def make(port: int) -> CaptureServer:
+ server = CaptureServer(llm_url=UNUSED_ENGINE, model="test-model", port=port)
+ built.append(server)
+ return server
+
+ yield make
+ for server in reversed(built):
+ server.stop()
+
+
+def test_health_reports_this_instance(capture):
+ """`/health` must identify which app answered, so a probe can check identity not reachability."""
+ import httpx
+
+ server = capture(_free_port())
+ server.start()
+
+ payload = httpx.get(f"http://127.0.0.1:{server.port}/health", timeout=5.0).json()
+ assert payload["instance"] == server.app.state.instance_id
+ assert payload["status"] == "ok"
+
+
+def test_instance_ids_are_distinct(capture):
+ """Two apps must never share an id, or the identity probe cannot tell them apart."""
+ first, second = capture(_free_port()), capture(_free_port())
+ assert first.app.state.instance_id != second.app.state.instance_id
+
+
+def test_start_refuses_a_port_another_server_holds(capture):
+ """The regression: a second server on a held port used to report healthy.
+
+ Its own uvicorn fails to bind on a background thread where nothing observes the error, while the
+ liveness probe connects successfully to the *incumbent*. Reachability is not ownership.
+ """
+ port = _free_port()
+ incumbent = capture(port)
+ incumbent.start()
+
+ intruder = capture(port)
+ with pytest.raises(RuntimeError, match=f"{port}"):
+ intruder.start()
+
+ # The incumbent must be untouched: a failed start may not disturb a working server.
+ import httpx
+
+ payload = httpx.get(f"http://127.0.0.1:{port}/health", timeout=5.0).json()
+ assert payload["instance"] == incumbent.app.state.instance_id
+
+
+def test_start_refuses_a_port_held_by_a_non_capture_listener(capture):
+ """Any listener counts, not just another capture server."""
+ with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as squatter:
+ squatter.bind(("127.0.0.1", 0))
+ squatter.listen(1)
+ port = int(squatter.getsockname()[1])
+
+ with pytest.raises(RuntimeError, match="already in use"):
+ capture(port).start()
+
+
+def test_start_succeeds_on_a_free_port_after_a_refusal(capture):
+ """A refusal must leave no state behind that breaks the next attempt."""
+ port = _free_port()
+ capture(port).start()
+
+ with pytest.raises(RuntimeError):
+ capture(port).start()
+
+ recovered = capture(_free_port())
+ recovered.start()
+ assert recovered._thread is not None and recovered._thread.is_alive()
+
+
+def test_stop_releases_the_port(capture):
+ """Otherwise a restart in the same process hits the new guard and looks like a collision."""
+ port = _free_port()
+ server = capture(port)
+ server.start()
+ server.stop()
+
+ successor = capture(port)
+ successor.start()
+ assert successor.app.state.instance_id != server.app.state.instance_id
diff --git a/tests/envs/test_harbor_capture_trace_entries.py b/tests/envs/test_harbor_capture_trace_entries.py
new file mode 100644
index 0000000000..d91579fd57
--- /dev/null
+++ b/tests/envs/test_harbor_capture_trace_entries.py
@@ -0,0 +1,121 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""The loop-owning training endpoint, and the contract types it serves.
+
+`GET /sessions/{id}/trace_entries` exists because a loop-owning consumer -- an external agent that
+drives its own tool loop, opencode or codex or claude-code -- wants per-model-call records, not the
+stitched sequence document `/rollout` returns. `to_trace_entries` already produced that shape but
+could not be asked for it over HTTP, because it needs the session's graph and that is server-side
+state.
+
+These tests pin the two things a consumer depends on and cannot check for itself:
+
+ * the endpoint answers with `{"session_id", "entries"}` and 404s an unknown id rather than 500ing,
+ so a caller can distinguish "no such rollout" from "the server broke";
+ * `TraceEntry` carries exactly the five keys the record is defined to have. A consumer builds
+ training rows off those key names, so a rename is a silent breakage -- the trainer would read
+ empty token fields and report a rollout that learned nothing.
+
+No engine is contacted: `llm_url` is the discard port, so a stray request would fail loudly.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+pytest.importorskip("fastapi")
+capture_server = pytest.importorskip("openenv.core.harness.capture.server")
+from fastapi.testclient import TestClient # noqa: E402
+from openenv.core.harness import LoopOwningSession, TraceEntry # noqa: E402
+
+
+# Never contacted. Discard port, so a request that escaped would fail rather than reach a host.
+UNUSED_ENGINE = "http://127.0.0.1:9/v1"
+
+
+@pytest.fixture
+def client() -> TestClient:
+ return TestClient(capture_server.create_app(llm_url=UNUSED_ENGINE, model="unused"))
+
+
+def test_unknown_session_is_404_not_500(client: TestClient) -> None:
+ # A 500 here would be read as "the capture server is broken" and send someone to the wrong
+ # place; the distinction between an unknown rollout and a broken server has to survive.
+ response = client.get("/sessions/no-such-session/trace_entries")
+ assert response.status_code == 404
+ assert response.json() == {"error": "unknown session"}
+
+
+def test_fresh_session_returns_an_empty_entry_list(client: TestClient) -> None:
+ session_id = client.post("/sessions", json={}).json()["session_id"]
+
+ response = client.get(f"/sessions/{session_id}/trace_entries")
+
+ assert response.status_code == 200
+ body = response.json()
+ assert body["session_id"] == session_id
+ # Empty, not absent: a rollout that captured nothing yet is a valid answer, and a consumer
+ # must be able to tell it apart from a malformed reply.
+ assert body["entries"] == []
+
+
+def test_trace_entry_carries_exactly_the_documented_keys() -> None:
+ # Consumers index these by name to build training rows, so a rename breaks them silently --
+ # the token fields simply read empty and the rollout looks like it learned nothing.
+ #
+ # `prompt_token_ids`, `loss_mask`, `reward` and `metadata` were added 2026-09. The first is the
+ # load-bearing one: without it a consumer must re-render the prompt with apply_chat_template,
+ # which matched the engine on 0 of 28 measured turns on Qwen3.5-4B and collapsed a run at its
+ # first weight update.
+ assert set(TraceEntry.__annotations__) == {
+ "request",
+ "response",
+ "prompt_token_ids",
+ "completion_token_ids",
+ "completion_tokens",
+ "per_token_logps",
+ "loss_mask",
+ "reward",
+ "metadata",
+ }
+
+
+def test_trace_entry_is_total_false_so_partial_records_are_legal() -> None:
+ # An eval-tier rollout has no token fields by design. If the record required them, every
+ # eval capture would be a type error rather than a legitimately partial record.
+ assert TraceEntry.__total__ is False
+
+
+def test_loop_owning_session_protocol_is_structural() -> None:
+ """A session satisfies the protocol by shape, never by inheritance.
+
+ That freedom is the whole point: opencode reads a file out of its sandbox, the capture proxy
+ answers over HTTP, and the consumer distinguishes neither.
+ """
+
+ class ReadsAFile:
+ def wait_for_completion(self, timeout_s: float | None = None) -> int:
+ return 0
+
+ def fetch_proxy_trace(self) -> list[TraceEntry]:
+ return []
+
+ class CallsAServer:
+ def wait_for_completion(self, timeout_s: float | None = None) -> int:
+ return 0
+
+ def fetch_proxy_trace(self) -> list[TraceEntry]:
+ return [{"request": {}, "response": {}, "completion_token_ids": [1]}]
+
+ for candidate in (ReadsAFile(), CallsAServer()):
+ assert isinstance(candidate, LoopOwningSession)
+
+ class MissingTheTrace:
+ def wait_for_completion(self, timeout_s: float | None = None) -> int:
+ return 0
+
+ assert not isinstance(MissingTheTrace(), LoopOwningSession)
diff --git a/tests/envs/test_harbor_capture_validate.py b/tests/envs/test_harbor_capture_validate.py
new file mode 100644
index 0000000000..82cc4fe72f
--- /dev/null
+++ b/tests/envs/test_harbor_capture_validate.py
@@ -0,0 +1,161 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""Validation on ingest, and the capability checks that run before a rollout starts.
+
+Both exist to convert a silent wrong answer into a loud one. A turn whose logprobs are misaligned
+has to be caught while we still know which turn it was; a sandbox that cannot be constructed has to
+be caught before it is offered rather than 90 seconds into a run.
+"""
+
+from __future__ import annotations
+
+import sys
+import types
+
+import pytest
+
+validate = pytest.importorskip("openenv.core.harness.capture.validate")
+capabilities = pytest.importorskip("openenv.harbor.capabilities")
+
+check_turn = validate.check_turn
+
+
+def codes(report) -> set[str]:
+ return {f.code for f in report.findings}
+
+
+# --- per-turn validation ----------------------------------------------------
+def test_a_well_formed_turn_passes():
+ report = check_turn([1, 2, 3], [4, 5], [-0.1, -0.2], finish_reason="stop")
+ assert report.ok
+
+
+def test_missing_prompt_ids_is_fatal():
+ """The endpoint was started without token-id capture: every rebuilt row would be empty."""
+ report = check_turn([], [4], [-0.1])
+ assert not report.ok
+ assert "no_prompt_ids" in codes(report)
+
+
+def test_logprob_count_must_match_sampled_count():
+ """Off-by-one here silently trains on the wrong token's probability."""
+ report = check_turn([1], [4, 5, 6], [-0.1, -0.2])
+ assert not report.ok
+
+
+def test_a_turn_that_sampled_nothing_is_reported_but_not_fatal():
+ """Reported, not rejected: a model can legitimately stop without emitting a token.
+
+ `ok` means usable, so an empty completion warns rather than invalidating the rollout. It still
+ has to be visible, because a run of these means the agent is looping without producing anything.
+ """
+ report = check_turn([1, 2], [], [])
+ assert report.ok
+ assert "no_sampled_ids" in codes(report)
+
+
+def test_findings_name_the_turn():
+ """A misalignment is only actionable if you know which call produced it."""
+ report = check_turn([], [1], [-0.1], index=7)
+ assert any("turn 7" in str(f) for f in report.findings)
+
+
+# --- sandbox capability -----------------------------------------------------
+def _module_with(**flags):
+ module = types.ModuleType("fake_backend_module")
+ for key, value in flags.items():
+ setattr(module, key, value)
+ sys.modules[module.__name__] = module
+ return module
+
+
+def test_missing_sdk_is_detected_from_the_backends_own_flag():
+ """Harbor guards each SDK with a module-level `_HAS_X` and raises from `__init__`.
+
+ So the module imports, the class loads, and the check passes, with the failure arriving only
+ once a rollout tries to build a sandbox, where it reads as a broken rollout rather than a
+ missing dependency. This is the case that shipped a Space offering `e2b` it could never run.
+ """
+ module = _module_with(_HAS_E2B=False)
+ cls = type("E2BEnvironment", (), {"__module__": module.__name__})
+ detail = capabilities._missing_sdk(cls)
+ assert detail and "e2b" in detail
+ assert "harbor[cloud]" in detail or "openenv[harbor]" in detail
+
+
+def test_present_sdk_reports_nothing():
+ module = _module_with(_HAS_E2B=True)
+ cls = type("E2BEnvironment", (), {"__module__": module.__name__})
+ assert capabilities._missing_sdk(cls) == ""
+
+
+def test_a_backend_without_flags_is_not_assumed_broken():
+ module = _module_with(SOMETHING_ELSE=1)
+ cls = type("Plain", (), {"__module__": module.__name__})
+ assert capabilities._missing_sdk(cls) == ""
+
+
+def test_several_missing_extras_are_all_named():
+ module = _module_with(_HAS_MODAL=False, _HAS_DOCKERFILE_PARSE=False)
+ cls = type("ModalEnvironment", (), {"__module__": module.__name__})
+ detail = capabilities._missing_sdk(cls)
+ assert "modal" in detail and "dockerfile_parse" in detail
+
+
+def test_an_unimportable_module_is_not_a_crash():
+ cls = type("Ghost", (), {"__module__": "module.that.does.not.exist"})
+ assert capabilities._missing_sdk(cls) == ""
+
+
+def test_unknown_sandbox_names_are_rejected_by_name():
+ status = capabilities.check_sandbox("not-a-real-backend")
+ assert status.available is False
+ assert "unknown" in status.detail.lower() or "harbor" in status.detail.lower()
+
+
+# --- capability reporting ---------------------------------------------------
+def test_render_says_why_a_sandbox_is_unavailable():
+ """The commonest cause of a rollout dying 90s in is a missing key, so it belongs at startup."""
+ caps = capabilities.Capabilities(
+ sandboxes=[
+ capabilities.SandboxStatus("e2b", True),
+ capabilities.SandboxStatus("daytona", False, "DAYTONA_API_KEY is not set"),
+ ]
+ )
+ out = caps.render()
+ assert "DAYTONA_API_KEY is not set" in out
+ assert caps.available_sandboxes == ["e2b"]
+
+
+def test_render_warns_when_nothing_is_usable():
+ caps = capabilities.Capabilities(
+ sandboxes=[capabilities.SandboxStatus("e2b", False, "no key")]
+ )
+ assert "WARNING" in caps.render()
+
+
+def test_capabilities_serialise_for_the_wire():
+ caps = capabilities.Capabilities(
+ sandboxes=[capabilities.SandboxStatus("e2b", True)],
+ llm={"model": "m", "ok": True},
+ )
+ payload = caps.to_dict()
+ assert set(payload) == {"harnesses", "sandboxes", "datasets", "llm"}
+ assert payload["llm"]["ok"] is True
+
+
+def test_the_install_hint_does_not_point_at_the_unsatisfiable_extra():
+ """`harbor[cloud]` cannot be installed: langsmith and tensorlake demand incompatible websockets.
+
+ Both pyprojects avoid it for that reason, so an error message telling someone to install it
+ would send them straight to a resolver failure.
+ """
+ module = _module_with(_HAS_DAYTONA=False)
+ cls = type("DaytonaEnvironment", (), {"__module__": module.__name__})
+ detail = capabilities._missing_sdk(cls)
+ assert "harbor[cloud]" not in detail
+ assert "openenv[harbor]" in detail
diff --git a/tests/envs/test_harbor_contract_export.py b/tests/envs/test_harbor_contract_export.py
new file mode 100644
index 0000000000..763f94fb13
--- /dev/null
+++ b/tests/envs/test_harbor_contract_export.py
@@ -0,0 +1,123 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""What the trainer-facing contract includes, and what it must exclude.
+
+Both directions are silent when wrong. Dropping agent turns trains on part of a rollout while
+reporting the whole reward; including auxiliary turns credits a next-speaker classification with
+solving the task.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+contract = pytest.importorskip("openenv.core.harness.capture.contract")
+graph_mod = pytest.importorskip("openenv.core.harness.capture.graph")
+
+RolloutGraph = graph_mod.RolloutGraph
+TurnNode = graph_mod.TurnNode
+
+
+def node(node_id: str, prompt: list[int], sampled: list[int]) -> TurnNode:
+ return TurnNode(
+ node_id=node_id,
+ prompt_ids=prompt,
+ sampled_ids=sampled,
+ sampled_logprobs=[-0.1] * len(sampled),
+ request_messages=[{"role": "user", "content": "hi"}],
+ response_message={"content": "ok"},
+ )
+
+
+@pytest.fixture
+def two_agent_roots():
+ """A harness that rewrote its system prompt mid-run, so the rollout has two agent roots."""
+ g = RolloutGraph()
+ g.add_turn(node("a1", [1], [2, 3]))
+ g.add_turn(node("a2", [1, 2, 3, 9], [4]))
+ g.add_turn(node("b1", [500], [6]))
+ document = {
+ "sequences": [
+ {"role": "agent", "root_id": "a1", "node_ids": ["a1", "a2"]},
+ {"role": "agent", "root_id": "b1", "node_ids": ["b1"]},
+ ]
+ }
+ return g, document
+
+
+def test_every_agent_root_is_exported(two_agent_roots):
+ """The regression: only the first agent sequence was kept, so later roots vanished."""
+ g, document = two_agent_roots
+ assert [n.node_id for n in contract._agent_nodes(g, document)] == ["a1", "a2", "b1"]
+
+
+def test_turn_records_cover_every_agent_turn(two_agent_roots):
+ g, document = two_agent_roots
+ records = contract.to_turn_records(g, document)
+ assert len(records) == 3
+ for prompt_ids, output_ids, logps in records:
+ assert output_ids, "a turn with no sampled tokens is not trainable"
+ assert len(output_ids) == len(logps)
+
+
+def test_trace_entries_cover_every_agent_turn(two_agent_roots):
+ g, document = two_agent_roots
+ assert len(contract.to_trace_entries(g, document)) == 3
+
+
+def test_auxiliary_sequences_are_excluded():
+ """An aux call must never be credited with the reward the agent earned."""
+ g = RolloutGraph()
+ g.add_turn(node("agent", [1], [2]))
+ g.add_turn(node("aux", [900], [3]))
+ document = {
+ "sequences": [
+ {"role": "agent", "root_id": "agent", "node_ids": ["agent"]},
+ {"role": "auxiliary", "root_id": "aux", "node_ids": ["aux"]},
+ ]
+ }
+ assert [n.node_id for n in contract._agent_nodes(g, document)] == ["agent"]
+
+
+def test_discarded_sequences_are_excluded():
+ g = RolloutGraph()
+ g.add_turn(node("kept", [1], [2]))
+ g.add_turn(node("dead", [1], [3]))
+ document = {
+ "sequences": [
+ {"role": "agent", "root_id": "kept", "node_ids": ["kept"]},
+ {"role": "discarded", "root_id": "kept", "node_ids": ["dead"]},
+ ]
+ }
+ assert [n.node_id for n in contract._agent_nodes(g, document)] == ["kept"]
+
+
+def test_a_node_shared_by_two_paths_appears_once():
+ """Forked paths share their prefix; the shared turn must not be exported twice."""
+ g = RolloutGraph()
+ g.add_turn(node("shared", [1], [2]))
+ g.add_turn(node("left", [1, 2], [3]))
+ g.add_turn(node("right", [1, 2], [4]))
+ document = {
+ "sequences": [
+ {"role": "agent", "root_id": "shared", "node_ids": ["shared", "left"]},
+ {"role": "agent", "root_id": "shared", "node_ids": ["shared", "right"]},
+ ]
+ }
+ ids = [n.node_id for n in contract._agent_nodes(g, document)]
+ assert ids.count("shared") == 1
+ assert set(ids) == {"shared", "left", "right"}
+
+
+def test_no_agent_sequences_is_empty_not_an_error():
+ g = RolloutGraph()
+ g.add_turn(node("aux", [1], [2]))
+ document = {
+ "sequences": [{"role": "auxiliary", "root_id": "aux", "node_ids": ["aux"]}]
+ }
+ assert contract._agent_nodes(g, document) == []
+ assert contract.to_turn_records(g, document) == []
diff --git a/tests/envs/test_harbor_e2b_stream.py b/tests/envs/test_harbor_e2b_stream.py
new file mode 100644
index 0000000000..073949f101
--- /dev/null
+++ b/tests/envs/test_harbor_e2b_stream.py
@@ -0,0 +1,96 @@
+import asyncio
+from types import SimpleNamespace
+
+import pytest
+
+pytest.importorskip("harbor.environments.e2b")
+
+from openenv.harbor.e2b_stream import E2BStreamingEnvironment
+from tenacity import wait_none
+
+
+def environment(files):
+ env = object.__new__(E2BStreamingEnvironment)
+ env._sandbox = SimpleNamespace(files=files)
+ return env
+
+
+def test_directory_upload_preserves_bytes_paths_and_closes_streams(tmp_path):
+ (tmp_path / "nested").mkdir()
+ (tmp_path / "nested/binary").write_bytes(bytes(range(256)) * 100)
+ (tmp_path / "text").write_text("hello\n")
+ observed, handles = {}, []
+
+ async def write_files(entries, **kwargs):
+ assert kwargs == {"gzip": True, "use_octet_stream": True, "request_timeout": 30}
+ for entry in entries:
+ handles.append(entry["data"])
+ observed[entry["path"]] = entry["data"].read()
+
+ env = environment(SimpleNamespace(write_files=write_files))
+ asyncio.run(env.upload_dir(tmp_path, "/logs/agent"))
+ assert observed == {
+ "/logs/agent/nested/binary": bytes(range(256)) * 100,
+ "/logs/agent/text": b"hello\n",
+ }
+ assert all(handle.closed for handle in handles)
+
+
+def test_retry_reopens_source_and_replays_identical_file_bytes(tmp_path, monkeypatch):
+ source = tmp_path / "log"
+ source.write_bytes(b"exact evidence")
+ seen, handles = [], []
+
+ async def write(path, stream, **kwargs):
+ handles.append(stream)
+ seen.append((path, stream.read()))
+ if len(seen) == 1:
+ raise TimeoutError("transport stalled")
+
+ monkeypatch.setattr(E2BStreamingEnvironment.upload_file.retry, "wait", wait_none())
+ asyncio.run(
+ environment(SimpleNamespace(write=write)).upload_file(source, "/logs/log")
+ )
+ assert seen == [("/logs/log", b"exact evidence")] * 2
+ assert all(handle.closed for handle in handles)
+
+
+def test_hung_upload_has_total_deadline_and_bounded_retries(tmp_path, monkeypatch):
+ (tmp_path / "log").write_text("data")
+ deadlines, calls = [], []
+ real_wait_for = asyncio.wait_for
+
+ async def bounded(coro, timeout):
+ deadlines.append(timeout)
+ return await real_wait_for(coro, timeout=0.005)
+
+ async def write_files(entries, **kwargs):
+ calls.extend(entry["data"] for entry in entries)
+ await asyncio.Event().wait()
+
+ monkeypatch.setattr(asyncio, "wait_for", bounded)
+ monkeypatch.setattr(E2BStreamingEnvironment.upload_dir.retry, "wait", wait_none())
+ with pytest.raises(TimeoutError):
+ asyncio.run(
+ environment(SimpleNamespace(write_files=write_files)).upload_dir(
+ tmp_path, "/logs"
+ )
+ )
+ assert deadlines == [120, 120]
+ assert len(calls) == 2 and all(handle.closed for handle in calls)
+
+
+def test_cancellation_is_propagated_without_retry(tmp_path):
+ source = tmp_path / "log"
+ source.write_bytes(b"data")
+ calls = []
+
+ async def write(path, stream, **kwargs):
+ calls.append(stream)
+ raise asyncio.CancelledError
+
+ with pytest.raises(asyncio.CancelledError):
+ asyncio.run(
+ environment(SimpleNamespace(write=write)).upload_file(source, "/logs/log")
+ )
+ assert len(calls) == 1 and calls[0].closed
diff --git a/tests/envs/test_harbor_forwarding_lifecycle.py b/tests/envs/test_harbor_forwarding_lifecycle.py
new file mode 100644
index 0000000000..0a44a300c8
--- /dev/null
+++ b/tests/envs/test_harbor_forwarding_lifecycle.py
@@ -0,0 +1,67 @@
+"""A live forwarder must not block when its child fills stdout or stderr."""
+
+import subprocess
+import sys
+import time
+from types import SimpleNamespace
+
+from openenv.core.harness.capture.forwarding import GradioForwarder
+
+
+def test_gradio_drains_both_pipes_and_stops_only_its_child(monkeypatch, tmp_path):
+ finished = tmp_path / "both-pipes-written"
+ children = []
+ unrelated = SimpleNamespace(share_token="other", proc=None)
+ tunnels = [unrelated]
+
+ def setup_tunnel(**kwargs):
+ process = subprocess.Popen(
+ [
+ sys.executable,
+ "-c",
+ (
+ "import os, pathlib, sys, time; "
+ "os.write(1, b'x' * 1048576 + b'\\n'); "
+ "os.write(2, b'y' * 1048576 + b'\\n'); "
+ "pathlib.Path(sys.argv[1]).write_text('done'); time.sleep(60)"
+ ),
+ str(finished),
+ ],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ )
+ children.append(process)
+ tunnel = SimpleNamespace(share_token=kwargs["share_token"], proc=process)
+
+ def kill():
+ process.terminate()
+ tunnel.proc = None
+
+ tunnel.kill = kill
+ tunnels.append(tunnel)
+ return "https://test.invalid"
+
+ monkeypatch.setitem(
+ sys.modules, "gradio.networking", SimpleNamespace(setup_tunnel=setup_tunnel)
+ )
+ monkeypatch.setitem(
+ sys.modules, "gradio.tunneling", SimpleNamespace(CURRENT_TUNNELS=tunnels)
+ )
+ forwarder = GradioForwarder()
+ try:
+ assert forwarder.start(8123) == "https://test.invalid"
+ deadline = time.monotonic() + 5
+ while not finished.exists() and time.monotonic() < deadline:
+ time.sleep(0.02)
+ assert finished.exists(), "forwarder blocked on a full child-process pipe"
+ forwarder.stop()
+ assert children[0].poll() is not None
+ assert unrelated in tunnels
+ forwarder.stop()
+ finally:
+ for process in children:
+ if process.poll() is None:
+ process.kill()
+ process.wait(timeout=5)
+ process.stdout.close()
+ process.stderr.close()
diff --git a/tests/envs/test_harbor_google_signature.py b/tests/envs/test_harbor_google_signature.py
new file mode 100644
index 0000000000..6eb1650892
--- /dev/null
+++ b/tests/envs/test_harbor_google_signature.py
@@ -0,0 +1,44 @@
+"""Google's bytes fields must survive strict SDK JSON decoding."""
+
+import base64
+
+from openenv.core.harness.capture.dialects.google import (
+ _GoogleStreamState,
+ GoogleTransformer,
+)
+from openenv.core.harness.capture.dialects.reasoning import make_signature
+
+
+def _assert_signature(response):
+ part = response["candidates"][0]["content"]["parts"][0]
+ assert part["thought"] is True
+ assert base64.b64decode(
+ part["thoughtSignature"], validate=True
+ ).decode() == make_signature(part["text"])
+
+
+def test_google_buffered_thought_signature_is_json_bytes():
+ response = GoogleTransformer().transform_response(
+ {
+ "choices": [
+ {
+ "message": {
+ "reasoning_content": "Inspect the CSV first.",
+ "content": "Working.",
+ },
+ "finish_reason": "stop",
+ }
+ ]
+ },
+ {},
+ )
+ _assert_signature(response)
+
+
+def test_google_streamed_thought_signature_is_json_bytes():
+ responses = _GoogleStreamState(GoogleTransformer()).process_chunk(
+ {"choices": [{"delta": {"reasoning_content": "Inspect the CSV first."}}]}
+ )
+ assert responses
+ for response in responses:
+ _assert_signature(response)
diff --git a/tests/envs/test_harbor_hosted_serving.py b/tests/envs/test_harbor_hosted_serving.py
new file mode 100644
index 0000000000..5bf9355a45
--- /dev/null
+++ b/tests/envs/test_harbor_hosted_serving.py
@@ -0,0 +1,180 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""Serving on a hosted platform, where there is one port and one URL.
+
+Locally the capture proxy runs on its own port and is published to the sandbox. A Space exposes
+exactly one port and already has a public URL, so the proxy is mounted onto the env server's app
+instead and nothing is forwarded. These tests pin that split, because getting it wrong is not a
+crash: it is a deployment that quietly opens a second listener it cannot publish.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+serving = pytest.importorskip("openenv.harbor.serving")
+
+CAPTURE_MOUNT = serving.CAPTURE_MOUNT
+HarborService = serving.HarborService
+space_public_url = serving.space_public_url
+
+# Never contacted: nothing here reaches an engine.
+UNUSED_LLM = "http://127.0.0.1:9/v1"
+
+
+@pytest.fixture(autouse=True)
+def _clear_space_env(monkeypatch):
+ """Tests must not inherit a Space identity from the developer's shell."""
+ monkeypatch.delenv("SPACE_HOST", raising=False)
+ monkeypatch.delenv("SPACE_ID", raising=False)
+
+
+def test_no_space_means_no_public_url():
+ assert space_public_url() == ""
+
+
+def test_space_host_is_used_verbatim(monkeypatch):
+ monkeypatch.setenv("SPACE_HOST", "owner-env.hf.space")
+ assert space_public_url() == "https://owner-env.hf.space"
+
+
+def test_space_host_tolerates_a_scheme_already_present(monkeypatch):
+ monkeypatch.setenv("SPACE_HOST", "https://owner-env.hf.space/")
+ assert space_public_url() == "https://owner-env.hf.space"
+
+
+def test_space_id_is_slugged_when_host_is_absent(monkeypatch):
+ """`SPACE_ID` is always set; the hostname lowercases and dash-separates it."""
+ monkeypatch.setenv("SPACE_ID", "AdithyaSK/harbor_data.agent-env")
+ assert space_public_url() == "https://adithyask-harbor-data-agent-env.hf.space"
+
+
+def test_hosted_start_mounts_and_never_forwards(monkeypatch):
+ """The regression that got a Space flagged: a hosted deployment must not forward."""
+ monkeypatch.setenv("SPACE_ID", "owner/env")
+
+ def explode(*_args, **_kwargs):
+ raise AssertionError("a hosted deployment must not create a forwarder")
+
+ monkeypatch.setattr(
+ "openenv.core.harness.capture.forwarding.make_forwarder", explode, raising=False
+ )
+
+ service = HarborService(llm_url=UNUSED_LLM, model="m", datasets=[])
+ url = service.start()
+
+ assert service.mounted is True
+ assert url == f"https://owner-env.hf.space{CAPTURE_MOUNT}"
+ # No port was bound, so stop() must be safe even though start() never launched a server.
+ service.stop()
+
+
+def test_mounted_capture_answers_under_the_prefix():
+ """Mounting strips the prefix, so every dialect route keeps working unchanged."""
+ fastapi = pytest.importorskip("fastapi")
+ from fastapi.testclient import TestClient
+ from openenv.core.harness.capture.server import create_app
+
+ host = fastapi.FastAPI()
+ host.mount(CAPTURE_MOUNT, create_app(llm_url=UNUSED_LLM, model="m"))
+ client = TestClient(host)
+
+ assert client.get(f"{CAPTURE_MOUNT}/health").json()["status"] == "ok"
+
+ # The proxy's catch-all must match /v1/chat/completions, not /capture/v1/chat/completions.
+ response = client.post(
+ f"{CAPTURE_MOUNT}/v1/chat/completions",
+ json={"model": "m", "messages": [{"role": "user", "content": "hi"}]},
+ headers={"Authorization": "Bearer not-a-session"},
+ )
+ # 401 rather than 404 proves it routed to the proxy and was rejected on identity, which is also
+ # what stops a publicly mounted proxy being an open relay.
+ assert response.status_code == 401
+ assert "unknown API key" in response.json()["error"]["message"]
+
+
+def test_a_failed_forwarder_does_not_leave_the_capture_server_running(monkeypatch):
+ """A half-started service poisons every later attempt.
+
+ The capture server binds its port and starts a thread before the forwarder is built. If the
+ forwarder then fails, leaving that server up means the next `start()` fails on a port conflict
+ that says nothing about the real error.
+ """
+ stopped: list[bool] = []
+
+ class FakeCapture:
+ port = 8123
+
+ def start(self):
+ pass
+
+ def stop(self):
+ stopped.append(True)
+
+ def explode(*_args, **_kwargs):
+ raise RuntimeError("cloudflared is not installed")
+
+ monkeypatch.setattr(
+ "openenv.core.harness.capture.forwarding.make_forwarder", explode, raising=False
+ )
+
+ service = HarborService(llm_url=UNUSED_LLM, model="m", datasets=[])
+ service.capture = FakeCapture()
+
+ with pytest.raises(RuntimeError, match="cloudflared"):
+ service.start()
+
+ assert stopped == [True], "the capture server was left holding its port"
+ assert service.public_url in (None, "")
+
+
+def test_a_space_that_cannot_probe_does_not_claim_to_be_trainable(
+ monkeypatch, tmp_path
+):
+ """The Space entry point defaulted `_CAPTURE_LEVEL` to "tokens" and only corrected it when a model
+ resolved AND the probe succeeded. An ambiguous model list, an unset model or a raising probe left
+ it at token level, so the proxy was built for capture and every rollout was stamped trainable —
+ the exact mislabelling the capture level exists to prevent. Unknown must mean the weaker tier.
+ """
+ import runpy
+
+ monkeypatch.setenv("OPENENV_LLM_URL", "http://127.0.0.1:9/v1")
+ monkeypatch.delenv("OPENENV_MODEL", raising=False)
+ monkeypatch.delenv("SPACE_HOST", raising=False)
+ monkeypatch.delenv("SPACE_ID", raising=False)
+ # Serves two models and names neither, so nothing resolves and the probe never runs.
+ #
+ # Patched through the module OBJECT, not the dotted string: the package re-exports a function
+ # called `validate_llm` which shadows the same-named submodule, so the string form resolves to the
+ # function and monkeypatch fails with "'function' object has no attribute 'list_models'".
+ import importlib
+
+ validate_llm_mod = importlib.import_module(
+ "openenv.core.harness.capture.validate_llm"
+ )
+ monkeypatch.setattr(validate_llm_mod, "list_models", lambda *a, **k: ["one", "two"])
+ # The service must not be started for real; capture the level it would have been built with.
+ built: dict = {}
+
+ class FakeService:
+ def __init__(self, **kwargs):
+ built.update(kwargs)
+
+ def start(self):
+ return ""
+
+ @classmethod
+ def set_current(cls, _service):
+ pass
+
+ monkeypatch.setattr(serving, "HarborService", FakeService)
+ monkeypatch.setattr(serving, "build_app", lambda **kwargs: kwargs)
+
+ runpy.run_module("harbor_env.server.app", run_name="not_main")
+ assert built.get("capture_level") == "text", (
+ "an unprobed endpoint must default to the weakest tier, never to tokens"
+ )
diff --git a/tests/envs/test_harbor_install_fixes.py b/tests/envs/test_harbor_install_fixes.py
new file mode 100644
index 0000000000..355ed5cca6
--- /dev/null
+++ b/tests/envs/test_harbor_install_fixes.py
@@ -0,0 +1,320 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""The upstream Harbor defect that cost `openclaw` its ATIF trajectory.
+
+The sibling `hermes` fix went away with the seam itself: hermes-agent fails to install
+(`exit 127`, 5/5 attempts), so there is nothing left to intercept.
+
+Both failed the same way: no error, no exception, no missing file -- just an agent that quietly
+produced no trace, so every rollout reported `atif=none` and the cross-check silently did not exist.
+Neither is detectable from a passing rollout, which is why they are pinned here.
+"""
+
+from __future__ import annotations
+
+import json
+
+import pytest
+
+install_fixes = pytest.importorskip("openenv.harbor.install_fixes")
+openclaw_mod = pytest.importorskip("harbor.agents.installed.openclaw")
+
+TRIM = install_fixes._OPENCLAW_TRIM_TRAILING_LOG
+OpenClaw = openclaw_mod.OpenClaw
+
+_CONTAINER_PATH = "/logs/agent/openclaw.txt"
+_SESSION_FILE = "/root/.openclaw/agents/main/sessions/790c93f1.jsonl"
+
+
+def test_sqlite_export_preserves_per_call_usage_for_harbor(tmp_path, monkeypatch):
+ import subprocess
+ from types import SimpleNamespace
+
+ meta = {"sessionId": "session-1", "sessionFile": "agent:main:main"}
+ (tmp_path / "openclaw.txt").write_text(json.dumps({"meta": {"agentMeta": meta}}))
+ entries = [
+ {
+ "type": "message",
+ "message": {
+ "role": "assistant",
+ "content": [{"type": "text", "text": "answer"}],
+ "usage": {"input": 100 + count, "output": count},
+ },
+ }
+ for count in [137, 124, 96, 5]
+ ]
+ bundle = tmp_path / "bundle"
+ bundle.mkdir()
+ (bundle / "session-branch.json").write_text(json.dumps({"entries": entries}))
+
+ def run(command, **kwargs):
+ assert command[:5] == [
+ "openclaw",
+ "sessions",
+ "export-trajectory",
+ "--session-key",
+ "agent:main:main",
+ ]
+ assert kwargs["check"] and kwargs["timeout"] == 60
+ return SimpleNamespace(
+ stdout=json.dumps({"sessionId": "session-1", "outputDir": str(bundle)})
+ )
+
+ monkeypatch.setattr(subprocess, "run", run)
+ install_fixes._export_openclaw_sqlite_transcript(str(tmp_path))
+ target = tmp_path / "openclaw.session.jsonl"
+ assert [json.loads(line) for line in target.read_text().splitlines()] == entries
+ steps = openclaw_mod.openclaw_session_jsonl_to_atif_steps(
+ target, instruction="task", model_name="test"
+ )
+ assert [
+ step.metrics.completion_tokens for step in steps if step.source == "agent"
+ ] == [137, 124, 96, 5]
+
+
+def test_sqlite_export_rejects_other_session(tmp_path, monkeypatch):
+ import subprocess
+ from types import SimpleNamespace
+
+ (tmp_path / "openclaw.txt").write_text(
+ json.dumps(
+ {
+ "meta": {
+ "agentMeta": {
+ "sessionId": "expected",
+ "sessionFile": "agent:main:main",
+ }
+ }
+ }
+ )
+ )
+ monkeypatch.setattr(
+ subprocess,
+ "run",
+ lambda *args, **kwargs: SimpleNamespace(
+ stdout=json.dumps({"sessionId": "other"})
+ ),
+ )
+ with pytest.raises(ValueError, match="different session"):
+ install_fixes._export_openclaw_sqlite_transcript(str(tmp_path))
+ assert not (tmp_path / "openclaw.session.jsonl").exists()
+
+
+def test_sqlite_export_preserves_existing_native_jsonl(tmp_path, monkeypatch):
+ import subprocess
+
+ target = tmp_path / "openclaw.session.jsonl"
+ target.write_text("existing native transcript\n")
+ monkeypatch.setattr(
+ subprocess,
+ "run",
+ lambda *args, **kwargs: pytest.fail("must preserve legacy transcript"),
+ )
+ install_fixes._export_openclaw_sqlite_transcript(str(tmp_path))
+ assert target.read_text() == "existing native transcript\n"
+
+
+# The shape openclaw actually produces: a pretty-printed envelope whose closing brace sits at
+# column 0, and whose LAST nested object is the `completion` block. Both details matter below, and
+# the key ORDER is taken from a real capture file (`payloads` first, `meta` last) because the
+# backwards-scan trap depends on which nested object happens to be last.
+_ENVELOPE = {
+ "payloads": [],
+ "meta": {
+ "agentMeta": {"sessionId": "790c93f1", "sessionFile": _SESSION_FILE},
+ "completion": {"stopReason": "stop", "finishReason": "stop"},
+ },
+}
+# Harbor merges the agent's stderr into the same file with `2>&1`, so this lands after the JSON.
+_TRAILING_LOG = (
+ "[agents/agent-command] [agent] run 9e921697-bfe9-4266-ad60-6e9f65d0de5e "
+ "ended with stopReason=stop"
+)
+
+
+def _capture_file(with_trailing_log: bool = True) -> str:
+ body = json.dumps(_ENVELOPE, indent=2)
+ return f"{body}\n{_TRAILING_LOG}\n" if with_trailing_log else f"{body}\n"
+
+
+def _run_trim(tmp_path) -> str:
+ """Execute the real production trim script against a temp file, not a copy of its logic."""
+ target = tmp_path / "openclaw.txt"
+ script = TRIM.replace(_CONTAINER_PATH, str(target))
+ # If the constant is ever reworded, the substitution stops matching and this test would
+ # silently exercise nothing. Fail instead.
+ assert script != TRIM, f"{_CONTAINER_PATH!r} no longer appears in the trim script"
+ target.write_text(_capture_file(), encoding="utf-8")
+ exec(compile(script, "", "exec"), {})
+ return target.read_text(encoding="utf-8")
+
+
+# --- openclaw ---------------------------------------------------------------
+def test_harbor_cannot_parse_its_own_capture_file_when_openclaw_logs_after_the_json():
+ """The bug itself: one stderr line after the envelope and Harbor's parser gives up.
+
+ `_load_json_object` requires the JSON object to consume the entire remaining suffix, but Harbor's
+ own `2>&1` is what put a non-JSON line there. Returning None means `populate_context_post_run`
+ returns at `if not envelope` and no `trajectory.json` is ever written.
+ """
+ assert OpenClaw._load_json_object(_capture_file()) is None
+
+
+def test_trimming_the_trailing_log_line_makes_harbors_own_parser_succeed(tmp_path):
+ """The fix, stated as the only thing it is allowed to be: Harbor's parser does the parsing.
+
+ The subclass removes the trailing lines and nothing else, so the envelope that comes back is
+ Harbor's own -- including `agentMeta.sessionFile`, which is what the session copy needs.
+ """
+ parsed = OpenClaw._load_json_object(_run_trim(tmp_path))
+
+ assert parsed is not None
+ assert parsed["meta"]["agentMeta"]["sessionFile"] == _SESSION_FILE
+
+
+def test_trim_leaves_an_already_clean_capture_file_untouched(tmp_path):
+ """A run whose stopReason is `end_turn` logs nothing, so the file is already parseable."""
+ target = tmp_path / "openclaw.txt"
+ script = TRIM.replace(_CONTAINER_PATH, str(target))
+ clean = _capture_file(with_trailing_log=False)
+ target.write_text(clean, encoding="utf-8")
+
+ exec(compile(script, "", "exec"), {})
+
+ assert target.read_text(encoding="utf-8") == clean
+
+
+def test_trim_survives_a_capture_file_with_no_envelope_at_all(tmp_path):
+ """An agent that died before emitting JSON must not turn into a crash in our override."""
+ target = tmp_path / "openclaw.txt"
+ script = TRIM.replace(_CONTAINER_PATH, str(target))
+ garbage = "openclaw: command not found\n"
+ target.write_text(garbage, encoding="utf-8")
+
+ exec(compile(script, "", "exec"), {})
+
+ assert target.read_text(encoding="utf-8") == garbage
+
+
+def test_a_backwards_scan_without_the_suffix_rule_latches_onto_the_wrong_object():
+ """Why the fix trims text instead of loosening the parser -- the obvious loosening is wrong.
+
+ Dropping Harbor's "must consume the suffix" rule looks like the one-line fix. It is not: the scan
+ walks backwards, so the first thing that decodes is the LAST nested object, and `completion`
+ decodes perfectly. The caller then gets a dict with no `meta` at all and builds a degenerate
+ 2-step trajectory from it -- which still reports `atif=match`, because `reconcile` downgrades a
+ trace carrying no token counts instead of failing it. A silently wrong trace is worse than none.
+ """
+ text = _capture_file().strip()
+ decoder = json.JSONDecoder()
+ found = None
+ for start in range(len(text) - 1, -1, -1):
+ if text[start] != "{":
+ continue
+ try:
+ obj, _ = decoder.raw_decode(text[start:])
+ except ValueError:
+ continue
+ if isinstance(obj, dict):
+ found = obj
+ break
+
+ assert found == {"stopReason": "stop", "finishReason": "stop"}
+ assert "meta" not in found
+
+
+@pytest.mark.asyncio
+async def test_openhands_clean_install_still_prepares_local_runtime(monkeypatch):
+ """A dependency-successful install must not leave LocalRuntime invoking real Poetry."""
+ from unittest.mock import AsyncMock
+
+ monkeypatch.setattr(install_fixes.OpenHands, "install", AsyncMock())
+ agent = object.__new__(install_fixes.InterceptOpenHands)
+ root_exec = AsyncMock()
+ monkeypatch.setattr(agent, "exec_as_root", root_exec)
+ environment = object()
+ await agent.install(environment)
+ commands = [call.kwargs["command"] for call in root_exec.call_args_list]
+ assert len(commands) == 3
+ assert all('/opt/openhands-venv/bin/python "$@"' in command for command in commands)
+ assert any("/usr/local/bin/poetry" in command for command in commands)
+ assert any("/opt/openhands-venv/bin/poetry" in command for command in commands)
+
+
+def test_kimi_terminal_signal_does_not_kill_its_exec_transport():
+ import subprocess
+
+ wrapped = install_fixes._isolated_process_group("echo finished; kill 0")
+ completed = subprocess.run(
+ ["bash", "-c", wrapped],
+ start_new_session=True,
+ capture_output=True,
+ text=True,
+ timeout=5,
+ )
+ assert completed.returncode == 143
+ assert completed.stdout.strip() == "finished"
+
+
+@pytest.mark.asyncio
+async def test_openclaw_install_and_runtime_select_supported_node(monkeypatch):
+ from unittest.mock import AsyncMock
+
+ execution = AsyncMock()
+ monkeypatch.setattr(install_fixes.OpenClaw, "exec_as_agent", execution)
+ agent = object.__new__(install_fixes.InterceptOpenClaw)
+ await agent.exec_as_agent(
+ object(), command="nvm install 22 && nvm use 22 && openclaw --version"
+ )
+ assert (
+ execution.call_args.kwargs["command"]
+ == "nvm install 24.16.0 && nvm use 24.16.0 && openclaw --version"
+ )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ "adapter,parent",
+ [
+ (install_fixes.InterceptOpenClaw, install_fixes.OpenClaw),
+ (install_fixes.InterceptKimi, install_fixes.KimiCli),
+ ],
+)
+async def test_command_wrappers_preserve_harbor_positional_call_contract(
+ monkeypatch, adapter, parent
+):
+ from unittest.mock import AsyncMock
+
+ execution = AsyncMock(return_value="executed")
+ monkeypatch.setattr(parent, "exec_as_agent", execution)
+ agent = object.__new__(adapter)
+ environment = object()
+ env = {"TEST_SETTING": "test"}
+ result = await agent.exec_as_agent(environment, "echo ready", env, "/tmp", 30)
+ assert result == "executed"
+ execution.assert_awaited_once_with(
+ environment, command="echo ready", env=env, cwd="/tmp", timeout_sec=30
+ )
+
+
+def test_openclaw_catalog_model_id_is_local_to_its_provider(tmp_path):
+ from openenv.harbor.seams import get
+
+ selected, kwargs, env, _ = get("openclaw").resolve(
+ base_url="https://capture.example",
+ session="session-test",
+ model="Qwen/Qwen3.5-4B",
+ )
+ agent = install_fixes.InterceptOpenClaw(
+ logs_dir=tmp_path, model_name=selected, extra_env=env, **kwargs
+ )
+ config = agent._build_full_openclaw_config()
+ provider, model_id = selected.split("/", 1)
+ catalog = config["models"]["providers"][provider]
+ assert any(model["id"] == model_id for model in catalog["models"])
+ assert catalog["baseUrl"] == "https://capture.example/v1"
+ assert catalog["apiKey"] == "session-test"
diff --git a/tests/envs/test_harbor_native_provider.py b/tests/envs/test_harbor_native_provider.py
new file mode 100644
index 0000000000..8174fcee96
--- /dev/null
+++ b/tests/envs/test_harbor_native_provider.py
@@ -0,0 +1,337 @@
+"""Native provider semantics and honest capability classification."""
+
+import copy
+import json
+
+import httpx
+import pytest
+from openenv.core.harness.capture.providers import (
+ anthropic_request,
+ anthropic_response,
+ replay_anthropic,
+)
+from openenv.core.harness.capture.upstream import InferenceClient, UpstreamRequestError
+from openenv.core.harness.capture.validate_llm import validate_llm
+
+
+def native_message():
+ return {
+ "id": "msg_test",
+ "type": "message",
+ "role": "assistant",
+ "model": "claude-test",
+ "content": [
+ {
+ "type": "thinking",
+ "thinking": "Check first",
+ "signature": "signed-original",
+ },
+ {
+ "type": "tool_use",
+ "id": "toolu_1",
+ "name": "run",
+ "input": {"cmd": "pwd"},
+ },
+ ],
+ "stop_reason": "tool_use",
+ "stop_sequence": None,
+ "usage": {"input_tokens": 10, "output_tokens": 5, "cache_read_input_tokens": 7},
+ }
+
+
+def test_native_history_and_signatures_are_preserved_without_mutation():
+ original = {
+ "model": "alias",
+ "max_tokens": 200,
+ "stream": True,
+ "messages": [{"role": "assistant", "content": native_message()["content"]}],
+ }
+ before = copy.deepcopy(original)
+ body = anthropic_request(
+ {"_openenv_native_request": original, "max_tokens": 100}, "pinned"
+ )
+ assert original == before
+ assert body["messages"] == before["messages"]
+ assert (
+ body["model"] == "pinned"
+ and body["max_tokens"] == 100
+ and body["stream"] is False
+ )
+
+
+def test_chat_tool_roundtrip_keeps_ids_and_json():
+ response = anthropic_response(native_message())
+ assistant = response["choices"][0]["message"]
+ assistant.pop("reasoning_content")
+ body = anthropic_request(
+ {
+ "messages": [
+ assistant,
+ {"role": "tool", "tool_call_id": "toolu_1", "content": "/tmp"},
+ ]
+ },
+ "pinned",
+ )
+ call = body["messages"][0]["content"][0]
+ result = body["messages"][1]["content"][0]
+ assert call["id"] == result["tool_use_id"] == "toolu_1"
+ assert call["input"] == {"cmd": "pwd"}
+ assert response["usage"] == {
+ "prompt_tokens": 17,
+ "completion_tokens": 5,
+ "total_tokens": 22,
+ }
+ assert "prompt_token_ids" not in response
+ assert response["choices"][0]["logprobs"] is None
+
+
+def test_unsigned_reasoning_is_rejected():
+ with pytest.raises(UpstreamRequestError, match="signed"):
+ anthropic_request(
+ {"messages": [{"role": "assistant", "reasoning_content": "thought"}]}, "m"
+ )
+
+
+@pytest.mark.asyncio
+async def test_native_wire_uses_messages_and_native_headers():
+ def respond(request):
+ assert request.url.path == "/v1/messages"
+ assert request.headers["x-api-key"] == "test-secret"
+ assert request.headers["anthropic-version"] == "2023-06-01"
+ assert request.headers["anthropic-beta"] == "context-management-2025-06-27"
+ body = json.loads(request.content)
+ assert body["model"] == "pinned"
+ assert "return_token_ids" not in body and "logprobs" not in body
+ return httpx.Response(200, json=native_message())
+
+ client = InferenceClient(
+ "https://test/v1",
+ served_model="pinned",
+ api_key="test-secret",
+ provider="anthropic",
+ )
+ http_client = await client._get_client()
+ await http_client.aclose()
+ client._client = httpx.AsyncClient(
+ base_url="https://test",
+ transport=httpx.MockTransport(respond),
+ headers={"x-api-key": "test-secret", "anthropic-version": "2023-06-01"},
+ )
+ try:
+ response = await client.completion(
+ {
+ "messages": [{"role": "user", "content": "go"}],
+ "_openenv_native_headers": {
+ "anthropic-beta": "context-management-2025-06-27",
+ "x-api-key": "must-not-replace-real-key",
+ },
+ }
+ )
+ assert client.capture_level == "text"
+ assert response["_openenv_native_response"] == native_message()
+ finally:
+ await client.aclose()
+
+
+def test_sse_preserves_real_signature():
+ events = [
+ json.loads(frame.split("data: ", 1)[1])
+ for frame in replay_anthropic(native_message())
+ ]
+ signatures = [
+ event["delta"]["signature"]
+ for event in events
+ if event.get("delta", {}).get("type") == "signature_delta"
+ ]
+ assert signatures == ["signed-original"]
+ assert events[0]["type"] == "message_start" and events[-1]["type"] == "message_stop"
+
+
+def test_probe_native_tools_never_certifies_training(monkeypatch):
+ class Response:
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *args):
+ pass
+
+ def read(self):
+ payload = native_message()
+ payload["content"] = [
+ {
+ "type": "tool_use",
+ "id": "toolu_probe",
+ "name": "report_ok",
+ "input": {"value": "ok"},
+ }
+ ]
+ return json.dumps(payload).encode()
+
+ def urlopen(request, timeout):
+ assert request.full_url == "https://test/v1/messages"
+ assert request.get_header("X-api-key") == "key"
+ return Response()
+
+ monkeypatch.setattr("urllib.request.urlopen", urlopen)
+ report = validate_llm(
+ "https://test/v1", "claude-test", provider="anthropic", api_key="key"
+ )
+ assert report.reachable and report.tool_support == "ok"
+ assert report.capture_level == "text" and not report.trainable and not report.ok
+
+
+def test_logprob_permission_fix_removes_dependent_top_logprobs():
+ from openenv.core.harness.capture.compat import diagnose
+
+ body = {
+ "logprobs": True,
+ "top_logprobs": 0,
+ "messages": [{"role": "user", "content": "hi"}],
+ }
+ fix = diagnose(
+ {
+ "error": {
+ "message": "You are not allowed to request logprobs from this model"
+ }
+ }
+ )
+ assert fix.apply(body)
+ assert body == {"messages": [{"role": "user", "content": "hi"}]}
+ assert not fix.apply(body)
+
+
+@pytest.mark.parametrize(
+ "key,value",
+ [
+ ("frequency_penalty", 1),
+ ("presence_penalty", 1),
+ ("repetition_penalty", 1.1),
+ ("min_p", 0.1),
+ ],
+)
+def test_native_conversion_rejects_sampling_semantics_it_cannot_preserve(key, value):
+ with pytest.raises(UpstreamRequestError, match=key):
+ anthropic_request({"messages": [], key: value}, "model")
+
+
+def test_explicit_eval_keeps_capability_but_disables_supervision():
+ from openenv.core.harness.capture.export import export_session
+ from openenv.core.harness.capture.sessions import rollout_type_for, SessionRegistry
+
+ registry = SessionRegistry()
+ session = registry.create(capture_level="tokens", purpose="eval")
+ result = export_session(session, capture_level="tokens")
+ assert result["capture_level"] == "tokens" and result["rollout_type"] == "eval"
+ assert not result["trainable"]
+ assert rollout_type_for("auto", "tokens") == "train"
+ with pytest.raises(ValueError, match="exact engine"):
+ registry.create(capture_level="text", purpose="train")
+ with pytest.raises(ValueError, match="sampling"):
+ registry.create(
+ capture_level="tokens", purpose="eval", sampling={"temperature": 0.8}
+ )
+
+
+@pytest.mark.asyncio
+async def test_native_stream_is_consumable_by_anthropic_sdk():
+ anthropic = pytest.importorskip("anthropic")
+ message = native_message()
+
+ def respond(request):
+ return httpx.Response(
+ 200,
+ headers={"content-type": "text/event-stream"},
+ content="".join(replay_anthropic(message)).encode(),
+ )
+
+ async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http_client:
+ client = anthropic.AsyncAnthropic(api_key="test", http_client=http_client)
+ async with client.messages.stream(
+ model="claude-test",
+ max_tokens=64,
+ messages=[{"role": "user", "content": "go"}],
+ ) as stream:
+ reconstructed = await stream.get_final_message()
+ assert reconstructed.content[0].signature == "signed-original"
+ assert reconstructed.content[1].id == "toolu_1"
+ assert reconstructed.content[1].input == {"cmd": "pwd"}
+ assert reconstructed.stop_reason == "tool_use"
+
+
+def test_eval_sampling_requires_explicit_eval_and_preserves_requested_policy():
+ from openenv.core.harness.capture.sessions import SessionRegistry
+
+ registry = SessionRegistry()
+ policy = {"temperature": 0.8, "top_p": 1, "top_k": -1}
+ session = registry.create(
+ purpose="eval", capture_level="text", eval_sampling=policy
+ )
+ assert session.eval_sampling == policy and not session.sampling
+ with pytest.raises(ValueError, match="explicit eval"):
+ registry.create(purpose="auto", eval_sampling=policy)
+ native = anthropic_request(
+ {"_openenv_native_request": {"messages": [], **policy}}, "model"
+ )
+ assert native["temperature"] == 0.8
+ assert "top_p" not in native and "top_k" not in native
+
+
+def test_strict_tools_preserve_constraint_on_native_anthropic():
+ request = {
+ "messages": [],
+ "tools": [
+ {
+ "type": "function",
+ "function": {
+ "name": "run",
+ "strict": True,
+ "parameters": {
+ "type": "object",
+ "properties": {},
+ "additionalProperties": False,
+ },
+ },
+ }
+ ],
+ }
+ assert anthropic_request(request, "model")["tools"][0]["strict"] is True
+
+
+@pytest.mark.parametrize(
+ "block",
+ [
+ {"type": "redacted_thinking", "data": "opaque"},
+ {"type": "server_tool_use", "id": "srv", "name": "web_search", "input": {}},
+ {
+ "type": "text",
+ "text": "cited answer",
+ "citations": [{"type": "web_search_result_location"}],
+ },
+ ],
+)
+def test_response_semantics_are_not_silently_dropped_across_protocols(block):
+ from openenv.core.harness.capture.providers import ProviderConversionError
+
+ native = native_message()
+ native["content"].append(block)
+ with pytest.raises(ProviderConversionError, match="cannot be preserved"):
+ anthropic_response(native)
+ response = anthropic_response(native, native_passthrough=True)
+ assert response["_openenv_native_response"] == native
+ response["_openenv_native_response"]["content"].clear()
+ assert native["content"]
+
+
+def test_native_pause_is_not_translated_to_successful_stop():
+ from openenv.core.harness.capture.providers import ProviderConversionError
+
+ native = native_message()
+ native["stop_reason"] = "pause_turn"
+ with pytest.raises(ProviderConversionError, match="stop reason"):
+ anthropic_response(native)
+ assert (
+ anthropic_response(native, native_passthrough=True)["_openenv_native_response"][
+ "stop_reason"
+ ]
+ == "pause_turn"
+ )
diff --git a/tests/envs/test_harbor_nemo_profile.py b/tests/envs/test_harbor_nemo_profile.py
new file mode 100644
index 0000000000..107b727cf7
--- /dev/null
+++ b/tests/envs/test_harbor_nemo_profile.py
@@ -0,0 +1,80 @@
+"""The opt-in NeMo profile keeps endpoint routing and exercises real sandbox commands."""
+
+import importlib.util
+import json
+from pathlib import Path
+
+import pytest
+import yaml
+
+pytest.importorskip("harbor.agents.installed.nemo_agent")
+
+from openenv.harbor.nemo_profile import NemoShellProfile
+
+
+def test_nemo_react_profile_reuses_harbor_provider_configuration(tmp_path):
+ agent = NemoShellProfile(
+ logs_dir=tmp_path,
+ model_name="openai/Qwen3.5-4B",
+ llm_type="openai",
+ version="1.9.0",
+ extra_env={"OPENAI_BASE_URL": "https://capture.example/v1"},
+ )
+ config = yaml.safe_load(agent._generate_config_yaml("Qwen3.5-4B", "session-test"))
+ llm = config["llms"][config["workflow"]["llm_name"]]
+ assert llm["base_url"] == "https://capture.example/v1"
+ assert llm["api_key"] == "session-test"
+ assert llm["model_name"] == "Qwen3.5-4B"
+ assert config["workflow"]["use_native_tool_calling"] is True
+ assert config["workflow"]["tool_names"] == ["shell"]
+
+
+def shell_module():
+ path = (
+ Path(__file__).parents[2]
+ / "examples/harbor/nemo_shell_profile/src/openenv_nat_shell/shell.py"
+ )
+ spec = importlib.util.spec_from_file_location("qualification_shell", path)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+@pytest.mark.asyncio
+async def test_shell_executes_and_reports_failure_without_faking_success(tmp_path):
+ shell = shell_module()
+ result = json.loads(
+ await shell.execute(
+ "printf data > answer.txt; cat answer.txt; exit 7", cwd=str(tmp_path)
+ )
+ )
+ assert result["exit_code"] == 7
+ assert result["stdout"] == "data"
+ assert (tmp_path / "answer.txt").read_text() == "data"
+
+
+@pytest.mark.asyncio
+async def test_shell_timeout_terminates_command_group(tmp_path):
+ shell = shell_module()
+ with pytest.raises(TimeoutError):
+ await shell.execute(
+ "sleep 5; touch late-output", timeout=0.1, cwd=str(tmp_path)
+ )
+ assert not (tmp_path / "late-output").exists()
+
+
+def test_explicit_profile_uses_packaged_workflow_without_mutating_generic_seam():
+ from pathlib import Path
+
+ from openenv.harbor.seams import get
+
+ generic = get("nemo-agent")
+ selected = get("nemo-agent", profile="shell-1.9.0")
+ _, kwargs, _, _ = selected.resolve(
+ base_url="https://proxy.example", session="session", model="model"
+ )
+ assert selected.import_path == "openenv.harbor.nemo_profile:NemoShellProfile"
+ assert kwargs["version"] == "1.9.0"
+ assert (Path(kwargs["workflow_package"]) / "pyproject.toml").is_file()
+ assert get("nemo-agent") is generic
+ assert generic.import_path != selected.import_path
diff --git a/tests/envs/test_harbor_per_session_engine.py b/tests/envs/test_harbor_per_session_engine.py
new file mode 100644
index 0000000000..9b1155e7cb
--- /dev/null
+++ b/tests/envs/test_harbor_per_session_engine.py
@@ -0,0 +1,194 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""The engine is a per-rollout property, not a per-server one.
+
+A dataset server is the expensive thing to keep alive: thousands of task files, prebuilt sandbox
+templates. An inference engine is the cheap, changing part — it restarts every training run, and a
+train-tier engine and an eval-tier one are usually both wanted against the same task suite. Pinning
+the engine at boot made the durable thing hostage to the ephemeral one: no URL meant no capture proxy
+at all, and every rollout answered "server not initialised".
+
+So a caller names its engine when it mints a session, that engine is probed THEN (so the caller learns
+its tier at submit time rather than when the token fields come back empty), and the measurement is
+cached per engine so a whole GRPO group naming one vLLM pays for it once.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+pytest.importorskip("fastapi")
+from fastapi.testclient import TestClient # noqa: E402
+
+server = pytest.importorskip("openenv.core.harness.capture.server")
+sessions = pytest.importorskip("openenv.core.harness.capture.sessions")
+
+
+def app_with(monkeypatch, *, llm_url="", probe=None):
+ """A capture app whose probe is stubbed, so no engine is contacted."""
+ calls: list[str] = []
+
+ def fake_probe(self, upstream):
+ calls.append(upstream.llm_url)
+ return (
+ upstream.model or "stub-model",
+ (probe or {}).get(upstream.llm_url, "tokens"),
+ )
+
+ monkeypatch.setattr(server.UpstreamPool, "_probe", fake_probe, raising=True)
+ app = server.create_app(llm_url=llm_url, model="boot-model" if llm_url else None)
+ app.state.admin_key = None # these tests are about engines, not auth
+ return app, calls
+
+
+def test_a_server_boots_with_no_engine_at_all(monkeypatch):
+ """The regression: this used to be unusable rather than merely engineless."""
+ app, _ = app_with(monkeypatch)
+ with TestClient(app) as client:
+ assert client.get("/health").status_code == 200
+ # Nothing to list, and an empty list is the honest answer rather than a 500.
+ assert client.get("/v1/models").json() == {"object": "list", "data": []}
+
+
+def test_naming_an_engine_probes_it_and_returns_the_tier(monkeypatch):
+ app, calls = app_with(monkeypatch, probe={"http://train:8000": "tokens"})
+ with TestClient(app) as client:
+ body = client.post("/sessions", json={"llm_url": "http://train:8000"}).json()
+ assert body["capture_level"] == "tokens"
+ assert body["rollout_type"] == "train", "token-capable engine must be trainable"
+ assert calls == ["http://train:8000"], "the engine should be probed exactly once"
+
+
+def test_a_weaker_engine_comes_back_as_eval(monkeypatch):
+ """Same server, same session route, different engine — the tier follows the engine."""
+ app, _ = app_with(monkeypatch, probe={"http://evalonly:8000": "text"})
+ with TestClient(app) as client:
+ body = client.post("/sessions", json={"llm_url": "http://evalonly:8000"}).json()
+ assert body["capture_level"] == "text"
+ assert body["rollout_type"] == "eval"
+
+
+def test_two_engines_coexist_on_one_server(monkeypatch):
+ """The whole point: a trainer and an eval run share a server and get different tiers."""
+ app, _ = app_with(
+ monkeypatch, probe={"http://train:8000": "tokens", "http://eval:8000": "text"}
+ )
+ with TestClient(app) as client:
+ train = client.post("/sessions", json={"llm_url": "http://train:8000"}).json()
+ evaluate = client.post("/sessions", json={"llm_url": "http://eval:8000"}).json()
+ assert (train["rollout_type"], evaluate["rollout_type"]) == ("train", "eval")
+ assert train["session_id"] != evaluate["session_id"]
+
+
+def test_the_probe_is_cached_per_engine(monkeypatch):
+ """A GRPO group is N sessions on ONE engine; probing N times would add round trips per rollout."""
+ app, calls = app_with(monkeypatch, probe={"http://train:8000": "tokens"})
+ with TestClient(app) as client:
+ for _ in range(5):
+ client.post("/sessions", json={"llm_url": "http://train:8000"})
+ assert calls == ["http://train:8000"], f"probed {len(calls)} times, expected 1"
+
+
+def test_a_session_with_no_engine_and_no_default_is_told_so(monkeypatch):
+ """Better than forwarding to an empty base URL, which reads as a connection fault."""
+ app, _ = app_with(monkeypatch)
+ with TestClient(app) as client:
+ sid = client.post("/sessions", json={}).json()["session_id"]
+ response = client.post(
+ "/v1/chat/completions",
+ json={"model": "m", "messages": [{"role": "user", "content": "hi"}]},
+ headers={"Authorization": f"Bearer {sid}"},
+ )
+ assert response.status_code == 503
+ assert "no inference engine" in response.json()["error"]["message"]
+
+
+def test_booting_with_an_engine_still_works(monkeypatch):
+ """Backwards compatibility: the boot engine is the default for sessions that name none."""
+ app, calls = app_with(monkeypatch, llm_url="http://default:8000")
+ with TestClient(app) as client:
+ body = client.post("/sessions", json={}).json()
+ assert body["llm_url"] == "http://default:8000"
+ assert body["capture_level"] == "tokens" # create_app's default level
+ assert calls == [], "naming no engine must not trigger a probe"
+
+
+def test_health_lists_the_engines_it_has_measured(monkeypatch):
+ app, _ = app_with(monkeypatch, probe={"http://train:8000": "tokens"})
+ with TestClient(app) as client:
+ client.post("/sessions", json={"llm_url": "http://train:8000"})
+ upstreams = client.get("/health").json()["upstreams"]
+ assert upstreams == [
+ {
+ "llm_url": "http://train:8000",
+ "model": "stub-model",
+ "capture_level": "tokens",
+ }
+ ]
+
+
+def test_an_unprobeable_engine_is_the_weakest_tier_not_a_crash():
+ """`text` is the floor because claiming `tokens` without evidence is how an eval rollout gets
+ stamped trainable — the one failure the capture level exists to prevent."""
+ pool = server.UpstreamPool(default_client=None, default_level="tokens")
+ model, level = pool._probe(
+ sessions.Upstream(llm_url="http://nope.invalid:9/v1", model="m")
+ )
+ assert level == "text"
+ assert model == "m"
+
+
+def test_credentials_isolate_clients_without_exposing_secrets():
+ """Credentials may select different tenants, quotas, and capabilities at the same URL."""
+ a = sessions.Upstream(llm_url="http://x/v1", model="m", api_key="secret-a")
+ b = sessions.Upstream(llm_url="http://x/v1", model="m", api_key="secret-b")
+ assert a.cache_key != b.cache_key
+ assert "secret-a" not in str(a.cache_key)
+ assert "secret-a" not in repr(a)
+
+
+def test_the_outgoing_model_comes_from_the_session_engine(monkeypatch):
+ """The engine 404s on a mangled model name, and an engineless server used to send one.
+
+ Harnesses rewrite the model: opencode is configured with `intercepted/` and its provider
+ layer forwards only the last path segment, so `Qwen/Qwen3.5-2B` arrives as `Qwen3.5-2B` and the
+ engine answers `404 The model does not exist`. The proxy rewriting `model` is what makes the call
+ work. Reading the SERVER's model to do it meant an engineless server skipped the rewrite entirely
+ and every agent call 404'd — captured live before this test existed.
+ """
+ app, _ = app_with(monkeypatch)
+ sent: dict = {}
+
+ class FakeClient:
+ served_model = "Qwen/Qwen3.5-2B"
+
+ async def completion(self, request):
+ sent.update(request)
+ raise server.UpstreamError("stop here; the request is what matters")
+
+ with TestClient(app) as client:
+ body = client.post(
+ "/sessions",
+ json={"llm_url": "http://train:8000", "model": "Qwen/Qwen3.5-2B"},
+ ).json()
+ # Swap in a client that records what it was asked to send.
+ key = sessions.Upstream(
+ llm_url="http://train:8000", model="Qwen/Qwen3.5-2B"
+ ).cache_key
+ app.state.upstreams._by_engine[key] = (FakeClient(), "tokens")
+ client.post(
+ "/v1/chat/completions",
+ json={
+ "model": "Qwen3.5-2B",
+ "messages": [{"role": "user", "content": "hi"}],
+ },
+ headers={"Authorization": f"Bearer {body['session_id']}"},
+ )
+
+ assert sent.get("model") == "Qwen/Qwen3.5-2B", (
+ f"the mangled name reached the engine: {sent.get('model')!r}"
+ )
diff --git a/tests/envs/test_harbor_proc_env_context.py b/tests/envs/test_harbor_proc_env_context.py
new file mode 100644
index 0000000000..cad0ae17dc
--- /dev/null
+++ b/tests/envs/test_harbor_proc_env_context.py
@@ -0,0 +1,113 @@
+"""Concurrent rollouts of a credential-by-env harness must see DIFFERENT keys from `os.environ`.
+
+claude-code, gemini-cli and goose read `os.environ` inside `run()` to build the env dict they pass to
+the sandbox, and the API key IS the rollout's session id — so N concurrent rollouts need N different
+values of one variable at one instant. That is what forced `_PROC_ENV_LOCK` and made those three
+harnesses serialise.
+
+These tests assert the property that replaces the lock: an overlay is visible to the task that set it
+and invisible to every other, including while they interleave.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+
+import pytest
+from openenv.harbor import proc_env_context as ctx
+
+
+@pytest.fixture(autouse=True)
+def _restore():
+ """Uninstall the proxy between tests: it replaces a process-global."""
+ real = os.environ
+ ctx._installed = False
+ yield
+ os.environ = real
+ ctx._installed = False
+
+
+def test_the_overlay_is_visible_to_reads():
+ ctx.install()
+ with ctx.overlay({"OPENAI_API_KEY": "session-abc"}):
+ assert os.environ.get("OPENAI_API_KEY") == "session-abc"
+ assert os.environ["OPENAI_API_KEY"] == "session-abc"
+ assert "OPENAI_API_KEY" in os.environ
+
+
+def test_it_does_not_leak_after_the_block():
+ ctx.install()
+ with ctx.overlay({"OPENENV_TEST_ONLY": "x"}):
+ pass
+ assert os.environ.get("OPENENV_TEST_ONLY") is None
+
+
+def test_concurrent_tasks_see_their_own_key():
+ """The property the lock used to provide, now without serialising."""
+ ctx.install()
+ observed: dict[str, str | None] = {}
+
+ async def rollout(name: str, key: str):
+ with ctx.overlay({"OPENAI_API_KEY": key}):
+ # Yield control repeatedly so the tasks genuinely interleave inside their overlays;
+ # a process-global would be clobbered by whichever task ran last.
+ for _ in range(5):
+ await asyncio.sleep(0)
+ assert os.environ.get("OPENAI_API_KEY") == key
+ observed[name] = os.environ.get("OPENAI_API_KEY")
+
+ async def main():
+ await asyncio.gather(*(rollout(f"r{i}", f"session-{i}") for i in range(8)))
+
+ asyncio.run(main())
+ assert observed == {f"r{i}": f"session-{i}" for i in range(8)}
+
+
+def test_copy_is_merged_because_subprocess_uses_it():
+ """`subprocess` builds a child's env from `os.environ`; hiding the overlay would launch it
+ without credentials, and that failure would look like a bad key rather than a bad proxy."""
+ ctx.install()
+ with ctx.overlay({"OPENENV_OVERLAY_ONLY": "yes"}):
+ assert os.environ.copy().get("OPENENV_OVERLAY_ONLY") == "yes"
+ assert "OPENENV_OVERLAY_ONLY" in dict(os.environ)
+ assert "OPENENV_OVERLAY_ONLY" in list(os.environ)
+
+
+def test_the_real_environment_still_shows_through():
+ ctx.install()
+ os.environ["OPENENV_REAL"] = "base"
+ try:
+ with ctx.overlay({"OTHER": "1"}):
+ assert os.environ.get("OPENENV_REAL") == "base"
+ finally:
+ del os.environ["OPENENV_REAL"]
+
+
+def test_writes_reach_the_real_environment():
+ """Only reads are context-local; a write that vanished would break unrelated libraries."""
+ ctx.install()
+ with ctx.overlay({"A": "1"}):
+ os.environ["OPENENV_WRITTEN"] = "persisted"
+ assert os.environ.get("OPENENV_WRITTEN") == "persisted"
+ del os.environ["OPENENV_WRITTEN"]
+
+
+def test_it_can_be_switched_off():
+ """This swaps a global the whole process reads, so it must be disableable without a rollback."""
+ os.environ["OPENENV_CONCURRENT_PROC_ENV"] = "0"
+ try:
+ assert ctx.enabled() is False
+ assert ctx.install() is False
+ finally:
+ del os.environ["OPENENV_CONCURRENT_PROC_ENV"]
+
+
+def test_an_overlay_nests():
+ """A rollout inside a rollout's context must not lose the outer values."""
+ ctx.install()
+ with ctx.overlay({"OUTER": "1"}):
+ with ctx.overlay({"INNER": "2"}):
+ assert os.environ.get("OUTER") == "1"
+ assert os.environ.get("INNER") == "2"
+ assert os.environ.get("INNER") is None
diff --git a/tests/envs/test_harbor_qualification.py b/tests/envs/test_harbor_qualification.py
new file mode 100644
index 0000000000..8403de8a4b
--- /dev/null
+++ b/tests/envs/test_harbor_qualification.py
@@ -0,0 +1,117 @@
+import pytest
+from openenv.harbor.models import HarborRolloutResult
+from openenv.harbor.qualification import evaluate_eval_capture, qualification_rows
+from openenv.harbor.seams import SEAMS
+
+
+def test_all_current_adapters_start_unqualified_for_every_provider():
+ rows = qualification_rows(list(SEAMS))
+ assert len(rows) == 29
+ assert all(row[1:] == ["not_run"] * 4 for row in rows)
+
+
+def test_eval_export_rejection_is_expected_and_zero_reward_is_valid():
+ result = HarborRolloutResult(
+ rollout_type="eval", capture_level="text", n_turns=2, reward=0.0
+ )
+ assert all(evaluate_eval_capture(result).values())
+ result.reward = None
+ assert not evaluate_eval_capture(result)["verifier_graded"]
+
+
+def test_no_capture_cannot_be_certified_as_success():
+ result = HarborRolloutResult(rollout_type="eval", capture_level="text", reward=0.0)
+ assert not evaluate_eval_capture(result)["model_calls_captured"]
+
+
+def test_pass_requires_evidence_and_is_not_promoted_to_optimizer():
+ cell = {
+ "harness": "opencode",
+ "provider": "vllm",
+ "status": "capture_and_reader_pass",
+ }
+ with pytest.raises(ValueError, match="evidence"):
+ qualification_rows(["opencode"], {"cells": [cell]})
+ cell["evidence"] = ["two-task-run.json"]
+ assert (
+ qualification_rows(["opencode"], {"cells": [cell]})[0][-1]
+ == "capture_and_reader_pass"
+ )
+ with pytest.raises(ValueError, match="duplicate"):
+ qualification_rows(["opencode"], {"cells": [cell, cell]})
+
+
+def test_optimizer_status_requires_proof_for_current_captures():
+ from openenv.harbor.qualification import qualification_details
+
+ cell = {
+ "harness": "opencode",
+ "provider": "vllm",
+ "status": "optimizer_pass",
+ "evidence": ["capture.json"],
+ "optimizer_validated": True,
+ }
+ with pytest.raises(ValueError, match="matching, scoped"):
+ qualification_rows(["opencode"], {"cells": [cell]})
+ proof = {
+ "matches_current_captures": True,
+ "result": "result.json",
+ "inputs": "inputs.json",
+ "scope": "diagnostic replay; no weight sync",
+ "model": "test-model",
+ "revision": "pinned",
+ "rows": 2,
+ }
+ cell["optimizer_evidence"] = proof
+ assert (
+ qualification_rows(["opencode"], {"cells": [cell]})[0][-1] == "optimizer_pass"
+ )
+ assert (
+ "diagnostic replay; no weight sync"
+ in qualification_details({"cells": [cell]})[0][7]
+ )
+ proof["matches_current_captures"] = False
+ with pytest.raises(ValueError, match="matching, scoped"):
+ qualification_rows(["opencode"], {"cells": [cell]})
+ cell["status"] = "capture_and_reader_pass"
+ cell["optimizer_validated"] = False
+ assert qualification_details({"cells": [cell]})[0][7].startswith(
+ "previous captures only:"
+ )
+
+
+def test_unrecognized_provider_is_not_silently_hidden():
+ with pytest.raises(ValueError, match="provider"):
+ qualification_rows([], {"cells": [{"harness": "opencode", "provider": "typo"}]})
+
+
+def test_maturity_requires_all_providers_and_current_optimizer():
+ from openenv.harbor.qualification import harness_maturity_rows, PROVIDERS
+
+ cells = [
+ {"harness": "example", "provider": provider, "status": "failed"}
+ for provider in PROVIDERS
+ ]
+ report = {"cells": cells}
+ assert harness_maturity_rows(["example"], report)[0][1] == "unstable"
+ assert harness_maturity_rows(["unmeasured"], report)[0][1] == "experimental"
+ for cell in cells:
+ cell.update(status="eval_pass", evidence=["capture.json"])
+ cells[-1]["status"] = "capture_and_reader_pass"
+ assert harness_maturity_rows(["example"], report)[0][1] == "experimental"
+ cells[-1].update(
+ status="optimizer_pass",
+ optimizer_validated=True,
+ optimizer_evidence={
+ "matches_current_captures": True,
+ "result": "result.json",
+ "inputs": "inputs.json",
+ "scope": "diagnostic",
+ "model": "model",
+ "revision": "pinned",
+ "rows": 2,
+ },
+ )
+ assert harness_maturity_rows(["example"], report)[0][1] == "stable"
+ cells[1]["status"] = "failed"
+ assert harness_maturity_rows(["example"], report)[0][1] == "experimental"
diff --git a/tests/envs/test_harbor_reconcile.py b/tests/envs/test_harbor_reconcile.py
new file mode 100644
index 0000000000..03b9cec294
--- /dev/null
+++ b/tests/envs/test_harbor_reconcile.py
@@ -0,0 +1,301 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""`reconcile`: the only check on this path that is not self-referential.
+
+It compares the capture against ATIF, the trace the harness writes independently, and its verdict is
+what gates trainability. Until now only its helpers were tested — `load_trace` and
+`atif_turn_lengths` — so every decision that actually decides whether a rollout may be trained on was
+uncovered: the turn-count FATAL, the coverage floor, the auxiliary-subsequence inference, and the
+subagent refusal.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+atif = pytest.importorskip("openenv.harbor.atif")
+
+reconcile = atif.reconcile
+
+
+graph_mod = pytest.importorskip("openenv.core.harness.capture.graph")
+export_mod = pytest.importorskip("openenv.core.harness.capture.export")
+
+
+def document(lengths, *, role="agent", rollout_type="train"):
+ """A real capture document with one sampled span per entry in `lengths`.
+
+ Built by `export_session` over a real graph rather than hand-rolled, so the fixture cannot drift
+ from the document contract `reconcile` reads — three separate KeyErrors while writing these tests
+ were a hand-written dict missing fields the producer always sets.
+ """
+ graph = graph_mod.RolloutGraph()
+ prompt = [1]
+ for i, n_sampled in enumerate(lengths):
+ sampled = list(range(1000 + i * 100, 1000 + i * 100 + n_sampled))
+ graph.add_turn(
+ graph_mod.TurnNode(
+ node_id=f"n{i}",
+ prompt_ids=list(prompt),
+ sampled_ids=sampled,
+ sampled_logprobs=[-0.1] * n_sampled,
+ n_tools=1,
+ finish_reason="stop",
+ )
+ )
+ prompt = prompt + sampled + [9000 + i]
+
+ class Session:
+ session_id = "s"
+ metadata: dict = {}
+ findings: list = []
+
+ session = Session()
+ session.graph = graph
+ doc = export_mod.export_session(
+ session, capture_level="tokens" if rollout_type == "train" else "text"
+ )
+ if role != "agent":
+ # Same mutation rollout.py's auxiliary demotion performs.
+ for row in doc["sequences"]:
+ row["role"] = role
+ return doc
+
+
+def trace(lengths, **extra):
+ return {
+ "schema_version": "1.0",
+ "agent": {"name": "opencode"},
+ "steps": [
+ {"source": "agent", "metrics": {"completion_tokens": n}} for n in lengths
+ ],
+ **extra,
+ }
+
+
+def codes(report):
+ return {f.code for f in report.findings}
+
+
+def fatal_codes(report):
+ return {f.code for f in report.fatal}
+
+
+# --- the agreeing case ------------------------------------------------------
+def test_identical_turn_lengths_reconcile():
+ report = reconcile(document([10, 20, 30]), trace([10, 20, 30]))
+ assert report.ok
+ assert not fatal_codes(report)
+
+
+def test_explicit_synthetic_api_error_is_not_a_model_response():
+ native = trace([10, 20, 0])
+ native["steps"][-1].update(
+ model_name="", message="API Error: 400 unsupported image"
+ )
+ native["steps"][-1]["metrics"].update(prompt_tokens=0, cached_tokens=0)
+ report = reconcile(document([10, 20]), native)
+ assert report.ok
+ assert "atif_synthetic_api_error" in codes(report)
+ assert len(native["steps"]) == 3 # Preserve the native evidence.
+
+
+@pytest.mark.parametrize(
+ "change",
+ [
+ {"model_name": "Qwen3.5-2B"},
+ {"message": "An ordinary empty reply"},
+ {"tool_calls": [{"name": "read"}]},
+ {"metrics": {"prompt_tokens": 2, "completion_tokens": 0}},
+ {"metrics": {"prompt_tokens": 0, "completion_tokens": 1}},
+ ],
+)
+def test_unproven_or_sampled_zero_token_steps_still_fail(change):
+ native = trace([10, 20, 0])
+ native["steps"][-1].update(
+ model_name="",
+ message="API Error: 400 unsupported image",
+ metrics={"prompt_tokens": 0, "completion_tokens": 0},
+ )
+ native["steps"][-1].update(change)
+ assert "turn_mismatch" in fatal_codes(reconcile(document([10, 20]), native))
+
+
+def test_no_atif_is_not_a_failure():
+ """Three of sixteen harnesses emit no trajectory; that is an absent cross-check, not a fault."""
+ report = reconcile(document([10]), None)
+ assert report.ok
+ assert "no_atif" in codes(report)
+
+
+# --- disagreement -----------------------------------------------------------
+def test_atif_logging_more_calls_than_were_captured_is_fatal():
+ """Calls the harness made that never reached the proxy mean the capture is incomplete."""
+ report = reconcile(document([10, 20]), trace([10, 20, 30]))
+ assert not report.ok
+
+
+@pytest.mark.parametrize("recorded_stops", [0, 1])
+def test_proxy_stop_requires_recorded_provenance_and_leaves_trace_unchanged(
+ recorded_stops,
+):
+ doc = document([10, 20])
+ doc["budget_stop_count"] = recorded_stops
+ native = trace([10, 20, 0])
+ native["steps"][-1]["message"] = atif.BUDGET_STOP_MESSAGE
+ report = reconcile(doc, native)
+ assert report.ok == bool(recorded_stops)
+ assert len(native["steps"]) == 3
+
+
+@pytest.mark.parametrize(
+ "message,count",
+ [("unexpected missing generation", 0), (atif.BUDGET_STOP_MESSAGE, 7)],
+)
+def test_stop_allowance_cannot_hide_unknown_or_sampled_model_turns(message, count):
+ doc = document([10])
+ doc["budget_stop_count"] = 1
+ native = trace([10, count])
+ native["steps"][-1]["message"] = message
+ assert not reconcile(doc, native).ok
+
+
+def test_stop_allowance_is_bounded_by_responses_actually_emitted():
+ doc = document([10])
+ doc["budget_stop_count"] = 1
+ native = trace([10, 0, 0])
+ for step in native["steps"][1:]:
+ step["message"] = atif.BUDGET_STOP_MESSAGE
+ assert not reconcile(doc, native).ok
+
+
+def test_extra_captured_calls_embed_as_auxiliary_when_coverage_is_high():
+ """Seeing MORE than the harness logged is benign and explainable: a next-speaker check, a title
+ generator. The asymmetry is deliberate — missing calls mean we lost something, extra ones do not."""
+ report = reconcile(document([58, 132, 266, 370, 33]), trace([58, 132, 266, 370]))
+ assert report.ok
+ assert "atif_aux_calls" in codes(report)
+ assert report.aux_node_ids, "the aux call must be identified so it can be demoted"
+
+
+def test_low_coverage_refuses_rather_than_discarding_most_of_a_rollout():
+ """The mimo case: 49 captured calls, ATIF logged 5, and a subsequence match would have demoted 44
+ to auxiliary under a warning. Below the floor that inference is as likely coincidence as signal."""
+ captured = list(range(1, 50))
+ report = reconcile(document(captured), trace(captured[:5]))
+ assert not report.ok
+ assert "atif_coverage_too_low" in fatal_codes(report)
+
+
+# --- converters that give nothing to compare against ------------------------
+def test_all_zero_token_counts_downgrade_to_no_cross_check():
+ """vibe reports completion_tokens=0 on every step while capturing perfectly. Failing the rollout
+ would punish it for its trace converter rather than for anything wrong."""
+ report = reconcile(document([10, 20]), trace([0, 0]))
+ assert report.ok
+ assert "atif_no_token_counts" in codes(report)
+
+
+def test_no_agent_steps_at_all_downgrades_the_same_way():
+ report = reconcile(document([10, 20]), trace([]))
+ assert report.ok
+ assert "atif_no_token_counts" in codes(report)
+
+
+# --- structural edge cases --------------------------------------------------
+def test_nothing_captured_defers_to_the_rollouts_own_finding():
+ """`check_rollout` already reports no_turns plainly; a second FATAL here buries the real cause."""
+ empty: dict = {
+ "rollout_type": "train",
+ "sequences": [],
+ "turns": [],
+ "stats": {"n_turns": 0, "n_roots": 0, "n_discarded": 0},
+ }
+ report = reconcile(empty, trace([10]))
+ assert "no_turns_upstream" in codes(report)
+ assert not fatal_codes(report)
+
+
+def test_calls_captured_but_none_labelled_agent_is_fatal():
+ report = reconcile(document([10, 20], role="auxiliary"), trace([10, 20]))
+ assert not report.ok
+ assert "no_agent_sequence" in fatal_codes(report)
+
+
+def test_subagent_trajectories_are_refused_not_merely_noted():
+ """The warning used to say subagent turns must not carry the parent's reward and then did nothing
+ to stop it: no node ids collected, no role changed. ATIF does not say which captured calls belong
+ to the subagent, so the rollout cannot be attributed and is refused."""
+ report = reconcile(
+ document([10, 20]),
+ trace([10, 20], subagent_trajectories=[{"agent": {"name": "sub"}}]),
+ )
+ assert not report.ok
+ assert "atif_subagents" in fatal_codes(report)
+
+
+# --- the eval path ----------------------------------------------------------
+def test_eval_rollouts_compare_call_counts_since_token_counts_do_not_exist():
+ report = reconcile(document([0, 0, 0], rollout_type="eval"), trace([10, 20, 30]))
+ assert report.ok
+ assert "eval_reconcile_counts_only" in codes(report)
+
+
+def test_eval_rollouts_still_notice_a_truncated_harness_trace():
+ """The one real bug reconciliation has ever caught was a truncated trajectory. Counts alone are
+ enough to see it, which is why the eval path bothers comparing at all."""
+ report = reconcile(document([0] * 2, rollout_type="eval"), trace([1] * 6))
+ assert "atif_calls_missing" in codes(report)
+
+
+def _partial_usage_pair():
+ doc = document([64, 199, 10])
+ native = trace([None, None, 10])
+ for index in range(2):
+ call_id = f"model-call-{index}"
+ doc["turns"][index]["response_message"] = {"tool_calls": [{"id": call_id}]}
+ native["steps"][index]["tool_calls"] = [{"tool_call_id": call_id}]
+ return doc, native
+
+
+def test_partial_usage_requires_exact_ordered_call_identity():
+ doc, native = _partial_usage_pair()
+ report = reconcile(doc, native)
+ assert report.ok
+ assert "atif_partial_usage" in codes(report)
+ assert "turns_match" not in codes(report)
+ assert not report.aux_node_ids
+
+
+@pytest.mark.parametrize(
+ "failure",
+ [
+ "different_id",
+ "missing_id",
+ "duplicate_id",
+ "known_count",
+ "explicit_zero",
+ "extra_step",
+ ],
+)
+def test_partial_usage_cannot_hide_disagreement(failure):
+ doc, native = _partial_usage_pair()
+ if failure == "different_id":
+ native["steps"][0]["tool_calls"][0]["tool_call_id"] = "different"
+ elif failure == "missing_id":
+ doc["turns"][0]["response_message"] = {}
+ elif failure == "duplicate_id":
+ for index in range(2):
+ doc["turns"][index]["response_message"]["tool_calls"][0]["id"] = "duplicate"
+ native["steps"][index]["tool_calls"][0]["tool_call_id"] = "duplicate"
+ elif failure == "known_count":
+ native["steps"][-1]["metrics"]["completion_tokens"] = 11
+ elif failure == "explicit_zero":
+ native["steps"][0]["metrics"]["completion_tokens"] = 0
+ else:
+ native["steps"].append({"source": "agent", "metrics": {"completion_tokens": 5}})
+ assert not reconcile(doc, native).ok
diff --git a/tests/envs/test_harbor_result_rendering.py b/tests/envs/test_harbor_result_rendering.py
new file mode 100644
index 0000000000..28b1428e7a
--- /dev/null
+++ b/tests/envs/test_harbor_result_rendering.py
@@ -0,0 +1,330 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""Turning a capture document into a result, and a result into something readable.
+
+Two dialect families put tool calls in different places (`tool_calls` vs `tool_use` content blocks),
+so a reader that knows only one shows an agent as a stream of text with no visible actions. And a
+forked conversation appears once per path, which rendered as several near-identical transcripts all
+claiming to be the main one.
+"""
+
+from __future__ import annotations
+
+import json
+
+import pytest
+
+models = pytest.importorskip("openenv.harbor.models")
+ui = pytest.importorskip("openenv.harbor.ui")
+
+conversations_from_document = models.conversations_from_document
+turns_from_document = models.turns_from_document
+
+
+def document(sequences, turns):
+ return {"sequences": sequences, "turns": turns}
+
+
+# --- conversations ----------------------------------------------------------
+def test_a_forked_root_yields_one_conversation_not_one_per_path():
+ """The regression: two paths through one root rendered as two 'main conversation' blocks."""
+ doc = document(
+ sequences=[
+ {"root_id": "r1", "role": "agent", "node_ids": ["a"], "n_turns": 1},
+ {"root_id": "r1", "role": "agent", "node_ids": ["a", "b"], "n_turns": 2},
+ ],
+ turns=[
+ {
+ "node_id": "a",
+ "request_messages": [{"role": "user", "content": "hi"}],
+ "response_message": {"content": "one"},
+ },
+ {
+ "node_id": "b",
+ "request_messages": [
+ {"role": "user", "content": "hi"},
+ {"role": "assistant", "content": "one"},
+ ],
+ "response_message": {"content": "two"},
+ },
+ ],
+ )
+ convos = conversations_from_document(doc)
+ assert len(convos) == 1
+ assert convos[0].n_turns == 2, "the longest path is the complete one"
+
+
+def test_separate_roots_stay_separate():
+ doc = document(
+ sequences=[
+ {"root_id": "r1", "role": "agent", "node_ids": ["a"], "n_turns": 1},
+ {"root_id": "r2", "role": "auxiliary", "node_ids": ["b"], "n_turns": 1},
+ ],
+ turns=[
+ {
+ "node_id": "a",
+ "request_messages": [{"role": "user", "content": "task"}],
+ "response_message": {"content": "working"},
+ },
+ {
+ "node_id": "b",
+ "request_messages": [{"role": "user", "content": "who next?"}],
+ "response_message": {"content": "agent"},
+ },
+ ],
+ )
+ convos = conversations_from_document(doc)
+ assert {c.role for c in convos} == {"agent", "auxiliary"}
+
+
+def test_a_conversation_keeps_the_system_prompt_and_tool_results():
+ doc = document(
+ sequences=[{"root_id": "r", "role": "agent", "node_ids": ["a"], "n_turns": 1}],
+ turns=[
+ {
+ "node_id": "a",
+ "request_messages": [
+ {"role": "system", "content": "You are an assistant."},
+ {"role": "user", "content": "count rows"},
+ {"role": "tool", "content": "42"},
+ ],
+ "response_message": {"content": "42 rows"},
+ }
+ ],
+ )
+ roles = [m["role"] for m in conversations_from_document(doc)[0].messages]
+ assert roles == ["system", "user", "tool", "assistant"]
+
+
+def test_sequences_without_nodes_or_messages_are_skipped():
+ doc = document(
+ sequences=[{"root_id": "r", "role": "agent", "node_ids": [], "n_turns": 0}],
+ turns=[],
+ )
+ assert conversations_from_document(doc) == []
+
+
+# --- turns ------------------------------------------------------------------
+def _agent_doc(response):
+ return document(
+ sequences=[
+ {
+ "root_id": "r",
+ "role": "agent",
+ "node_ids": ["a"],
+ "n_turns": 1,
+ "input_ids": [1, 2, 3],
+ "loss_mask": [0, 1, 1],
+ "logprobs": [0.0, -0.1, -0.2],
+ "prompt_len": 1,
+ "turn_lengths": [2],
+ }
+ ],
+ turns=[
+ {
+ "node_id": "a",
+ "finish_reason": "stop",
+ "n_tools": 3,
+ "response_message": response,
+ }
+ ],
+ )
+
+
+def test_turn_text_and_tool_calls_from_chat_completions():
+ turns = turns_from_document(
+ _agent_doc(
+ {
+ "content": "Reading the file.",
+ "tool_calls": [
+ {"function": {"name": "bash", "arguments": '{"cmd":"ls"}'}}
+ ],
+ }
+ )
+ )
+ assert turns[0].text == "Reading the file."
+ assert turns[0].tool_calls == [{"name": "bash", "arguments": '{"cmd":"ls"}'}]
+
+
+def test_turn_text_and_tool_calls_from_anthropic_blocks():
+ """claude-code puts tool use in content blocks; reading only `tool_calls` shows no actions."""
+ turns = turns_from_document(
+ _agent_doc(
+ {
+ "content": [
+ {"type": "text", "text": "Checking."},
+ {"type": "tool_use", "name": "Bash", "input": {"command": "ls"}},
+ ],
+ }
+ )
+ )
+ assert turns[0].text == "Checking."
+ assert turns[0].tool_calls[0]["name"] == "Bash"
+
+
+def test_a_turn_with_no_response_is_still_a_turn():
+ turns = turns_from_document(_agent_doc({}))
+ assert len(turns) == 1 and turns[0].text == "" and turns[0].tool_calls == []
+
+
+def test_only_agent_sequences_become_turns():
+ """An auxiliary call must never be credited with the reward for solving the task."""
+ doc = _agent_doc({"content": "x"})
+ doc["sequences"][0]["role"] = "auxiliary"
+ assert turns_from_document(doc) == []
+
+
+# --- rendering --------------------------------------------------------------
+@pytest.mark.parametrize(
+ "result,marker",
+ [
+ ({"ok": True, "reward": 1.0}, "Solved"),
+ ({"ok": True, "reward": 0.0}, "Not solved"),
+ ({"ok": True, "reward": None}, "Not graded"),
+ ({"ok": False, "reward": None, "exception_type": "Boom"}, "Failed"),
+ ],
+)
+def test_every_verdict_state_renders(result, marker):
+ assert marker in ui._result_html({**result, "turns": []})
+
+
+def test_ungraded_shows_a_dash_rather_than_a_zero():
+ """A dead sandbox rendered as 0.00 reads as the model getting the answer wrong."""
+ out = ui._result_html({"ok": True, "reward": None, "turns": []})
+ assert "0.00" not in out
+
+
+def test_findings_are_grouped_by_severity():
+ out = ui._findings_html(["[FATAL] gone", "[WARN] odd", "[INFO] fyi"])
+ assert "FATAL" in out and "WARN" in out and "INFO" in out
+ assert out.index("FATAL") < out.index("WARN"), "worst first"
+
+
+def test_no_findings_renders_nothing():
+ assert ui._findings_html([]) == ""
+
+
+def test_conversation_labels_are_unambiguous_when_several_exist():
+ convos = [
+ {"role": "agent", "n_turns": 1, "messages": [{"role": "user", "content": "a"}]},
+ {"role": "agent", "n_turns": 1, "messages": [{"role": "user", "content": "b"}]},
+ ]
+ out = ui._conversation_html({"conversations": convos})
+ assert out.count("main conversation") == 0
+ assert "conversation 1 of 2" in out and "conversation 2 of 2" in out
+
+
+def test_turns_html_does_not_show_the_tools_offered_count():
+ """That number is a property of the harness, identical on every row, and told nobody anything."""
+ out = ui._turns_html(
+ {
+ "turns": [
+ {
+ "turn": 0,
+ "completion_token_ids": [1, 2],
+ "per_token_logps": [-0.1, -0.2],
+ "tool_calls": [{"name": "bash", "arguments": "ls"}],
+ "n_tools": 24,
+ "finish_reason": "tool_calls",
+ }
+ ]
+ }
+ )
+ assert "24 tools" not in out
+ assert "bash" in out and "confidence" in out
+
+
+def test_escaping_prevents_markup_injection_from_a_model_reply():
+ out = ui._conversation_html(
+ {
+ "conversations": [
+ {
+ "role": "agent",
+ "n_turns": 1,
+ "messages": [
+ {"role": "assistant", "content": ""}
+ ],
+ }
+ ]
+ }
+ )
+ assert "